Skip to content

Commit

Permalink
BUG: setitem with mixed-resolution dt64s
Browse files Browse the repository at this point in the history
  • Loading branch information
jbrockmendel committed Dec 8, 2023
1 parent 46c8da3 commit ed2d1b8
Show file tree
Hide file tree
Showing 6 changed files with 59 additions and 6 deletions.
1 change: 1 addition & 0 deletions doc/source/whatsnew/v2.2.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -545,6 +545,7 @@ Datetimelike
- Bug in creating a :class:`Index`, :class:`Series`, or :class:`DataFrame` with a non-nanosecond ``datetime64`` dtype and inputs that would be out of bounds for a ``datetime64[ns]`` incorrectly raising ``OutOfBoundsDatetime`` (:issue:`55756`)
- Bug in parsing datetime strings with nanosecond resolution with non-ISO8601 formats incorrectly truncating sub-microsecond components (:issue:`56051`)
- Bug in parsing datetime strings with sub-second resolution and trailing zeros incorrectly inferring second or millisecond resolution (:issue:`55737`)
- Bug in setting scalar values with mismatched resolution into arrays with non-nanosecond ``datetime64``, ``timedelta64`` or :class:`DatetimeTZDtype` incorrectly truncating those scalars (:issue:`56410`)
- Bug in the results of :func:`to_datetime` with an floating-dtype argument with ``unit`` not matching the pointwise results of :class:`Timestamp` (:issue:`56037`)
-

Expand Down
2 changes: 1 addition & 1 deletion pandas/core/arrays/datetimes.py
Original file line number Diff line number Diff line change
Expand Up @@ -540,7 +540,7 @@ def _unbox_scalar(self, value) -> np.datetime64:
if value is NaT:
return np.datetime64(value._value, self.unit)
else:
return value.as_unit(self.unit).asm8
return value.as_unit(self.unit, round_ok=False).asm8

def _scalar_from_string(self, value) -> Timestamp | NaTType:
return Timestamp(value, tz=self.tz)
Expand Down
2 changes: 1 addition & 1 deletion pandas/core/arrays/timedeltas.py
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,7 @@ def _unbox_scalar(self, value) -> np.timedelta64:
if value is NaT:
return np.timedelta64(value._value, self.unit)
else:
return value.as_unit(self.unit).asm8
return value.as_unit(self.unit, round_ok=False).asm8

def _scalar_from_string(self, value) -> Timedelta | NaTType:
return Timedelta(value)
Expand Down
2 changes: 2 additions & 0 deletions pandas/core/indexes/datetimes.py
Original file line number Diff line number Diff line change
Expand Up @@ -533,6 +533,8 @@ def _parsed_string_to_bounds(self, reso: Resolution, parsed: dt.datetime):
freq = OFFSET_TO_PERIOD_FREQSTR.get(reso.attr_abbrev, reso.attr_abbrev)
per = Period(parsed, freq=freq)
start, end = per.start_time, per.end_time
start = start.as_unit(self.unit)
end = end.as_unit(self.unit)

# GH 24076
# If an incoming date string contained a UTC offset, need to localize
Expand Down
33 changes: 33 additions & 0 deletions pandas/tests/series/indexing/test_setitem.py
Original file line number Diff line number Diff line change
Expand Up @@ -1469,6 +1469,39 @@ def test_slice_key(self, obj, key, expected, warn, val, indexer_sli, is_inplace)
raise AssertionError("xfail not relevant for this test.")


@pytest.mark.parametrize(
"exp_dtype",
[
"M8[ms]",
"M8[ms, UTC]",
"m8[ms]",
],
)
class TestCoercionDatetime64HigherReso(CoercionTest):
@pytest.fixture
def obj(self, exp_dtype):
idx = date_range("2011-01-01", freq="D", periods=4, unit="s")
if exp_dtype == "m8[ms]":
idx = idx - Timestamp("1970-01-01")
assert idx.dtype == "m8[s]"
elif exp_dtype == "M8[ms, UTC]":
idx = idx.tz_localize("UTC")
return Series(idx)

@pytest.fixture
def val(self, exp_dtype):
ts = Timestamp("2011-01-02 03:04:05.678").as_unit("ms")
if exp_dtype == "m8[ms]":
return ts - Timestamp("1970-01-01")
elif exp_dtype == "M8[ms, UTC]":
return ts.tz_localize("UTC")
return ts

@pytest.fixture
def warn(self):
return FutureWarning


@pytest.mark.parametrize(
"val,exp_dtype,warn",
[
Expand Down
25 changes: 21 additions & 4 deletions pandas/tests/series/methods/test_clip.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import numpy as np
import pytest

from pandas.errors import OutOfBoundsDatetime

import pandas as pd
from pandas import (
Series,
Expand Down Expand Up @@ -135,12 +137,27 @@ def test_clip_with_datetimes(self):
)
tm.assert_series_equal(result, expected)

@pytest.mark.parametrize("dtype", [object, "M8[us]"])
def test_clip_with_timestamps_and_oob_datetimes(self, dtype):
def test_clip_with_timestamps_and_oob_datetimes_object(self):
# GH-42794
ser = Series([datetime(1, 1, 1), datetime(9999, 9, 9)], dtype=dtype)
ser = Series([datetime(1, 1, 1), datetime(9999, 9, 9)], dtype=object)

result = ser.clip(lower=Timestamp.min, upper=Timestamp.max)
expected = Series([Timestamp.min, Timestamp.max], dtype=dtype)
expected = Series([Timestamp.min, Timestamp.max], dtype=object)

tm.assert_series_equal(result, expected)

def test_clip_with_timestamps_and_oob_datetimes_non_nano(self):
# GH#56410
dtype = "M8[us]"
ser = Series([datetime(1, 1, 1), datetime(9999, 9, 9)], dtype=dtype)

msg = "Out of bounds nanosecond timestamp"
with pytest.raises(OutOfBoundsDatetime, match=msg):
ser.clip(lower=Timestamp.min, upper=Timestamp.max)

lower = Timestamp.min.as_unit("us")
upper = Timestamp.max.as_unit("us")
result = ser.clip(lower=lower, upper=upper)
expected = Series([lower, upper], dtype=dtype)

tm.assert_series_equal(result, expected)

0 comments on commit ed2d1b8

Please sign in to comment.