From c9b201f6499572d54135b908933c4e4681c936ea Mon Sep 17 00:00:00 2001 From: Brock Date: Wed, 29 Nov 2023 08:49:34 -0800 Subject: [PATCH] TST: dt64 units --- pandas/core/tools/datetimes.py | 6 ++-- pandas/tests/arithmetic/test_timedelta64.py | 10 ------- pandas/tests/arrays/test_array.py | 6 ++-- .../frame/constructors/test_from_records.py | 21 +++++++------- pandas/tests/frame/methods/test_replace.py | 14 ++++++---- .../tests/frame/methods/test_reset_index.py | 5 +++- .../indexes/datetimes/test_constructors.py | 4 +-- pandas/tests/io/json/test_pandas.py | 3 +- pandas/tests/reshape/concat/test_datetimes.py | 1 - pandas/tests/reshape/test_pivot.py | 6 ++-- pandas/tests/series/indexing/test_setitem.py | 7 +++-- pandas/tests/tools/test_to_datetime.py | 28 ++++++++++++------- pandas/tests/tools/test_to_timedelta.py | 11 ++++++++ 13 files changed, 73 insertions(+), 49 deletions(-) diff --git a/pandas/core/tools/datetimes.py b/pandas/core/tools/datetimes.py index 149bc2d932f0e..26cbc77e4e8ae 100644 --- a/pandas/core/tools/datetimes.py +++ b/pandas/core/tools/datetimes.py @@ -467,13 +467,15 @@ def _array_strptime_with_fallback( """ result, tz_out = array_strptime(arg, fmt, exact=exact, errors=errors, utc=utc) if tz_out is not None: - dtype = DatetimeTZDtype(tz=tz_out) + unit = np.datetime_data(result.dtype)[0] + dtype = DatetimeTZDtype(tz=tz_out, unit=unit) dta = DatetimeArray._simple_new(result, dtype=dtype) if utc: dta = dta.tz_convert("UTC") return Index(dta, name=name) elif result.dtype != object and utc: - res = Index(result, dtype="M8[ns, UTC]", name=name) + unit = np.datetime_data(result.dtype)[0] + res = Index(result, dtype=f"M8[{unit}, UTC]", name=name) return res return Index(result, dtype=result.dtype, name=name) diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index e88bde07aee90..73c6b4a1b2a0d 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -746,20 +746,10 @@ def test_timedelta_ops_with_missing_values(self): s1 = pd.to_timedelta(Series(["00:00:01"])) s2 = pd.to_timedelta(Series(["00:00:02"])) - msg = r"dtype datetime64\[ns\] cannot be converted to timedelta64\[ns\]" - with pytest.raises(TypeError, match=msg): - # Passing datetime64-dtype data to TimedeltaIndex is no longer - # supported GH#29794 - pd.to_timedelta(Series([NaT])) # TODO: belongs elsewhere? - sn = pd.to_timedelta(Series([NaT], dtype="m8[ns]")) df1 = DataFrame(["00:00:01"]).apply(pd.to_timedelta) df2 = DataFrame(["00:00:02"]).apply(pd.to_timedelta) - with pytest.raises(TypeError, match=msg): - # Passing datetime64-dtype data to TimedeltaIndex is no longer - # supported GH#29794 - DataFrame([NaT]).apply(pd.to_timedelta) # TODO: belongs elsewhere? dfn = DataFrame([NaT._value]).apply(pd.to_timedelta) diff --git a/pandas/tests/arrays/test_array.py b/pandas/tests/arrays/test_array.py index eb6e93b490574..e2b8ebcb79a3b 100644 --- a/pandas/tests/arrays/test_array.py +++ b/pandas/tests/arrays/test_array.py @@ -268,7 +268,7 @@ def test_array_copy(): ), ( [datetime.datetime(2000, 1, 1), datetime.datetime(2001, 1, 1)], - DatetimeArray._from_sequence(["2000", "2001"]), + DatetimeArray._from_sequence(["2000", "2001"], dtype="M8[ns]"), ), ( np.array([1, 2], dtype="M8[ns]"), @@ -284,7 +284,7 @@ def test_array_copy(): ( [pd.Timestamp("2000", tz="CET"), pd.Timestamp("2001", tz="CET")], DatetimeArray._from_sequence( - ["2000", "2001"], dtype=pd.DatetimeTZDtype(tz="CET") + ["2000", "2001"], dtype=pd.DatetimeTZDtype(tz="CET", unit="ns") ), ), ( @@ -293,7 +293,7 @@ def test_array_copy(): datetime.datetime(2001, 1, 1, tzinfo=cet), ], DatetimeArray._from_sequence( - ["2000", "2001"], dtype=pd.DatetimeTZDtype(tz=cet) + ["2000", "2001"], dtype=pd.DatetimeTZDtype(tz=cet, unit="ns") ), ), # timedelta diff --git a/pandas/tests/frame/constructors/test_from_records.py b/pandas/tests/frame/constructors/test_from_records.py index 4ad4e29550d56..bcf4e8fb0e64a 100644 --- a/pandas/tests/frame/constructors/test_from_records.py +++ b/pandas/tests/frame/constructors/test_from_records.py @@ -442,26 +442,27 @@ def test_from_records_misc_brokenness(self): exp = DataFrame(data, index=["a", "b", "c"]) tm.assert_frame_equal(result, exp) + def test_from_records_misc_brokenness2(self): # GH#2623 rows = [] rows.append([datetime(2010, 1, 1), 1]) rows.append([datetime(2010, 1, 2), "hi"]) # test col upconverts to obj - df2_obj = DataFrame.from_records(rows, columns=["date", "test"]) - result = df2_obj.dtypes - expected = Series( - [np.dtype("datetime64[ns]"), np.dtype("object")], index=["date", "test"] + result = DataFrame.from_records(rows, columns=["date", "test"]) + expected = DataFrame( + {"date": [row[0] for row in rows], "test": [row[1] for row in rows]} ) - tm.assert_series_equal(result, expected) + tm.assert_frame_equal(result, expected) + assert result.dtypes["test"] == np.dtype(object) + def test_from_records_misc_brokenness3(self): rows = [] rows.append([datetime(2010, 1, 1), 1]) rows.append([datetime(2010, 1, 2), 1]) - df2_obj = DataFrame.from_records(rows, columns=["date", "test"]) - result = df2_obj.dtypes - expected = Series( - [np.dtype("datetime64[ns]"), np.dtype("int64")], index=["date", "test"] + result = DataFrame.from_records(rows, columns=["date", "test"]) + expected = DataFrame( + {"date": [row[0] for row in rows], "test": [row[1] for row in rows]} ) - tm.assert_series_equal(result, expected) + tm.assert_frame_equal(result, expected) def test_from_records_empty(self): # GH#3562 diff --git a/pandas/tests/frame/methods/test_replace.py b/pandas/tests/frame/methods/test_replace.py index f07c53060a06b..4b32d3de59ca2 100644 --- a/pandas/tests/frame/methods/test_replace.py +++ b/pandas/tests/frame/methods/test_replace.py @@ -809,11 +809,13 @@ def test_replace_for_new_dtypes(self, datetime_frame): Timestamp("20130104", tz="US/Eastern"), DataFrame( { - "A": [ - Timestamp("20130101", tz="US/Eastern"), - Timestamp("20130104", tz="US/Eastern"), - Timestamp("20130103", tz="US/Eastern"), - ], + "A": pd.DatetimeIndex( + [ + Timestamp("20130101", tz="US/Eastern"), + Timestamp("20130104", tz="US/Eastern"), + Timestamp("20130103", tz="US/Eastern"), + ] + ).as_unit("ns"), "B": [0, np.nan, 2], } ), @@ -1174,6 +1176,7 @@ def test_replace_datetimetz(self): "B": [0, np.nan, 2], } ) + expected["A"] = expected["A"].dt.as_unit("ns") tm.assert_frame_equal(result, expected) result = df.copy() @@ -1195,6 +1198,7 @@ def test_replace_datetimetz(self): "B": [0, np.nan, 2], } ) + expected["A"] = expected["A"].dt.as_unit("ns") tm.assert_frame_equal(result, expected) result = df.copy() diff --git a/pandas/tests/frame/methods/test_reset_index.py b/pandas/tests/frame/methods/test_reset_index.py index 377128ee12ee6..c989b3d26677c 100644 --- a/pandas/tests/frame/methods/test_reset_index.py +++ b/pandas/tests/frame/methods/test_reset_index.py @@ -699,9 +699,12 @@ def test_reset_index_multiindex_nat(): df = DataFrame({"id": idx, "tstamp": tstamp, "a": list("abc")}) df.loc[2, "tstamp"] = pd.NaT result = df.set_index(["id", "tstamp"]).reset_index("id") + exp_dti = pd.DatetimeIndex( + ["2015-07-01", "2015-07-02", "NaT"], dtype="M8[ns]", name="tstamp" + ) expected = DataFrame( {"id": range(3), "a": list("abc")}, - index=pd.DatetimeIndex(["2015-07-01", "2015-07-02", "NaT"], name="tstamp"), + index=exp_dti, ) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/indexes/datetimes/test_constructors.py b/pandas/tests/indexes/datetimes/test_constructors.py index 4a0b192244dc8..73969457135f0 100644 --- a/pandas/tests/indexes/datetimes/test_constructors.py +++ b/pandas/tests/indexes/datetimes/test_constructors.py @@ -592,13 +592,13 @@ def test_integer_values_and_tz_interpreted_as_utc(self): result = DatetimeIndex(values).tz_localize("US/Central") - expected = DatetimeIndex(["2000-01-01T00:00:00"], tz="US/Central") + expected = DatetimeIndex(["2000-01-01T00:00:00"], dtype="M8[ns, US/Central]") tm.assert_index_equal(result, expected) # but UTC is *not* deprecated. with tm.assert_produces_warning(None): result = DatetimeIndex(values, tz="UTC") - expected = DatetimeIndex(["2000-01-01T00:00:00"], tz="UTC") + expected = DatetimeIndex(["2000-01-01T00:00:00"], dtype="M8[ns, UTC]") tm.assert_index_equal(result, expected) def test_constructor_coverage(self): diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index 411cc90ba41a7..5275050391ca3 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1625,7 +1625,8 @@ def test_read_timezone_information(self): result = read_json( StringIO('{"2019-01-01T11:00:00.000Z":88}'), typ="series", orient="index" ) - expected = Series([88], index=DatetimeIndex(["2019-01-01 11:00:00"], tz="UTC")) + exp_dti = DatetimeIndex(["2019-01-01 11:00:00"], dtype="M8[ns, UTC]") + expected = Series([88], index=exp_dti) tm.assert_series_equal(result, expected) @pytest.mark.parametrize( diff --git a/pandas/tests/reshape/concat/test_datetimes.py b/pandas/tests/reshape/concat/test_datetimes.py index c00a2dc92a52b..71ddff7438254 100644 --- a/pandas/tests/reshape/concat/test_datetimes.py +++ b/pandas/tests/reshape/concat/test_datetimes.py @@ -61,7 +61,6 @@ def test_concat_datetime_timezone(self): dtype="M8[ns, Europe/Paris]", freq="h", ) - expected = DataFrame( [[1, 1], [2, 2], [3, 3]], index=exp_idx, columns=["a", "b"] ) diff --git a/pandas/tests/reshape/test_pivot.py b/pandas/tests/reshape/test_pivot.py index f8ececf6c0540..4a852daaadf98 100644 --- a/pandas/tests/reshape/test_pivot.py +++ b/pandas/tests/reshape/test_pivot.py @@ -437,8 +437,10 @@ def test_pivot_no_values(self): index=idx, ) res = df.pivot_table(index=df.index.month, columns=Grouper(key="dt", freq="ME")) - exp_columns = MultiIndex.from_tuples([("A", pd.Timestamp("2011-01-31"))]) - exp_columns.names = [None, "dt"] + exp_columns = MultiIndex.from_arrays( + [["A"], pd.DatetimeIndex(["2011-01-31"], dtype="M8[ns]")], + names=[None, "dt"], + ) exp = DataFrame( [3.25, 2.0], index=Index([1, 2], dtype=np.int32), columns=exp_columns ) diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index 0f3577a214186..fce0581260210 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -88,7 +88,8 @@ def test_setitem_with_tz(self, tz, indexer_sli): Timestamp("2016-01-01 00:00", tz=tz), Timestamp("2011-01-01 00:00", tz=tz), Timestamp("2016-01-01 02:00", tz=tz), - ] + ], + dtype=orig.dtype, ) # scalar @@ -100,6 +101,7 @@ def test_setitem_with_tz(self, tz, indexer_sli): vals = Series( [Timestamp("2011-01-01", tz=tz), Timestamp("2012-01-01", tz=tz)], index=[1, 2], + dtype=orig.dtype, ) assert vals.dtype == f"datetime64[ns, {tz}]" @@ -108,7 +110,8 @@ def test_setitem_with_tz(self, tz, indexer_sli): Timestamp("2016-01-01 00:00", tz=tz), Timestamp("2011-01-01 00:00", tz=tz), Timestamp("2012-01-01 00:00", tz=tz), - ] + ], + dtype=orig.dtype, ) ser = orig.copy() diff --git a/pandas/tests/tools/test_to_datetime.py b/pandas/tests/tools/test_to_datetime.py index 8139fe52c7037..f74fe459eb4d6 100644 --- a/pandas/tests/tools/test_to_datetime.py +++ b/pandas/tests/tools/test_to_datetime.py @@ -2058,7 +2058,11 @@ def test_to_datetime_unit(self, dtype): ser = Series([epoch + t for t in range(20)]).astype(dtype) result = to_datetime(ser, unit="s") expected = Series( - [Timestamp("2013-06-09 02:42:28") + timedelta(seconds=t) for t in range(20)] + [ + Timestamp("2013-06-09 02:42:28") + timedelta(seconds=t) + for t in range(20) + ], + dtype="M8[ns]", ) tm.assert_series_equal(result, expected) @@ -2208,7 +2212,8 @@ def test_dataframe_field_aliases_column_subset(self, df, cache, unit): # unit mappings result = to_datetime(df[list(unit.keys())].rename(columns=unit), cache=cache) expected = Series( - [Timestamp("20150204 06:58:10"), Timestamp("20160305 07:59:11")] + [Timestamp("20150204 06:58:10"), Timestamp("20160305 07:59:11")], + dtype="M8[ns]", ) tm.assert_series_equal(result, expected) @@ -2970,7 +2975,8 @@ def test_to_datetime_iso8601_noleading_0s(self, cache, format): Timestamp("2015-03-03"), ] ) - tm.assert_series_equal(to_datetime(ser, format=format, cache=cache), expected) + result = to_datetime(ser, format=format, cache=cache) + tm.assert_series_equal(result, expected) def test_parse_dates_infer_datetime_format_warning(self): # GH 49024 @@ -3364,7 +3370,8 @@ def test_julian(self, julian_dates): def test_unix(self): result = Series(to_datetime([0, 1, 2], unit="D", origin="unix")) expected = Series( - [Timestamp("1970-01-01"), Timestamp("1970-01-02"), Timestamp("1970-01-03")] + [Timestamp("1970-01-01"), Timestamp("1970-01-02"), Timestamp("1970-01-03")], + dtype="M8[ns]", ) tm.assert_series_equal(result, expected) @@ -3483,7 +3490,7 @@ def test_arg_tz_ns_unit(self, offset, utc, exp): # GH 25546 arg = "2019-01-01T00:00:00.000" + offset result = to_datetime([arg], unit="ns", utc=utc) - expected = to_datetime([exp]) + expected = to_datetime([exp]).as_unit("ns") tm.assert_index_equal(result, expected) @@ -3610,11 +3617,12 @@ def test_to_datetime_monotonic_increasing_index(cache): ) def test_to_datetime_cache_coerce_50_lines_outofbounds(series_length): # GH#45319 - s = Series( + ser = Series( [datetime.fromisoformat("1446-04-12 00:00:00+00:00")] - + ([datetime.fromisoformat("1991-10-20 00:00:00+00:00")] * series_length) + + ([datetime.fromisoformat("1991-10-20 00:00:00+00:00")] * series_length), + dtype=object, ) - result1 = to_datetime(s, errors="coerce", utc=True) + result1 = to_datetime(ser, errors="coerce", utc=True) expected1 = Series( [NaT] + ([Timestamp("1991-10-20 00:00:00+00:00")] * series_length) @@ -3622,7 +3630,7 @@ def test_to_datetime_cache_coerce_50_lines_outofbounds(series_length): tm.assert_series_equal(result1, expected1) - result2 = to_datetime(s, errors="ignore", utc=True) + result2 = to_datetime(ser, errors="ignore", utc=True) expected2 = Series( [datetime.fromisoformat("1446-04-12 00:00:00+00:00")] @@ -3632,7 +3640,7 @@ def test_to_datetime_cache_coerce_50_lines_outofbounds(series_length): tm.assert_series_equal(result2, expected2) with pytest.raises(OutOfBoundsDatetime, match="Out of bounds nanosecond timestamp"): - to_datetime(s, errors="raise", utc=True) + to_datetime(ser, errors="raise", utc=True) def test_to_datetime_format_f_parse_nanos(): diff --git a/pandas/tests/tools/test_to_timedelta.py b/pandas/tests/tools/test_to_timedelta.py index e588bc83b0de8..b3d4d9d67190f 100644 --- a/pandas/tests/tools/test_to_timedelta.py +++ b/pandas/tests/tools/test_to_timedelta.py @@ -21,6 +21,17 @@ class TestTimedeltas: + def test_to_timedelta_dt64_raises(self): + # Passing datetime64-dtype data to TimedeltaIndex is no longer + # supported GH#29794 + msg = r"dtype datetime64\[ns\] cannot be converted to timedelta64\[ns\]" + + ser = Series([pd.NaT]) + with pytest.raises(TypeError, match=msg): + to_timedelta(ser) + with pytest.raises(TypeError, match=msg): + ser.to_frame().apply(to_timedelta) + @pytest.mark.parametrize("readonly", [True, False]) def test_to_timedelta_readonly(self, readonly): # GH#34857