-
-
Notifications
You must be signed in to change notification settings - Fork 18.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Handling conversion of empty categorical with dtype_backend='pyarrow' #59935
Changes from 1 commit
e6ef750
c005850
5ce426d
94769a1
8e879da
a5b3882
c1ed1f0
6747736
82cecd1
ae0a148
c0d8178
902601f
482080d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -1112,7 +1112,7 @@ def convert_dtypes( | |
|
||
else: | ||
inferred_dtype = input_array.dtype | ||
|
||
if dtype_backend == "pyarrow": | ||
from pandas.core.arrays.arrow.array import to_pyarrow_type | ||
from pandas.core.arrays.string_ import StringDtype | ||
|
@@ -1145,12 +1145,16 @@ def convert_dtypes( | |
and isna(input_array).all() | ||
): | ||
import pyarrow as pa | ||
|
||
pa_type = pa.null() | ||
else: | ||
pa_type = to_pyarrow_type(base_dtype) | ||
if pa_type is not None: | ||
inferred_dtype = ArrowDtype(pa_type) | ||
if isna(input_array).all() and hasattr(input_array, 'categories'): | ||
inferred_dtype = input_array.dtype | ||
else: | ||
inferred_dtype = ArrowDtype(pa_type) | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you revert the addition of this newline |
||
elif dtype_backend == "numpy_nullable" and isinstance(inferred_dtype, ArrowDtype): | ||
# GH 53648 | ||
inferred_dtype = _arrow_dtype_mapping()[inferred_dtype.pyarrow_dtype] | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -34,6 +34,22 @@ def test_convert_empty(self): | |
# Empty DataFrame can pass convert_dtypes, see GH#40393 | ||
empty_df = pd.DataFrame() | ||
tm.assert_frame_equal(empty_df, empty_df.convert_dtypes()) | ||
|
||
def test_convert_empty_categorical_to_pyarrow(self): | ||
rhshadrach marked this conversation as resolved.
Show resolved
Hide resolved
|
||
df = pd.DataFrame( | ||
rhshadrach marked this conversation as resolved.
Show resolved
Hide resolved
|
||
{ | ||
"A": pd.Series(pd.Categorical([None] * 5)), | ||
"B": pd.Series(pd.Categorical([None] * 5, categories=["B1", "B2"])), | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No need to wrap in |
||
} | ||
) | ||
converted = df.convert_dtypes(dtype_backend="pyarrow") | ||
expected = df | ||
tm.assert_frame_equal(converted, expected) | ||
|
||
assert df.A.dtype == "category", "Dtype in column A is not 'category'" | ||
assert df.B.dtype == "category", "Dtype in column B is not 'category'" | ||
assert df.A.cat.categories.empty, "Categories in column A are not empty" | ||
assert (df.B.cat.categories == ["B1", "B2"]).all(), "Categories in column A are not empty" | ||
|
||
def test_convert_dtypes_retain_column_names(self): | ||
# GH#41435 | ||
|
Original file line number | Diff line number | Diff line change | ||||||
---|---|---|---|---|---|---|---|---|
|
@@ -297,3 +297,16 @@ def test_convert_dtypes_pyarrow_null(self): | |||||||
result = ser.convert_dtypes(dtype_backend="pyarrow") | ||||||||
expected = pd.Series([None, None], dtype=pd.ArrowDtype(pa.null())) | ||||||||
tm.assert_series_equal(result, expected) | ||||||||
|
||||||||
def test_convert_empty_categorical_to_pyarrow(self): | ||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||
ser = pd.Series(pd.Series(pd.Categorical([None] * 5))) | ||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Similar - You can remove one |
||||||||
|
||||||||
converted = ser.convert_dtypes(dtype_backend="pyarrow") | ||||||||
expected = ser | ||||||||
tm.assert_series_equal(converted, expected) | ||||||||
|
||||||||
assert ser.dtype == "category", "Series dtype is not 'category'" | ||||||||
assert ser.cat.categories.empty, "Series categories are not empty" | ||||||||
|
||||||||
ser2 = pd.Series(pd.Series(pd.Categorical([None] * 5, categories=["S1", "S2"]))) | ||||||||
assert (ser2.cat.categories == ["S1", "S2"]).all(), "Series categories are not empty" | ||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What is this testing? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. here we need to make sure that the set categories in empty categorical are propagated after conversion... i've adapted this in new pr |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Instead of this, can you add the check
not isinstance(input_array.dtype, CategoricalDtype)
to the if statement that starts on L1142. Then in the case to categorical dtypes,to_pyarrow_type
returnsNone
on L1151 and we do not modifyinferred_dtype
.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
good point!