From 2580d1fd4f73e822778c134083c7a62251860cec Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 13 Dec 2024 13:46:35 -0800 Subject: [PATCH 1/6] Debug free threading build --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 07fb0c19262a1..5cc9b54371f9c 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -387,7 +387,7 @@ jobs: - name: Build Environment run: | python --version - python -m pip install --upgrade pip setuptools wheel numpy meson[ninja]==1.2.1 meson-python==0.13.1 + python -m pip install --upgrade pip setuptools wheel numpy=2.1 meson[ninja]==1.2.1 meson-python==0.13.1 python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple cython python -m pip install versioneer[toml] python -m pip install python-dateutil pytz tzdata hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov From 9fe5614368e1a2a438d0793fdc01342e8196167f Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:05:02 -0800 Subject: [PATCH 2/6] Double = --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5cc9b54371f9c..58ca11de06657 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -387,7 +387,7 @@ jobs: - name: Build Environment run: | python --version - python -m pip install --upgrade pip setuptools wheel numpy=2.1 meson[ninja]==1.2.1 meson-python==0.13.1 + python -m pip install --upgrade pip setuptools wheel numpy==2.1 meson[ninja]==1.2.1 meson-python==0.13.1 python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple cython python -m pip install versioneer[toml] python -m pip install python-dateutil pytz tzdata hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov From a403d278000d0a588faf6915279ed1fa1f73df18 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:48:26 -0800 Subject: [PATCH 3/6] skip test_replace_regex_inplace_refs for numpy 2.2 and ft --- .github/workflows/unit-tests.yml | 2 +- pandas/compat/__init__.py | 4 ++++ pandas/compat/_constants.py | 4 ++++ pandas/compat/numpy/__init__.py | 1 + pandas/tests/copy_view/test_replace.py | 7 +++++++ scripts/validate_unwanted_patterns.py | 3 ++- 6 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 58ca11de06657..07fb0c19262a1 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -387,7 +387,7 @@ jobs: - name: Build Environment run: | python --version - python -m pip install --upgrade pip setuptools wheel numpy==2.1 meson[ninja]==1.2.1 meson-python==0.13.1 + python -m pip install --upgrade pip setuptools wheel numpy meson[ninja]==1.2.1 meson-python==0.13.1 python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple cython python -m pip install versioneer[toml] python -m pip install python-dateutil pytz tzdata hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov diff --git a/pandas/compat/__init__.py b/pandas/compat/__init__.py index e7674386408f7..e472d81dd99b7 100644 --- a/pandas/compat/__init__.py +++ b/pandas/compat/__init__.py @@ -17,9 +17,11 @@ from pandas.compat._constants import ( IS64, + IS_FREE_THREADING, ISMUSL, PY311, PY312, + PY313, PYPY, WASM, ) @@ -153,8 +155,10 @@ def is_ci_environment() -> bool: "HAS_PYARROW", "IS64", "ISMUSL", + "IS_FREE_THREADING", "PY311", "PY312", + "PY313", "PYPY", "WASM", "is_numpy_dev", diff --git a/pandas/compat/_constants.py b/pandas/compat/_constants.py index c7b7341013251..0ed75397d3225 100644 --- a/pandas/compat/_constants.py +++ b/pandas/compat/_constants.py @@ -15,16 +15,20 @@ PY311 = sys.version_info >= (3, 11) PY312 = sys.version_info >= (3, 12) +PY313 = sys.version_info >= (3, 13) PYPY = platform.python_implementation() == "PyPy" WASM = (sys.platform == "emscripten") or (platform.machine() in ["wasm32", "wasm64"]) +IS_FREE_THREADING = False if not PY313 else sys._is_gil_enabled() ISMUSL = "musl" in (sysconfig.get_config_var("HOST_GNU_TYPE") or "") REF_COUNT = 2 if PY311 else 3 __all__ = [ "IS64", "ISMUSL", + "IS_FREE_THREADING", "PY311", "PY312", + "PY313", "PYPY", "WASM", ] diff --git a/pandas/compat/numpy/__init__.py b/pandas/compat/numpy/__init__.py index 3306b36d71806..af93db7833dfe 100644 --- a/pandas/compat/numpy/__init__.py +++ b/pandas/compat/numpy/__init__.py @@ -13,6 +13,7 @@ np_version_gte1p24p3 = _nlv >= Version("1.24.3") np_version_gte1p25 = _nlv >= Version("1.25") np_version_gt2 = _nlv >= Version("2.0.0") +np_version_gt2p2 = _nlv >= Version("2.2.0") is_numpy_dev = _nlv.dev is not None _min_numpy_ver = "1.23.5" diff --git a/pandas/tests/copy_view/test_replace.py b/pandas/tests/copy_view/test_replace.py index d4838a5e68ab8..2192913d8da2b 100644 --- a/pandas/tests/copy_view/test_replace.py +++ b/pandas/tests/copy_view/test_replace.py @@ -1,6 +1,9 @@ import numpy as np import pytest +from pandas.compat import IS_FREE_THREADING +from pandas.compat.numpy import np_version_gt2p2 + from pandas import ( Categorical, DataFrame, @@ -45,6 +48,10 @@ def test_replace(replace_kwargs): tm.assert_frame_equal(df, df_orig) +@pytest.mark.skipif( + IS_FREE_THREADING and np_version_gt2p2, + reason="Segfaults in array_algos.replace.replace_regex", +) def test_replace_regex_inplace_refs(): df = DataFrame({"a": ["aaa", "bbb"]}) df_orig = df.copy() diff --git a/scripts/validate_unwanted_patterns.py b/scripts/validate_unwanted_patterns.py index d804e15f6d48f..60ada0db8a751 100755 --- a/scripts/validate_unwanted_patterns.py +++ b/scripts/validate_unwanted_patterns.py @@ -89,7 +89,8 @@ def _get_literal_string_prefix_len(token_string: str) -> int: return 0 -PRIVATE_FUNCTIONS_ALLOWED = {"sys._getframe"} # no known alternative + # no known alternative +PRIVATE_FUNCTIONS_ALLOWED = {"sys._getframe", "sys._is_gil_enabled"} def private_function_across_module(file_obj: IO[str]) -> Iterable[tuple[int, str]]: From 5a6052b9a3dfa9354b2fba10c26db72d1c42a1fe Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:20:41 -0800 Subject: [PATCH 4/6] Ignore typing --- pandas/compat/_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/compat/_constants.py b/pandas/compat/_constants.py index 0ed75397d3225..9e62c28d88d73 100644 --- a/pandas/compat/_constants.py +++ b/pandas/compat/_constants.py @@ -18,7 +18,7 @@ PY313 = sys.version_info >= (3, 13) PYPY = platform.python_implementation() == "PyPy" WASM = (sys.platform == "emscripten") or (platform.machine() in ["wasm32", "wasm64"]) -IS_FREE_THREADING = False if not PY313 else sys._is_gil_enabled() +IS_FREE_THREADING = False if not PY313 else sys._is_gil_enabled() # type: ignore[attr-defined] ISMUSL = "musl" in (sysconfig.get_config_var("HOST_GNU_TYPE") or "") REF_COUNT = 2 if PY311 else 3 From a8661496c6340c0d6f4f1adefa9ec5ec5068e2d7 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:37:58 -0800 Subject: [PATCH 5/6] reverse condition --- pandas/compat/_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/compat/_constants.py b/pandas/compat/_constants.py index 9e62c28d88d73..8c817c1b6e386 100644 --- a/pandas/compat/_constants.py +++ b/pandas/compat/_constants.py @@ -18,7 +18,7 @@ PY313 = sys.version_info >= (3, 13) PYPY = platform.python_implementation() == "PyPy" WASM = (sys.platform == "emscripten") or (platform.machine() in ["wasm32", "wasm64"]) -IS_FREE_THREADING = False if not PY313 else sys._is_gil_enabled() # type: ignore[attr-defined] +IS_FREE_THREADING = False if not PY313 else not sys._is_gil_enabled() # type: ignore[attr-defined] ISMUSL = "musl" in (sysconfig.get_config_var("HOST_GNU_TYPE") or "") REF_COUNT = 2 if PY311 else 3 From 69b4716511d1f4a2f40b3994ac91456abc527a14 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 16 Dec 2024 11:29:55 -0800 Subject: [PATCH 6/6] More skips --- pandas/tests/arrays/string_/test_string.py | 6 ++++++ pandas/tests/copy_view/test_replace.py | 4 ++++ pandas/tests/series/methods/test_astype.py | 10 ++++++++++ pandas/tests/series/methods/test_replace.py | 7 +++++++ 4 files changed, 27 insertions(+) diff --git a/pandas/tests/arrays/string_/test_string.py b/pandas/tests/arrays/string_/test_string.py index a32ac7db4656a..e5c31ec28b4ca 100644 --- a/pandas/tests/arrays/string_/test_string.py +++ b/pandas/tests/arrays/string_/test_string.py @@ -10,6 +10,8 @@ from pandas._config import using_string_dtype +from pandas.compat import IS_FREE_THREADING +from pandas.compat.numpy import np_version_gt2p2 from pandas.compat.pyarrow import pa_version_under12p0 from pandas.core.dtypes.common import is_dtype_equal @@ -140,6 +142,10 @@ def test_setitem_with_array_with_missing(dtype): tm.assert_numpy_array_equal(value, value_orig) +@pytest.mark.skipif( + IS_FREE_THREADING and np_version_gt2p2, + reason="Segfaults in TimedeltaArray._format_native_types", +) def test_astype_roundtrip(dtype): ser = pd.Series(pd.date_range("2000", periods=12)) ser[0] = None diff --git a/pandas/tests/copy_view/test_replace.py b/pandas/tests/copy_view/test_replace.py index 2192913d8da2b..6022adc6ce490 100644 --- a/pandas/tests/copy_view/test_replace.py +++ b/pandas/tests/copy_view/test_replace.py @@ -63,6 +63,10 @@ def test_replace_regex_inplace_refs(): tm.assert_frame_equal(view, df_orig) +@pytest.mark.skipif( + IS_FREE_THREADING and np_version_gt2p2, + reason="Segfaults in array_algos.replace.replace_regex", +) def test_replace_regex_inplace(): df = DataFrame({"a": ["aaa", "bbb"]}) arr = get_array(df, "a") diff --git a/pandas/tests/series/methods/test_astype.py b/pandas/tests/series/methods/test_astype.py index 4a7e204ee4161..200120e22be27 100644 --- a/pandas/tests/series/methods/test_astype.py +++ b/pandas/tests/series/methods/test_astype.py @@ -10,6 +10,8 @@ import pytest from pandas._libs.tslibs import iNaT +from pandas.compat import IS_FREE_THREADING +from pandas.compat.numpy import np_version_gt2p2 import pandas.util._test_decorators as td from pandas import ( @@ -299,6 +301,10 @@ def test_astype_str_cast_dt64(self): expected = Series(["2010-01-04 00:00:00-05:00"], dtype="str") tm.assert_series_equal(res, expected) + @pytest.mark.skipif( + IS_FREE_THREADING and np_version_gt2p2, + reason="Segfaults in TimedeltaArray._format_native_types", + ) def test_astype_str_cast_td64(self): # see GH#9757 @@ -493,6 +499,10 @@ def test_astype_retain_attrs(self, any_numpy_dtype): class TestAstypeString: + @pytest.mark.skipif( + IS_FREE_THREADING and np_version_gt2p2, + reason="Segfaults in TimedeltaArray._format_native_types", + ) @pytest.mark.parametrize( "data, dtype", [ diff --git a/pandas/tests/series/methods/test_replace.py b/pandas/tests/series/methods/test_replace.py index ecfe3d1b39d31..240f8a986e63a 100644 --- a/pandas/tests/series/methods/test_replace.py +++ b/pandas/tests/series/methods/test_replace.py @@ -3,6 +3,9 @@ import numpy as np import pytest +from pandas.compat import IS_FREE_THREADING +from pandas.compat.numpy import np_version_gt2p2 + import pandas as pd import pandas._testing as tm from pandas.core.arrays import IntervalArray @@ -510,6 +513,10 @@ def test_replace_extension_other(self, frame_or_series): # should not have changed dtype tm.assert_equal(obj, result) + @pytest.mark.skipif( + IS_FREE_THREADING and np_version_gt2p2, + reason="Segfaults in array_algos.replace.compare_or_regex_search", + ) def test_replace_with_compiled_regex(self): # https://github.com/pandas-dev/pandas/issues/35680 s = pd.Series(["a", "b", "c"])