Backport PR #56308 on branch 2.1.x (REGR: Fix reading old pickles fro…

…m pandas 1.3.5) (#56335) Backport PR #56308: REGR: Fix reading old pickles from pandas 1.3.5 Co-authored-by: Thomas Li <[email protected]>
pandas-dev · Dec 5, 2023 · 3ee9b3e · 3ee9b3e
1 parent c6efdb0
commit 3ee9b3e
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 2 deletions.
diff --git a/doc/source/whatsnew/v2.1.4.rst b/doc/source/whatsnew/v2.1.4.rst
@@ -13,7 +13,7 @@ including other versions of pandas.
 
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
--
+- Fixed regression when trying to read a pickled pandas :class:`DataFrame` from pandas 1.3 (:issue:`55137`)
 -
 
 .. ---------------------------------------------------------------------------

diff --git a/pandas/compat/pickle_compat.py b/pandas/compat/pickle_compat.py
@@ -65,6 +65,12 @@ def load_reduce(self):
     ("pandas.core.sparse.array", "SparseArray"): ("pandas.core.arrays", "SparseArray"),
     # 15477
     ("pandas.core.base", "FrozenNDArray"): ("numpy", "ndarray"),
+    # Re-routing unpickle block logic to go through _unpickle_block instead
+    # for pandas <= 1.3.5
+    ("pandas.core.internals.blocks", "new_block"): (
+        "pandas._libs.internals",
+        "_unpickle_block",
+    ),
     ("pandas.core.indexes.frozen", "FrozenNDArray"): ("numpy", "ndarray"),
     ("pandas.core.base", "FrozenList"): ("pandas.core.indexes.frozen", "FrozenList"),
     # 10890

diff --git a/pandas/tests/io/data/legacy_pickle/1.3.5/1.3.5_x86_64_darwin_3.10.13.pickle b/pandas/tests/io/data/legacy_pickle/1.3.5/1.3.5_x86_64_darwin_3.10.13.pickle
diff --git a/pandas/tests/io/generate_legacy_storage_files.py b/pandas/tests/io/generate_legacy_storage_files.py
@@ -38,6 +38,10 @@
 import platform as pl
 import sys
 
+# Remove script directory from path, otherwise Python will try to
+# import the JSON test directory as the json module
+sys.path.pop(0)
+
 import numpy as np
 
 import pandas
@@ -320,7 +324,7 @@ def write_legacy_pickles(output_dir):
 
 def write_legacy_file():
     # force our cwd to be the first searched
-    sys.path.insert(0, ".")
+    sys.path.insert(0, "")
 
     if not 3 <= len(sys.argv) <= 4:
         sys.exit(
@@ -331,6 +335,9 @@ def write_legacy_file():
     output_dir = str(sys.argv[1])
     storage_type = str(sys.argv[2])
 
+    if not os.path.exists(output_dir):
+        os.mkdir(output_dir)
+
     if storage_type == "pickle":
         write_legacy_pickles(output_dir=output_dir)
     else: