Coverage for /home/martinb/.local/share/virtualenvs/camcops/lib/python3.6/site-packages/pandas/compat/pickle_compat.py : 31%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Support pre-0.12 series pickle compatibility.
3"""
5import copy
6import pickle as pkl
7from typing import TYPE_CHECKING, Optional
8import warnings
10from pandas import Index
12if TYPE_CHECKING:
13 from pandas import Series, DataFrame
16def load_reduce(self):
17 stack = self.stack
18 args = stack.pop()
19 func = stack[-1]
21 if len(args) and type(args[0]) is type:
22 n = args[0].__name__ # noqa
24 try:
25 stack[-1] = func(*args)
26 return
27 except TypeError as err:
29 # If we have a deprecated function,
30 # try to replace and try again.
32 msg = "_reconstruct: First argument must be a sub-type of ndarray"
34 if msg in str(err):
35 try:
36 cls = args[0]
37 stack[-1] = object.__new__(cls)
38 return
39 except TypeError:
40 pass
42 raise
45_sparse_msg = """\
47Loading a saved '{cls}' as a {new} with sparse values.
48'{cls}' is now removed. You should re-save this dataset in its new format.
49"""
52class _LoadSparseSeries:
53 # To load a SparseSeries as a Series[Sparse]
55 # https://github.com/python/mypy/issues/1020
56 # error: Incompatible return type for "__new__" (returns "Series", but must return
57 # a subtype of "_LoadSparseSeries")
58 def __new__(cls) -> "Series": # type: ignore
59 from pandas import Series
61 warnings.warn(
62 _sparse_msg.format(cls="SparseSeries", new="Series"),
63 FutureWarning,
64 stacklevel=6,
65 )
67 return Series(dtype=object)
70class _LoadSparseFrame:
71 # To load a SparseDataFrame as a DataFrame[Sparse]
73 # https://github.com/python/mypy/issues/1020
74 # error: Incompatible return type for "__new__" (returns "DataFrame", but must
75 # return a subtype of "_LoadSparseFrame")
76 def __new__(cls) -> "DataFrame": # type: ignore
77 from pandas import DataFrame
79 warnings.warn(
80 _sparse_msg.format(cls="SparseDataFrame", new="DataFrame"),
81 FutureWarning,
82 stacklevel=6,
83 )
85 return DataFrame()
88# If classes are moved, provide compat here.
89_class_locations_map = {
90 ("pandas.core.sparse.array", "SparseArray"): ("pandas.core.arrays", "SparseArray"),
91 # 15477
92 ("pandas.core.base", "FrozenNDArray"): ("numpy", "ndarray"),
93 ("pandas.core.indexes.frozen", "FrozenNDArray"): ("numpy", "ndarray"),
94 ("pandas.core.base", "FrozenList"): ("pandas.core.indexes.frozen", "FrozenList"),
95 # 10890
96 ("pandas.core.series", "TimeSeries"): ("pandas.core.series", "Series"),
97 ("pandas.sparse.series", "SparseTimeSeries"): (
98 "pandas.core.sparse.series",
99 "SparseSeries",
100 ),
101 # 12588, extensions moving
102 ("pandas._sparse", "BlockIndex"): ("pandas._libs.sparse", "BlockIndex"),
103 ("pandas.tslib", "Timestamp"): ("pandas._libs.tslib", "Timestamp"),
104 # 18543 moving period
105 ("pandas._period", "Period"): ("pandas._libs.tslibs.period", "Period"),
106 ("pandas._libs.period", "Period"): ("pandas._libs.tslibs.period", "Period"),
107 # 18014 moved __nat_unpickle from _libs.tslib-->_libs.tslibs.nattype
108 ("pandas.tslib", "__nat_unpickle"): (
109 "pandas._libs.tslibs.nattype",
110 "__nat_unpickle",
111 ),
112 ("pandas._libs.tslib", "__nat_unpickle"): (
113 "pandas._libs.tslibs.nattype",
114 "__nat_unpickle",
115 ),
116 # 15998 top-level dirs moving
117 ("pandas.sparse.array", "SparseArray"): (
118 "pandas.core.arrays.sparse",
119 "SparseArray",
120 ),
121 ("pandas.sparse.series", "SparseSeries"): (
122 "pandas.compat.pickle_compat",
123 "_LoadSparseSeries",
124 ),
125 ("pandas.sparse.frame", "SparseDataFrame"): (
126 "pandas.core.sparse.frame",
127 "_LoadSparseFrame",
128 ),
129 ("pandas.indexes.base", "_new_Index"): ("pandas.core.indexes.base", "_new_Index"),
130 ("pandas.indexes.base", "Index"): ("pandas.core.indexes.base", "Index"),
131 ("pandas.indexes.numeric", "Int64Index"): (
132 "pandas.core.indexes.numeric",
133 "Int64Index",
134 ),
135 ("pandas.indexes.range", "RangeIndex"): ("pandas.core.indexes.range", "RangeIndex"),
136 ("pandas.indexes.multi", "MultiIndex"): ("pandas.core.indexes.multi", "MultiIndex"),
137 ("pandas.tseries.index", "_new_DatetimeIndex"): (
138 "pandas.core.indexes.datetimes",
139 "_new_DatetimeIndex",
140 ),
141 ("pandas.tseries.index", "DatetimeIndex"): (
142 "pandas.core.indexes.datetimes",
143 "DatetimeIndex",
144 ),
145 ("pandas.tseries.period", "PeriodIndex"): (
146 "pandas.core.indexes.period",
147 "PeriodIndex",
148 ),
149 # 19269, arrays moving
150 ("pandas.core.categorical", "Categorical"): ("pandas.core.arrays", "Categorical"),
151 # 19939, add timedeltaindex, float64index compat from 15998 move
152 ("pandas.tseries.tdi", "TimedeltaIndex"): (
153 "pandas.core.indexes.timedeltas",
154 "TimedeltaIndex",
155 ),
156 ("pandas.indexes.numeric", "Float64Index"): (
157 "pandas.core.indexes.numeric",
158 "Float64Index",
159 ),
160 ("pandas.core.sparse.series", "SparseSeries"): (
161 "pandas.compat.pickle_compat",
162 "_LoadSparseSeries",
163 ),
164 ("pandas.core.sparse.frame", "SparseDataFrame"): (
165 "pandas.compat.pickle_compat",
166 "_LoadSparseFrame",
167 ),
168}
171# our Unpickler sub-class to override methods and some dispatcher
172# functions for compat and uses a non-public class of the pickle module.
174# error: Name 'pkl._Unpickler' is not defined
175class Unpickler(pkl._Unpickler): # type: ignore
176 def find_class(self, module, name):
177 # override superclass
178 key = (module, name)
179 module, name = _class_locations_map.get(key, key)
180 return super().find_class(module, name)
183Unpickler.dispatch = copy.copy(Unpickler.dispatch)
184Unpickler.dispatch[pkl.REDUCE[0]] = load_reduce
187def load_newobj(self):
188 args = self.stack.pop()
189 cls = self.stack[-1]
191 # compat
192 if issubclass(cls, Index):
193 obj = object.__new__(cls)
194 else:
195 obj = cls.__new__(cls, *args)
197 self.stack[-1] = obj
200Unpickler.dispatch[pkl.NEWOBJ[0]] = load_newobj
203def load_newobj_ex(self):
204 kwargs = self.stack.pop()
205 args = self.stack.pop()
206 cls = self.stack.pop()
208 # compat
209 if issubclass(cls, Index):
210 obj = object.__new__(cls)
211 else:
212 obj = cls.__new__(cls, *args, **kwargs)
213 self.append(obj)
216try:
217 Unpickler.dispatch[pkl.NEWOBJ_EX[0]] = load_newobj_ex
218except (AttributeError, KeyError):
219 pass
222def load(fh, encoding: Optional[str] = None, is_verbose: bool = False):
223 """
224 Load a pickle, with a provided encoding,
226 Parameters
227 ----------
228 fh : a filelike object
229 encoding : an optional encoding
230 is_verbose : show exception output
231 """
233 try:
234 fh.seek(0)
235 if encoding is not None:
236 up = Unpickler(fh, encoding=encoding)
237 else:
238 up = Unpickler(fh)
239 up.is_verbose = is_verbose
241 return up.load()
242 except (ValueError, TypeError):
243 raise