TypeError: cannot pickle '_contextvars.Context' object or 'PyCapsule' object during design matrix creation #738
Comments
|
Our users have reported similar errors with more contextual information. Please see this issue: lnccbrown/HSSM#301 |
|
@digicosmos86 thanks for opening the issue and linking the one in HSSM. I'll try to have a look at it soon. |
|
@digicosmos86 do you know in which OS this is happening? |
|
Thanks for looking into this, @tomicapretto! I am using MacOS (ARM64) with Python 3.11, but the user who reported the issue seems to be using linux, with Python versions both 3.9 and 3.11 |
|
I'm on Ubuntu 22.04.3 LTS and I get a different error: ---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
/home/tomas/Desktop/OSS/bambinos/bambi/test_hssm.ipynb Cell 3 line 1
----> 1 model_side = hssm.HSSM(
2 data=cav_data,
3 model="ddm",
4 a = 1,
5 hierarchical = False,
6 include=[
7 {
8 "name": "v", # Drift rate
9 "prior": {
10 "Intercept": {
11 "name": "Normal",
12 "sigma": 1,
13 "mu": 0.0,
14 }
15 },
16 "formula": "v ~ 1 + (stim|subj_idx) + (1|subj_idx) + C(dbs):C(conf)",
17 "link": "identity",
18 },
19 {
20 "name": "z", # Starting point
21 "prior": {
22 "Intercept": {
23 "name": "Normal",
24 "sigma": 1,
25 "mu": 0.0,
26 }
27 },
28 "formula": "z ~ 1 + (stim|subj_idx) + (1|subj_idx) + C(dbs):C(conf)",
29 "link": "identity",
30 },
31 {
32 "name": "t", # Starting point
33 "prior": {
34 "Intercept": {
35 "name": "Normal",
36 "sigma": 1,
37 "mu": 0.0,
38 }
39 },
40 "formula": "t ~ 1 + (stim|subj_idx) + (1|subj_idx) + C(dbs):C(conf)",
41 "link": "identity",
42 },
43 ],
44 )
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/site-packages/hssm/hssm.py:268, in HSSM.__init__(self, data, model, include, model_config, loglik, loglik_kind, p_outlier, lapse, hierarchical, **kwargs)
259 self.model_distribution = self._make_model_distribution()
261 self.family = make_family(
262 self.model_distribution,
263 self.list_params,
264 self.link,
265 self._parent,
266 )
--> 268 self.model = bmb.Model(
269 self.formula, data, family=self.family, priors=self.priors, **other_kwargs
270 )
272 self._aliases = get_alias_dict(self.model, self._parent_param)
273 self.set_alias(self._aliases)
File ~/Desktop/OSS/bambinos/bambi/bambi/models.py:168, in Model.__init__(self, formula, data, family, priors, link, categorical, potentials, dropna, auto_scale, noncentered, center_predictors, extra_namespace)
166 design = remove_common_intercept(design)
167 else:
--> 168 design = fm.design_matrices(
169 self.formula.main, self.data, na_action, 1, additional_namespace
170 )
172 if design.response is None:
173 raise ValueError(
174 "No outcome variable is set! "
175 "Please specify an outcome variable using the formula interface."
176 )
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/site-packages/formulae/matrices.py:556, in design_matrices(formula, data, na_action, env, extra_namespace)
553 else:
554 raise ValueError(f"'data' contains {incomplete_rows_n} incomplete rows.")
--> 556 design = DesignMatrices(description, data, env)
557 return design
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/site-packages/formulae/matrices.py:54, in DesignMatrices.__init__(self, model, data, env)
51 self.model = model
53 # Evaluate terms in the model
---> 54 self.model.eval(data, env)
56 if self.model.response:
57 self.response = ResponseMatrix(self.model.response)
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/site-packages/formulae/terms/terms.py:1247, in Model.eval(self, data, env)
1245 # Evaluate common terms
1246 encodings = self._get_encoding_bools()
-> 1247 self.add_extra_terms(encodings, data, env)
1249 # Need to get encodings again after creating possible extra terms
1250 encodings = self._get_encoding_bools()
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/site-packages/formulae/terms/terms.py:1229, in Model.add_extra_terms(self, encodings, data, env)
1226 if hasattr(encoding, "__len__") and len(encoding) > 1:
1227 # Last encoding is the one for the original term
1228 for subencoding in encoding[:-1]:
-> 1229 extra_term = create_extra_term(term, subencoding, data, env)
1230 self.common_terms.insert(self.common_terms.index(term), extra_term)
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/site-packages/formulae/terms/terms.py:1283, in create_extra_term(term, encoding, data, env)
1281 components = [term.get_component(name) for name in component_names if name in encoding.keys()]
1282 components += [component for component in term.components if component.kind == "numeric"]
-> 1283 extra_term = Term(*deepcopy(components))
1284 extra_term.set_type(data, env)
1285 return extra_term
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:146, in deepcopy(x, memo, _nil)
144 copier = _deepcopy_dispatch.get(cls)
145 if copier is not None:
--> 146 y = copier(x, memo)
147 else:
148 if issubclass(cls, type):
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:206, in _deepcopy_list(x, memo, deepcopy)
204 append = y.append
205 for a in x:
--> 206 append(deepcopy(a, memo))
207 return y
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:172, in deepcopy(x, memo, _nil)
170 y = x
171 else:
--> 172 y = _reconstruct(x, memo, *rv)
174 # If is its own copy, don't memoize.
175 if y is not x:
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:271, in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
269 if state is not None:
270 if deep:
--> 271 state = deepcopy(state, memo)
272 if hasattr(y, '__setstate__'):
273 y.__setstate__(state)
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:146, in deepcopy(x, memo, _nil)
144 copier = _deepcopy_dispatch.get(cls)
145 if copier is not None:
--> 146 y = copier(x, memo)
147 else:
148 if issubclass(cls, type):
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:231, in _deepcopy_dict(x, memo, deepcopy)
229 memo[id(x)] = y
230 for key, value in x.items():
--> 231 y[deepcopy(key, memo)] = deepcopy(value, memo)
232 return y
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:172, in deepcopy(x, memo, _nil)
170 y = x
171 else:
--> 172 y = _reconstruct(x, memo, *rv)
174 # If is its own copy, don't memoize.
175 if y is not x:
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:271, in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
269 if state is not None:
270 if deep:
--> 271 state = deepcopy(state, memo)
272 if hasattr(y, '__setstate__'):
273 y.__setstate__(state)
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:146, in deepcopy(x, memo, _nil)
144 copier = _deepcopy_dispatch.get(cls)
145 if copier is not None:
--> 146 y = copier(x, memo)
147 else:
148 if issubclass(cls, type):
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:231, in _deepcopy_dict(x, memo, deepcopy)
229 memo[id(x)] = y
230 for key, value in x.items():
--> 231 y[deepcopy(key, memo)] = deepcopy(value, memo)
232 return y
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:146, in deepcopy(x, memo, _nil)
144 copier = _deepcopy_dispatch.get(cls)
145 if copier is not None:
--> 146 y = copier(x, memo)
147 else:
148 if issubclass(cls, type):
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:206, in _deepcopy_list(x, memo, deepcopy)
204 append = y.append
205 for a in x:
--> 206 append(deepcopy(a, memo))
207 return y
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:146, in deepcopy(x, memo, _nil)
144 copier = _deepcopy_dispatch.get(cls)
145 if copier is not None:
--> 146 y = copier(x, memo)
147 else:
148 if issubclass(cls, type):
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:231, in _deepcopy_dict(x, memo, deepcopy)
229 memo[id(x)] = y
230 for key, value in x.items():
--> 231 y[deepcopy(key, memo)] = deepcopy(value, memo)
232 return y
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:146, in deepcopy(x, memo, _nil)
144 copier = _deepcopy_dispatch.get(cls)
145 if copier is not None:
--> 146 y = copier(x, memo)
147 else:
148 if issubclass(cls, type):
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:231, in _deepcopy_dict(x, memo, deepcopy)
229 memo[id(x)] = y
230 for key, value in x.items():
--> 231 y[deepcopy(key, memo)] = deepcopy(value, memo)
232 return y
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:146, in deepcopy(x, memo, _nil)
144 copier = _deepcopy_dispatch.get(cls)
145 if copier is not None:
--> 146 y = copier(x, memo)
147 else:
148 if issubclass(cls, type):
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:238, in _deepcopy_method(x, memo)
237 def _deepcopy_method(x, memo): # Copy instance methods
--> 238 return type(x)(x.__func__, deepcopy(x.__self__, memo))
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:172, in deepcopy(x, memo, _nil)
170 y = x
171 else:
--> 172 y = _reconstruct(x, memo, *rv)
174 # If is its own copy, don't memoize.
175 if y is not x:
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:271, in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
269 if state is not None:
270 if deep:
--> 271 state = deepcopy(state, memo)
272 if hasattr(y, '__setstate__'):
273 y.__setstate__(state)
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:146, in deepcopy(x, memo, _nil)
144 copier = _deepcopy_dispatch.get(cls)
145 if copier is not None:
--> 146 y = copier(x, memo)
147 else:
148 if issubclass(cls, type):
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:231, in _deepcopy_dict(x, memo, deepcopy)
229 memo[id(x)] = y
230 for key, value in x.items():
--> 231 y[deepcopy(key, memo)] = deepcopy(value, memo)
232 return y
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:146, in deepcopy(x, memo, _nil)
144 copier = _deepcopy_dispatch.get(cls)
145 if copier is not None:
--> 146 y = copier(x, memo)
147 else:
148 if issubclass(cls, type):
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:231, in _deepcopy_dict(x, memo, deepcopy)
229 memo[id(x)] = y
230 for key, value in x.items():
--> 231 y[deepcopy(key, memo)] = deepcopy(value, memo)
232 return y
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:172, in deepcopy(x, memo, _nil)
170 y = x
171 else:
--> 172 y = _reconstruct(x, memo, *rv)
174 # If is its own copy, don't memoize.
175 if y is not x:
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:271, in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
269 if state is not None:
270 if deep:
--> 271 state = deepcopy(state, memo)
272 if hasattr(y, '__setstate__'):
273 y.__setstate__(state)
[... skipping similar frames: deepcopy at line 146 (4 times), _deepcopy_dict at line 231 (3 times), _reconstruct at line 271 (1 times), deepcopy at line 172 (1 times)]
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:231, in _deepcopy_dict(x, memo, deepcopy)
229 memo[id(x)] = y
230 for key, value in x.items():
--> 231 y[deepcopy(key, memo)] = deepcopy(value, memo)
232 return y
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:172, in deepcopy(x, memo, _nil)
170 y = x
171 else:
--> 172 y = _reconstruct(x, memo, *rv)
174 # If is its own copy, don't memoize.
175 if y is not x:
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:271, in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
269 if state is not None:
270 if deep:
--> 271 state = deepcopy(state, memo)
272 if hasattr(y, '__setstate__'):
273 y.__setstate__(state)
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:146, in deepcopy(x, memo, _nil)
144 copier = _deepcopy_dispatch.get(cls)
145 if copier is not None:
--> 146 y = copier(x, memo)
147 else:
148 if issubclass(cls, type):
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:211, in _deepcopy_tuple(x, memo, deepcopy)
210 def _deepcopy_tuple(x, memo, deepcopy=deepcopy):
--> 211 y = [deepcopy(a, memo) for a in x]
212 # We're not going to put the tuple in the memo, but it's still important we
213 # check for it, in case the tuple contains recursive mutable structures.
214 try:
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:211, in <listcomp>(.0)
210 def _deepcopy_tuple(x, memo, deepcopy=deepcopy):
--> 211 y = [deepcopy(a, memo) for a in x]
212 # We're not going to put the tuple in the memo, but it's still important we
213 # check for it, in case the tuple contains recursive mutable structures.
214 try:
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:146, in deepcopy(x, memo, _nil)
144 copier = _deepcopy_dispatch.get(cls)
145 if copier is not None:
--> 146 y = copier(x, memo)
147 else:
148 if issubclass(cls, type):
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:231, in _deepcopy_dict(x, memo, deepcopy)
229 memo[id(x)] = y
230 for key, value in x.items():
--> 231 y[deepcopy(key, memo)] = deepcopy(value, memo)
232 return y
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/copy.py:161, in deepcopy(x, memo, _nil)
159 reductor = getattr(x, "__reduce_ex__", None)
160 if reductor is not None:
--> 161 rv = reductor(4)
162 else:
163 reductor = getattr(x, "__reduce__", None)
TypeError: cannot pickle '_hashlib.HMAC' objectI will keep looking into it. |
|
@digicosmos86 so I think I fixed it. I'm going to tell you about my highly professional debugging process 😄
File ~/anaconda3/envs/bambi_hssm/lib/python3.10/site-packages/formulae/terms/terms.py:1283, in create_extra_term(term, encoding, data, env)
1281 components = [term.get_component(name) for name in component_names if name in encoding.keys()]
1282 components += [component for component in term.components if component.kind == "numeric"]
-> 1283 extra_term = Term(*deepcopy(components))
1284 extra_term.set_type(data, env)
1285 return extra_termSo I realized the problem is when we try to
The problem occurs when it is creating a
This is the model model_side = hssm.HSSM(
data=cav_data,
model="ddm",
a = 1,
hierarchical = False,
include=[
{
"name": "v", # Drift rate
"prior": {
"Intercept": {
"name": "Normal",
"sigma": 1,
"mu": 0.0,
}
},
"formula": "v ~ 1 + (stim|subj_idx) + (1|subj_idx) + C(dbs):conf",
"link": "identity",
},
{
"name": "z", # Starting point
"prior": {
"Intercept": {
"name": "Normal",
"sigma": 1,
"mu": 0.0,
}
},
"formula": "z ~ 1 + (stim|subj_idx) + (1|subj_idx) + C(dbs):conf",
"link": "identity",
},
{
"name": "t", # Starting point
"prior": {
"Intercept": {
"name": "Normal",
"sigma": 1,
"mu": 0.0,
}
},
"formula": "t ~ 1 + (stim|subj_idx) + (1|subj_idx) + C(dbs):conf",
"link": "identity",
},
],
)NOTE I didn't try to fit it. If you do, let me know what happens. I still think there's a problem because this shouldn't happen. But at least for now, there's a fix. If you have this problem again and you have numeric variables that need to be interpreted as categoric, convert them to object type before creating the model or tell Bambi they're categorical with the |
|
Thanks @tomicapretto! Looks like this fixed it on my end 👍 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
The
hssmpackage relies onbambifor model creation. A Type error is thrown during the creation of certain models with certain term combinations:This is the error (Python=3.11, bambi=0.12)
The text was updated successfully, but these errors were encountered: