diff --git a/devel/CHANGELOG/index.html b/devel/CHANGELOG/index.html
index c27210b44..cf751b9a4 100644
--- a/devel/CHANGELOG/index.html
+++ b/devel/CHANGELOG/index.html
@@ -2725,6 +2725,8 @@
 <h1 id="changelog">Changelog<a class="headerlink" href="#changelog" title="Permanent link">&para;</a></h1>
 <h2 id="unreleased">Unreleased<a class="headerlink" href="#unreleased" title="Permanent link">&para;</a></h2>
 <ul>
+<li>Implementation of Data-OOB by @BastienZim
+  <a href="https://github.com/aai-institute/pyDVL/pull/426">PR #426</a></li>
 <li>Refactoring of parallel module. Old imports will stop working in v0.9.0
   <a href="https://github.com/aai-institute/pyDVL/pull/421">PR #421</a></li>
 </ul>
@@ -2745,6 +2747,10 @@ <h3 id="added">Added<a class="headerlink" href="#added" title="Permanent link">&
   <code>pydvl.value.semivalues</code>. Introduced new type <code>Seed</code> and conversion function 
   <code>ensure_seed_sequence</code>.
   <a href="https://github.com/aai-institute/pyDVL/pull/396">PR #396</a></li>
+<li>Added <code>batch_size</code> parameter to <code>compute_banzhaf_semivalues</code>,
+  <code>compute_beta_shapley_semivalues</code>, <code>compute_shapley_semivalues</code> and
+  <code>compute_generic_semivalues</code>.
+  <a href="https://github.com/aai-institute/pyDVL/pull/428">PR #428</a></li>
 </ul>
 <h3 id="changed">Changed<a class="headerlink" href="#changed" title="Permanent link">&para;</a></h3>
 <ul>
@@ -2968,11 +2974,11 @@ <h2 id="010-first-release">0.1.0 - 🎉 first release<a class="headerlink" href=
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/index.html b/devel/api/pydvl/index.html
index eda1b1db5..29cbf14b3 100644
--- a/devel/api/pydvl/index.html
+++ b/devel/api/pydvl/index.html
@@ -2502,11 +2502,11 @@ <h2 id="pydvl--the-python-data-valuation-library-api">The Python Data Valuation
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/influence/general/index.html b/devel/api/pydvl/influence/general/index.html
index 60e7117b7..920967e8e 100644
--- a/devel/api/pydvl/influence/general/index.html
+++ b/devel/api/pydvl/influence/general/index.html
@@ -3887,11 +3887,11 @@ <h2 id="pydvl.influence.general.compute_influences" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/influence/index.html b/devel/api/pydvl/influence/index.html
index 7f24b8631..5ffb8764b 100644
--- a/devel/api/pydvl/influence/index.html
+++ b/devel/api/pydvl/influence/index.html
@@ -2497,11 +2497,11 @@ <h1>Influence</h1>
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/influence/inversion/index.html b/devel/api/pydvl/influence/inversion/index.html
index 720c4a3af..24538e7ea 100644
--- a/devel/api/pydvl/influence/inversion/index.html
+++ b/devel/api/pydvl/influence/inversion/index.html
@@ -3449,11 +3449,11 @@ <h2 id="pydvl.influence.inversion.solve_hvp" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/influence/torch/functional/index.html b/devel/api/pydvl/influence/torch/functional/index.html
index c3135b610..9b5706a35 100644
--- a/devel/api/pydvl/influence/torch/functional/index.html
+++ b/devel/api/pydvl/influence/torch/functional/index.html
@@ -3626,11 +3626,11 @@ <h2 id="pydvl.influence.torch.functional.get_hvp_function" class="doc doc-headin
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/influence/torch/index.html b/devel/api/pydvl/influence/torch/index.html
index b29886193..8ee196b4b 100644
--- a/devel/api/pydvl/influence/torch/index.html
+++ b/devel/api/pydvl/influence/torch/index.html
@@ -2493,11 +2493,11 @@ <h1>Torch</h1>
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/influence/torch/torch_differentiable/index.html b/devel/api/pydvl/influence/torch/torch_differentiable/index.html
index 7781c2df3..d6160e05f 100644
--- a/devel/api/pydvl/influence/torch/torch_differentiable/index.html
+++ b/devel/api/pydvl/influence/torch/torch_differentiable/index.html
@@ -6524,11 +6524,11 @@ <h2 id="pydvl.influence.torch.torch_differentiable.solve_arnoldi" class="doc doc
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/influence/torch/util/index.html b/devel/api/pydvl/influence/torch/util/index.html
index 02e59c433..6ae72ecbc 100644
--- a/devel/api/pydvl/influence/torch/util/index.html
+++ b/devel/api/pydvl/influence/torch/util/index.html
@@ -3430,11 +3430,11 @@ <h2 id="pydvl.influence.torch.util.as_tensor" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/influence/twice_differentiable/index.html b/devel/api/pydvl/influence/twice_differentiable/index.html
index bfc55babb..24b0ddef9 100644
--- a/devel/api/pydvl/influence/twice_differentiable/index.html
+++ b/devel/api/pydvl/influence/twice_differentiable/index.html
@@ -4230,11 +4230,11 @@ <h3 id="pydvl.influence.twice_differentiable.TensorUtilities.from_twice_differen
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/parallel/backend/index.html b/devel/api/pydvl/parallel/backend/index.html
index 16fbf2c15..1a0446c6b 100644
--- a/devel/api/pydvl/parallel/backend/index.html
+++ b/devel/api/pydvl/parallel/backend/index.html
@@ -3195,11 +3195,11 @@ <h2 id="pydvl.parallel.backend.effective_n_jobs" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/parallel/backends/index.html b/devel/api/pydvl/parallel/backends/index.html
index 1c950760d..d9fc1b9ca 100644
--- a/devel/api/pydvl/parallel/backends/index.html
+++ b/devel/api/pydvl/parallel/backends/index.html
@@ -2493,11 +2493,11 @@ <h1>Backends</h1>
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/parallel/backends/joblib/index.html b/devel/api/pydvl/parallel/backends/joblib/index.html
index fec72b942..2cfca978d 100644
--- a/devel/api/pydvl/parallel/backends/joblib/index.html
+++ b/devel/api/pydvl/parallel/backends/joblib/index.html
@@ -2753,11 +2753,11 @@ <h3 id="pydvl.parallel.backends.joblib.JoblibParallelBackend.wrap" class="doc do
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/parallel/backends/ray/index.html b/devel/api/pydvl/parallel/backends/ray/index.html
index 02413798e..e02235483 100644
--- a/devel/api/pydvl/parallel/backends/ray/index.html
+++ b/devel/api/pydvl/parallel/backends/ray/index.html
@@ -2779,11 +2779,11 @@ <h3 id="pydvl.parallel.backends.ray.RayParallelBackend.wrap" class="doc doc-head
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/parallel/config/index.html b/devel/api/pydvl/parallel/config/index.html
index 1c92a69e9..c6963ab71 100644
--- a/devel/api/pydvl/parallel/config/index.html
+++ b/devel/api/pydvl/parallel/config/index.html
@@ -2694,11 +2694,11 @@ <h2 id="pydvl.parallel.config.ParallelConfig" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/parallel/futures/index.html b/devel/api/pydvl/parallel/futures/index.html
index be93fcf4a..017314059 100644
--- a/devel/api/pydvl/parallel/futures/index.html
+++ b/devel/api/pydvl/parallel/futures/index.html
@@ -2679,11 +2679,11 @@ <h2 id="pydvl.parallel.futures.init_executor" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/parallel/futures/ray/index.html b/devel/api/pydvl/parallel/futures/ray/index.html
index cfed77d32..f0caa2ff7 100644
--- a/devel/api/pydvl/parallel/futures/ray/index.html
+++ b/devel/api/pydvl/parallel/futures/ray/index.html
@@ -3213,11 +3213,11 @@ <h3 id="pydvl.parallel.futures.ray.RayExecutor.__exit__" class="doc doc-heading"
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/parallel/index.html b/devel/api/pydvl/parallel/index.html
index 89259414f..ca7d90b9e 100644
--- a/devel/api/pydvl/parallel/index.html
+++ b/devel/api/pydvl/parallel/index.html
@@ -2518,11 +2518,11 @@ <h1>Parallel</h1>
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/parallel/map_reduce/index.html b/devel/api/pydvl/parallel/map_reduce/index.html
index a84be7600..dd224549a 100644
--- a/devel/api/pydvl/parallel/map_reduce/index.html
+++ b/devel/api/pydvl/parallel/map_reduce/index.html
@@ -3032,11 +3032,11 @@ <h3 id="pydvl.parallel.map_reduce.MapReduceJob.__call__" class="doc doc-heading"
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/reporting/index.html b/devel/api/pydvl/reporting/index.html
index 46ca1eb55..9311a8bf2 100644
--- a/devel/api/pydvl/reporting/index.html
+++ b/devel/api/pydvl/reporting/index.html
@@ -2491,11 +2491,11 @@ <h1>Reporting</h1>
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/reporting/plots/index.html b/devel/api/pydvl/reporting/plots/index.html
index d0e9c6b0a..a7712c8d8 100644
--- a/devel/api/pydvl/reporting/plots/index.html
+++ b/devel/api/pydvl/reporting/plots/index.html
@@ -3436,11 +3436,11 @@ <h2 id="pydvl.reporting.plots.plot_influence_distribution_by_label" class="doc d
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/reporting/scores/index.html b/devel/api/pydvl/reporting/scores/index.html
index 9cc8105ce..1dbed6494 100644
--- a/devel/api/pydvl/reporting/scores/index.html
+++ b/devel/api/pydvl/reporting/scores/index.html
@@ -2770,11 +2770,11 @@ <h2 id="pydvl.reporting.scores.compute_removal_score" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/utils/caching/index.html b/devel/api/pydvl/utils/caching/index.html
index 664e486b8..113de7b86 100644
--- a/devel/api/pydvl/utils/caching/index.html
+++ b/devel/api/pydvl/utils/caching/index.html
@@ -3012,7 +3012,7 @@ <h2 id="pydvl.utils.caching.memcached" class="doc doc-heading">
             <div class="doc-md-description">
               <p>If <code>True</code>, repeated calls to a function
 with the same arguments will be allowed and outputs averaged until the
-running standard deviation of the mean stabilises below
+running standard deviation of the mean stabilizes below
 <code>rtol_stderr * mean</code>.</p>
             </div>
             <p>
@@ -3355,7 +3355,7 @@ <h2 id="pydvl.utils.caching.memcached" class="doc doc-heading">
 </span><span id="__span-0-181"><a id="__codelineno-0-181" name="__codelineno-0-181"></a><span class="sd">            not cached.</span>
 </span><span id="__span-0-182"><a id="__codelineno-0-182" name="__codelineno-0-182"></a><span class="sd">        allow_repeated_evaluations: If `True`, repeated calls to a function</span>
 </span><span id="__span-0-183"><a id="__codelineno-0-183" name="__codelineno-0-183"></a><span class="sd">            with the same arguments will be allowed and outputs averaged until the</span>
-</span><span id="__span-0-184"><a id="__codelineno-0-184" name="__codelineno-0-184"></a><span class="sd">            running standard deviation of the mean stabilises below</span>
+</span><span id="__span-0-184"><a id="__codelineno-0-184" name="__codelineno-0-184"></a><span class="sd">            running standard deviation of the mean stabilizes below</span>
 </span><span id="__span-0-185"><a id="__codelineno-0-185" name="__codelineno-0-185"></a><span class="sd">            `rtol_stderr * mean`.</span>
 </span><span id="__span-0-186"><a id="__codelineno-0-186" name="__codelineno-0-186"></a><span class="sd">        rtol_stderr: relative tolerance for repeated evaluations. More precisely,</span>
 </span><span id="__span-0-187"><a id="__codelineno-0-187" name="__codelineno-0-187"></a><span class="sd">            [memcached()][pydvl.utils.caching.memcached] will stop evaluating the function once the</span>
@@ -3530,11 +3530,11 @@ <h2 id="pydvl.utils.caching.memcached" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/utils/config/index.html b/devel/api/pydvl/utils/config/index.html
index 70939f579..66c685d36 100644
--- a/devel/api/pydvl/utils/config/index.html
+++ b/devel/api/pydvl/utils/config/index.html
@@ -3050,11 +3050,11 @@ <h2 id="pydvl.utils.config.MemcachedConfig" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/utils/dataset/index.html b/devel/api/pydvl/utils/dataset/index.html
index afdcba937..d1fa0cd36 100644
--- a/devel/api/pydvl/utils/dataset/index.html
+++ b/devel/api/pydvl/utils/dataset/index.html
@@ -2805,7 +2805,7 @@ <h2 id="pydvl.utils.dataset.Dataset" class="doc doc-heading">
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-external" title="typing.Union" href="https://docs.python.org/3/library/typing.html#typing.Union">Union</a>[<a class="autorefs autorefs-external" title="numpy.ndarray" href="https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html#numpy.ndarray">ndarray</a>, <a class="autorefs autorefs-external" title="pandas.DataFrame" href="https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html#pandas.DataFrame">DataFrame</a>]</code>
+                    <code><a class="autorefs autorefs-external" title="typing.Union" href="https://docs.python.org/3/library/typing.html#typing.Union">Union</a>[<a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a>, <a class="autorefs autorefs-external" title="pandas.DataFrame" href="https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html#pandas.DataFrame">DataFrame</a>]</code>
                 </span>
             </p>
           </td>
@@ -2819,7 +2819,7 @@ <h2 id="pydvl.utils.dataset.Dataset" class="doc doc-heading">
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-external" title="typing.Union" href="https://docs.python.org/3/library/typing.html#typing.Union">Union</a>[<a class="autorefs autorefs-external" title="numpy.ndarray" href="https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html#numpy.ndarray">ndarray</a>, <a class="autorefs autorefs-external" title="pandas.DataFrame" href="https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html#pandas.DataFrame">DataFrame</a>]</code>
+                    <code><a class="autorefs autorefs-external" title="typing.Union" href="https://docs.python.org/3/library/typing.html#typing.Union">Union</a>[<a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a>, <a class="autorefs autorefs-external" title="pandas.DataFrame" href="https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html#pandas.DataFrame">DataFrame</a>]</code>
                 </span>
             </p>
           </td>
@@ -2833,7 +2833,7 @@ <h2 id="pydvl.utils.dataset.Dataset" class="doc doc-heading">
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-external" title="typing.Union" href="https://docs.python.org/3/library/typing.html#typing.Union">Union</a>[<a class="autorefs autorefs-external" title="numpy.ndarray" href="https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html#numpy.ndarray">ndarray</a>, <a class="autorefs autorefs-external" title="pandas.DataFrame" href="https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html#pandas.DataFrame">DataFrame</a>]</code>
+                    <code><a class="autorefs autorefs-external" title="typing.Union" href="https://docs.python.org/3/library/typing.html#typing.Union">Union</a>[<a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a>, <a class="autorefs autorefs-external" title="pandas.DataFrame" href="https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html#pandas.DataFrame">DataFrame</a>]</code>
                 </span>
             </p>
           </td>
@@ -2847,7 +2847,7 @@ <h2 id="pydvl.utils.dataset.Dataset" class="doc doc-heading">
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-external" title="typing.Union" href="https://docs.python.org/3/library/typing.html#typing.Union">Union</a>[<a class="autorefs autorefs-external" title="numpy.ndarray" href="https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html#numpy.ndarray">ndarray</a>, <a class="autorefs autorefs-external" title="pandas.DataFrame" href="https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html#pandas.DataFrame">DataFrame</a>]</code>
+                    <code><a class="autorefs autorefs-external" title="typing.Union" href="https://docs.python.org/3/library/typing.html#typing.Union">Union</a>[<a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a>, <a class="autorefs autorefs-external" title="pandas.DataFrame" href="https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html#pandas.DataFrame">DataFrame</a>]</code>
                 </span>
             </p>
           </td>
@@ -3038,12 +3038,16 @@ <h2 id="pydvl.utils.dataset.Dataset" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-125">125</a></span>
 <span class="normal"><a href="#__codelineno-0-126">126</a></span>
 <span class="normal"><a href="#__codelineno-0-127">127</a></span>
-<span class="normal"><a href="#__codelineno-0-128">128</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-41"><a id="__codelineno-0-41" name="__codelineno-0-41"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+<span class="normal"><a href="#__codelineno-0-128">128</a></span>
+<span class="normal"><a href="#__codelineno-0-129">129</a></span>
+<span class="normal"><a href="#__codelineno-0-130">130</a></span>
+<span class="normal"><a href="#__codelineno-0-131">131</a></span>
+<span class="normal"><a href="#__codelineno-0-132">132</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-41"><a id="__codelineno-0-41" name="__codelineno-0-41"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
 </span><span id="__span-0-42"><a id="__codelineno-0-42" name="__codelineno-0-42"></a>    <span class="bp">self</span><span class="p">,</span>
-</span><span id="__span-0-43"><a id="__codelineno-0-43" name="__codelineno-0-43"></a>    <span class="n">x_train</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">],</span>
-</span><span id="__span-0-44"><a id="__codelineno-0-44" name="__codelineno-0-44"></a>    <span class="n">y_train</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">],</span>
-</span><span id="__span-0-45"><a id="__codelineno-0-45" name="__codelineno-0-45"></a>    <span class="n">x_test</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">],</span>
-</span><span id="__span-0-46"><a id="__codelineno-0-46" name="__codelineno-0-46"></a>    <span class="n">y_test</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">],</span>
+</span><span id="__span-0-43"><a id="__codelineno-0-43" name="__codelineno-0-43"></a>    <span class="n">x_train</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">NDArray</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">],</span>
+</span><span id="__span-0-44"><a id="__codelineno-0-44" name="__codelineno-0-44"></a>    <span class="n">y_train</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">NDArray</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">],</span>
+</span><span id="__span-0-45"><a id="__codelineno-0-45" name="__codelineno-0-45"></a>    <span class="n">x_test</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">NDArray</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">],</span>
+</span><span id="__span-0-46"><a id="__codelineno-0-46" name="__codelineno-0-46"></a>    <span class="n">y_test</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">NDArray</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">],</span>
 </span><span id="__span-0-47"><a id="__codelineno-0-47" name="__codelineno-0-47"></a>    <span class="n">feature_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
 </span><span id="__span-0-48"><a id="__codelineno-0-48" name="__codelineno-0-48"></a>    <span class="n">target_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
 </span><span id="__span-0-49"><a id="__codelineno-0-49" name="__codelineno-0-49"></a>    <span class="n">data_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -3124,8 +3128,12 @@ <h2 id="pydvl.utils.dataset.Dataset" class="doc doc-heading">
 </span><span id="__span-0-124"><a id="__codelineno-0-124" name="__codelineno-0-124"></a>            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Mismatching number of targets and names&quot;</span><span class="p">)</span>
 </span><span id="__span-0-125"><a id="__codelineno-0-125" name="__codelineno-0-125"></a>
 </span><span id="__span-0-126"><a id="__codelineno-0-126" name="__codelineno-0-126"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">description</span> <span class="o">=</span> <span class="n">description</span> <span class="ow">or</span> <span class="s2">&quot;No description&quot;</span>
-</span><span id="__span-0-127"><a id="__codelineno-0-127" name="__codelineno-0-127"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">x_train</span><span class="p">))</span>
-</span><span id="__span-0-128"><a id="__codelineno-0-128" name="__codelineno-0-128"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_data_names</span> <span class="o">=</span> <span class="n">data_names</span> <span class="k">if</span> <span class="n">data_names</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span>
+</span><span id="__span-0-127"><a id="__codelineno-0-127" name="__codelineno-0-127"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">x_train</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">)</span>
+</span><span id="__span-0-128"><a id="__codelineno-0-128" name="__codelineno-0-128"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_data_names</span> <span class="o">=</span> <span class="p">(</span>
+</span><span id="__span-0-129"><a id="__codelineno-0-129" name="__codelineno-0-129"></a>        <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">data_names</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">object</span><span class="p">)</span>
+</span><span id="__span-0-130"><a id="__codelineno-0-130" name="__codelineno-0-130"></a>        <span class="k">if</span> <span class="n">data_names</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+</span><span id="__span-0-131"><a id="__codelineno-0-131" name="__codelineno-0-131"></a>        <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">object</span><span class="p">)</span>
+</span><span id="__span-0-132"><a id="__codelineno-0-132" name="__codelineno-0-132"></a>    <span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
               </details>
 
@@ -3145,7 +3153,7 @@ <h2 id="pydvl.utils.dataset.Dataset" class="doc doc-heading">
 
 
 <h3 id="pydvl.utils.dataset.Dataset.indices" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">indices</span></code>
+          <code class="highlight language-python"><span class="n">indices</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span></code>
   
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
@@ -3168,7 +3176,7 @@ <h3 id="pydvl.utils.dataset.Dataset.indices" class="doc doc-heading">
 
 
 <h3 id="pydvl.utils.dataset.Dataset.data_names" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">data_names</span></code>
+          <code class="highlight language-python"><span class="n">data_names</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">object_</span><span class="p">]</span></code>
   
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
@@ -3191,7 +3199,7 @@ <h3 id="pydvl.utils.dataset.Dataset.data_names" class="doc doc-heading">
 
 
 <h3 id="pydvl.utils.dataset.Dataset.dim" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">dim</span></code>
+          <code class="highlight language-python"><span class="n">dim</span><span class="p">:</span> <span class="nb">int</span></code>
   
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
@@ -3290,11 +3298,7 @@ <h3 id="pydvl.utils.dataset.Dataset.get_training_data" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/dataset.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-139">139</a></span>
-<span class="normal"><a href="#__codelineno-0-140">140</a></span>
-<span class="normal"><a href="#__codelineno-0-141">141</a></span>
-<span class="normal"><a href="#__codelineno-0-142">142</a></span>
-<span class="normal"><a href="#__codelineno-0-143">143</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-143">143</a></span>
 <span class="normal"><a href="#__codelineno-0-144">144</a></span>
 <span class="normal"><a href="#__codelineno-0-145">145</a></span>
 <span class="normal"><a href="#__codelineno-0-146">146</a></span>
@@ -3313,30 +3317,34 @@ <h3 id="pydvl.utils.dataset.Dataset.get_training_data" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-159">159</a></span>
 <span class="normal"><a href="#__codelineno-0-160">160</a></span>
 <span class="normal"><a href="#__codelineno-0-161">161</a></span>
-<span class="normal"><a href="#__codelineno-0-162">162</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-139"><a id="__codelineno-0-139" name="__codelineno-0-139"></a><span class="k">def</span> <span class="nf">get_training_data</span><span class="p">(</span>
-</span><span id="__span-0-140"><a id="__codelineno-0-140" name="__codelineno-0-140"></a>    <span class="bp">self</span><span class="p">,</span> <span class="n">indices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Iterable</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
-</span><span id="__span-0-141"><a id="__codelineno-0-141" name="__codelineno-0-141"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">NDArray</span><span class="p">,</span> <span class="n">NDArray</span><span class="p">]:</span>
-</span><span id="__span-0-142"><a id="__codelineno-0-142" name="__codelineno-0-142"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Given a set of indices, returns the training data that refer to those</span>
-</span><span id="__span-0-143"><a id="__codelineno-0-143" name="__codelineno-0-143"></a><span class="sd">    indices.</span>
-</span><span id="__span-0-144"><a id="__codelineno-0-144" name="__codelineno-0-144"></a>
-</span><span id="__span-0-145"><a id="__codelineno-0-145" name="__codelineno-0-145"></a><span class="sd">    This is used mainly by [Utility][pydvl.utils.utility.Utility] to retrieve</span>
-</span><span id="__span-0-146"><a id="__codelineno-0-146" name="__codelineno-0-146"></a><span class="sd">    subsets of the data from indices. It is typically **not needed in</span>
-</span><span id="__span-0-147"><a id="__codelineno-0-147" name="__codelineno-0-147"></a><span class="sd">    algorithms**.</span>
+<span class="normal"><a href="#__codelineno-0-162">162</a></span>
+<span class="normal"><a href="#__codelineno-0-163">163</a></span>
+<span class="normal"><a href="#__codelineno-0-164">164</a></span>
+<span class="normal"><a href="#__codelineno-0-165">165</a></span>
+<span class="normal"><a href="#__codelineno-0-166">166</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-143"><a id="__codelineno-0-143" name="__codelineno-0-143"></a><span class="k">def</span> <span class="nf">get_training_data</span><span class="p">(</span>
+</span><span id="__span-0-144"><a id="__codelineno-0-144" name="__codelineno-0-144"></a>    <span class="bp">self</span><span class="p">,</span> <span class="n">indices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Iterable</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
+</span><span id="__span-0-145"><a id="__codelineno-0-145" name="__codelineno-0-145"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">NDArray</span><span class="p">,</span> <span class="n">NDArray</span><span class="p">]:</span>
+</span><span id="__span-0-146"><a id="__codelineno-0-146" name="__codelineno-0-146"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Given a set of indices, returns the training data that refer to those</span>
+</span><span id="__span-0-147"><a id="__codelineno-0-147" name="__codelineno-0-147"></a><span class="sd">    indices.</span>
 </span><span id="__span-0-148"><a id="__codelineno-0-148" name="__codelineno-0-148"></a>
-</span><span id="__span-0-149"><a id="__codelineno-0-149" name="__codelineno-0-149"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-150"><a id="__codelineno-0-150" name="__codelineno-0-150"></a><span class="sd">        indices: Optional indices that will be used to select points from</span>
-</span><span id="__span-0-151"><a id="__codelineno-0-151" name="__codelineno-0-151"></a><span class="sd">            the training data. If `None`, the entire training data will be</span>
-</span><span id="__span-0-152"><a id="__codelineno-0-152" name="__codelineno-0-152"></a><span class="sd">            returned.</span>
-</span><span id="__span-0-153"><a id="__codelineno-0-153" name="__codelineno-0-153"></a>
-</span><span id="__span-0-154"><a id="__codelineno-0-154" name="__codelineno-0-154"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-155"><a id="__codelineno-0-155" name="__codelineno-0-155"></a><span class="sd">        If `indices` is not `None`, the selected x and y arrays from the</span>
-</span><span id="__span-0-156"><a id="__codelineno-0-156" name="__codelineno-0-156"></a><span class="sd">            training data. Otherwise, the entire dataset.</span>
-</span><span id="__span-0-157"><a id="__codelineno-0-157" name="__codelineno-0-157"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-158"><a id="__codelineno-0-158" name="__codelineno-0-158"></a>    <span class="k">if</span> <span class="n">indices</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-</span><span id="__span-0-159"><a id="__codelineno-0-159" name="__codelineno-0-159"></a>        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">x_train</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">y_train</span>
-</span><span id="__span-0-160"><a id="__codelineno-0-160" name="__codelineno-0-160"></a>    <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">x_train</span><span class="p">[</span><span class="n">indices</span><span class="p">]</span>
-</span><span id="__span-0-161"><a id="__codelineno-0-161" name="__codelineno-0-161"></a>    <span class="n">y</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">y_train</span><span class="p">[</span><span class="n">indices</span><span class="p">]</span>
-</span><span id="__span-0-162"><a id="__codelineno-0-162" name="__codelineno-0-162"></a>    <span class="k">return</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span>
+</span><span id="__span-0-149"><a id="__codelineno-0-149" name="__codelineno-0-149"></a><span class="sd">    This is used mainly by [Utility][pydvl.utils.utility.Utility] to retrieve</span>
+</span><span id="__span-0-150"><a id="__codelineno-0-150" name="__codelineno-0-150"></a><span class="sd">    subsets of the data from indices. It is typically **not needed in</span>
+</span><span id="__span-0-151"><a id="__codelineno-0-151" name="__codelineno-0-151"></a><span class="sd">    algorithms**.</span>
+</span><span id="__span-0-152"><a id="__codelineno-0-152" name="__codelineno-0-152"></a>
+</span><span id="__span-0-153"><a id="__codelineno-0-153" name="__codelineno-0-153"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-154"><a id="__codelineno-0-154" name="__codelineno-0-154"></a><span class="sd">        indices: Optional indices that will be used to select points from</span>
+</span><span id="__span-0-155"><a id="__codelineno-0-155" name="__codelineno-0-155"></a><span class="sd">            the training data. If `None`, the entire training data will be</span>
+</span><span id="__span-0-156"><a id="__codelineno-0-156" name="__codelineno-0-156"></a><span class="sd">            returned.</span>
+</span><span id="__span-0-157"><a id="__codelineno-0-157" name="__codelineno-0-157"></a>
+</span><span id="__span-0-158"><a id="__codelineno-0-158" name="__codelineno-0-158"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-159"><a id="__codelineno-0-159" name="__codelineno-0-159"></a><span class="sd">        If `indices` is not `None`, the selected x and y arrays from the</span>
+</span><span id="__span-0-160"><a id="__codelineno-0-160" name="__codelineno-0-160"></a><span class="sd">            training data. Otherwise, the entire dataset.</span>
+</span><span id="__span-0-161"><a id="__codelineno-0-161" name="__codelineno-0-161"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-162"><a id="__codelineno-0-162" name="__codelineno-0-162"></a>    <span class="k">if</span> <span class="n">indices</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+</span><span id="__span-0-163"><a id="__codelineno-0-163" name="__codelineno-0-163"></a>        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">x_train</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">y_train</span>
+</span><span id="__span-0-164"><a id="__codelineno-0-164" name="__codelineno-0-164"></a>    <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">x_train</span><span class="p">[</span><span class="n">indices</span><span class="p">]</span>
+</span><span id="__span-0-165"><a id="__codelineno-0-165" name="__codelineno-0-165"></a>    <span class="n">y</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">y_train</span><span class="p">[</span><span class="n">indices</span><span class="p">]</span>
+</span><span id="__span-0-166"><a id="__codelineno-0-166" name="__codelineno-0-166"></a>    <span class="k">return</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3452,11 +3460,7 @@ <h3 id="pydvl.utils.dataset.Dataset.get_test_data" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/dataset.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-164">164</a></span>
-<span class="normal"><a href="#__codelineno-0-165">165</a></span>
-<span class="normal"><a href="#__codelineno-0-166">166</a></span>
-<span class="normal"><a href="#__codelineno-0-167">167</a></span>
-<span class="normal"><a href="#__codelineno-0-168">168</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-168">168</a></span>
 <span class="normal"><a href="#__codelineno-0-169">169</a></span>
 <span class="normal"><a href="#__codelineno-0-170">170</a></span>
 <span class="normal"><a href="#__codelineno-0-171">171</a></span>
@@ -3502,57 +3506,61 @@ <h3 id="pydvl.utils.dataset.Dataset.get_test_data" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-211">211</a></span>
 <span class="normal"><a href="#__codelineno-0-212">212</a></span>
 <span class="normal"><a href="#__codelineno-0-213">213</a></span>
-<span class="normal"><a href="#__codelineno-0-214">214</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-164"><a id="__codelineno-0-164" name="__codelineno-0-164"></a><span class="k">def</span> <span class="nf">get_test_data</span><span class="p">(</span>
-</span><span id="__span-0-165"><a id="__codelineno-0-165" name="__codelineno-0-165"></a>    <span class="bp">self</span><span class="p">,</span> <span class="n">indices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Iterable</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
-</span><span id="__span-0-166"><a id="__codelineno-0-166" name="__codelineno-0-166"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">NDArray</span><span class="p">,</span> <span class="n">NDArray</span><span class="p">]:</span>
-</span><span id="__span-0-167"><a id="__codelineno-0-167" name="__codelineno-0-167"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the entire test set regardless of the passed indices.</span>
-</span><span id="__span-0-168"><a id="__codelineno-0-168" name="__codelineno-0-168"></a>
-</span><span id="__span-0-169"><a id="__codelineno-0-169" name="__codelineno-0-169"></a><span class="sd">    The passed indices will not be used because for data valuation</span>
-</span><span id="__span-0-170"><a id="__codelineno-0-170" name="__codelineno-0-170"></a><span class="sd">    we generally want to score the trained model on the entire test data.</span>
-</span><span id="__span-0-171"><a id="__codelineno-0-171" name="__codelineno-0-171"></a>
-</span><span id="__span-0-172"><a id="__codelineno-0-172" name="__codelineno-0-172"></a><span class="sd">    Additionally, the way this method is used in the</span>
-</span><span id="__span-0-173"><a id="__codelineno-0-173" name="__codelineno-0-173"></a><span class="sd">    [Utility][pydvl.utils.utility.Utility] class, the passed indices will</span>
-</span><span id="__span-0-174"><a id="__codelineno-0-174" name="__codelineno-0-174"></a><span class="sd">    be those of the training data and would not work on the test data.</span>
+<span class="normal"><a href="#__codelineno-0-214">214</a></span>
+<span class="normal"><a href="#__codelineno-0-215">215</a></span>
+<span class="normal"><a href="#__codelineno-0-216">216</a></span>
+<span class="normal"><a href="#__codelineno-0-217">217</a></span>
+<span class="normal"><a href="#__codelineno-0-218">218</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-168"><a id="__codelineno-0-168" name="__codelineno-0-168"></a><span class="k">def</span> <span class="nf">get_test_data</span><span class="p">(</span>
+</span><span id="__span-0-169"><a id="__codelineno-0-169" name="__codelineno-0-169"></a>    <span class="bp">self</span><span class="p">,</span> <span class="n">indices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Iterable</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
+</span><span id="__span-0-170"><a id="__codelineno-0-170" name="__codelineno-0-170"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">NDArray</span><span class="p">,</span> <span class="n">NDArray</span><span class="p">]:</span>
+</span><span id="__span-0-171"><a id="__codelineno-0-171" name="__codelineno-0-171"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the entire test set regardless of the passed indices.</span>
+</span><span id="__span-0-172"><a id="__codelineno-0-172" name="__codelineno-0-172"></a>
+</span><span id="__span-0-173"><a id="__codelineno-0-173" name="__codelineno-0-173"></a><span class="sd">    The passed indices will not be used because for data valuation</span>
+</span><span id="__span-0-174"><a id="__codelineno-0-174" name="__codelineno-0-174"></a><span class="sd">    we generally want to score the trained model on the entire test data.</span>
 </span><span id="__span-0-175"><a id="__codelineno-0-175" name="__codelineno-0-175"></a>
-</span><span id="__span-0-176"><a id="__codelineno-0-176" name="__codelineno-0-176"></a><span class="sd">    There may be cases where it is desired to use parts of the test data.</span>
-</span><span id="__span-0-177"><a id="__codelineno-0-177" name="__codelineno-0-177"></a><span class="sd">    In those cases, it is recommended to inherit from</span>
-</span><span id="__span-0-178"><a id="__codelineno-0-178" name="__codelineno-0-178"></a><span class="sd">    [Dataset][pydvl.utils.dataset.Dataset] and override</span>
-</span><span id="__span-0-179"><a id="__codelineno-0-179" name="__codelineno-0-179"></a><span class="sd">    [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data].</span>
-</span><span id="__span-0-180"><a id="__codelineno-0-180" name="__codelineno-0-180"></a>
-</span><span id="__span-0-181"><a id="__codelineno-0-181" name="__codelineno-0-181"></a><span class="sd">    For example, the following snippet shows how one could go about</span>
-</span><span id="__span-0-182"><a id="__codelineno-0-182" name="__codelineno-0-182"></a><span class="sd">    mapping the training data indices into test data indices</span>
-</span><span id="__span-0-183"><a id="__codelineno-0-183" name="__codelineno-0-183"></a><span class="sd">    inside [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data]:</span>
+</span><span id="__span-0-176"><a id="__codelineno-0-176" name="__codelineno-0-176"></a><span class="sd">    Additionally, the way this method is used in the</span>
+</span><span id="__span-0-177"><a id="__codelineno-0-177" name="__codelineno-0-177"></a><span class="sd">    [Utility][pydvl.utils.utility.Utility] class, the passed indices will</span>
+</span><span id="__span-0-178"><a id="__codelineno-0-178" name="__codelineno-0-178"></a><span class="sd">    be those of the training data and would not work on the test data.</span>
+</span><span id="__span-0-179"><a id="__codelineno-0-179" name="__codelineno-0-179"></a>
+</span><span id="__span-0-180"><a id="__codelineno-0-180" name="__codelineno-0-180"></a><span class="sd">    There may be cases where it is desired to use parts of the test data.</span>
+</span><span id="__span-0-181"><a id="__codelineno-0-181" name="__codelineno-0-181"></a><span class="sd">    In those cases, it is recommended to inherit from</span>
+</span><span id="__span-0-182"><a id="__codelineno-0-182" name="__codelineno-0-182"></a><span class="sd">    [Dataset][pydvl.utils.dataset.Dataset] and override</span>
+</span><span id="__span-0-183"><a id="__codelineno-0-183" name="__codelineno-0-183"></a><span class="sd">    [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data].</span>
 </span><span id="__span-0-184"><a id="__codelineno-0-184" name="__codelineno-0-184"></a>
-</span><span id="__span-0-185"><a id="__codelineno-0-185" name="__codelineno-0-185"></a><span class="sd">    ??? Example</span>
-</span><span id="__span-0-186"><a id="__codelineno-0-186" name="__codelineno-0-186"></a><span class="sd">        ```pycon</span>
-</span><span id="__span-0-187"><a id="__codelineno-0-187" name="__codelineno-0-187"></a><span class="sd">        &gt;&gt;&gt; from pydvl.utils import Dataset</span>
-</span><span id="__span-0-188"><a id="__codelineno-0-188" name="__codelineno-0-188"></a><span class="sd">        &gt;&gt;&gt; import numpy as np</span>
-</span><span id="__span-0-189"><a id="__codelineno-0-189" name="__codelineno-0-189"></a><span class="sd">        &gt;&gt;&gt; class DatasetWithTestDataIndices(Dataset):</span>
-</span><span id="__span-0-190"><a id="__codelineno-0-190" name="__codelineno-0-190"></a><span class="sd">        ...    def get_test_data(self, indices=None):</span>
-</span><span id="__span-0-191"><a id="__codelineno-0-191" name="__codelineno-0-191"></a><span class="sd">        ...        if indices is None:</span>
-</span><span id="__span-0-192"><a id="__codelineno-0-192" name="__codelineno-0-192"></a><span class="sd">        ...            return self.x_test, self.y_test</span>
-</span><span id="__span-0-193"><a id="__codelineno-0-193" name="__codelineno-0-193"></a><span class="sd">        ...        fraction = len(list(indices)) / len(self)</span>
-</span><span id="__span-0-194"><a id="__codelineno-0-194" name="__codelineno-0-194"></a><span class="sd">        ...        mapped_indices = len(self.x_test) / len(self) * np.asarray(indices)</span>
-</span><span id="__span-0-195"><a id="__codelineno-0-195" name="__codelineno-0-195"></a><span class="sd">        ...        mapped_indices = np.unique(mapped_indices.astype(int))</span>
-</span><span id="__span-0-196"><a id="__codelineno-0-196" name="__codelineno-0-196"></a><span class="sd">        ...        return self.x_test[mapped_indices], self.y_test[mapped_indices]</span>
-</span><span id="__span-0-197"><a id="__codelineno-0-197" name="__codelineno-0-197"></a><span class="sd">        ...</span>
-</span><span id="__span-0-198"><a id="__codelineno-0-198" name="__codelineno-0-198"></a><span class="sd">        &gt;&gt;&gt; X = np.random.rand(100, 10)</span>
-</span><span id="__span-0-199"><a id="__codelineno-0-199" name="__codelineno-0-199"></a><span class="sd">        &gt;&gt;&gt; y = np.random.randint(0, 2, 100)</span>
-</span><span id="__span-0-200"><a id="__codelineno-0-200" name="__codelineno-0-200"></a><span class="sd">        &gt;&gt;&gt; dataset = DatasetWithTestDataIndices.from_arrays(X, y)</span>
-</span><span id="__span-0-201"><a id="__codelineno-0-201" name="__codelineno-0-201"></a><span class="sd">        &gt;&gt;&gt; indices = np.random.choice(dataset.indices, 30, replace=False)</span>
-</span><span id="__span-0-202"><a id="__codelineno-0-202" name="__codelineno-0-202"></a><span class="sd">        &gt;&gt;&gt; _ = dataset.get_training_data(indices)</span>
-</span><span id="__span-0-203"><a id="__codelineno-0-203" name="__codelineno-0-203"></a><span class="sd">        &gt;&gt;&gt; _ = dataset.get_test_data(indices)</span>
-</span><span id="__span-0-204"><a id="__codelineno-0-204" name="__codelineno-0-204"></a><span class="sd">        ```</span>
-</span><span id="__span-0-205"><a id="__codelineno-0-205" name="__codelineno-0-205"></a>
-</span><span id="__span-0-206"><a id="__codelineno-0-206" name="__codelineno-0-206"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-207"><a id="__codelineno-0-207" name="__codelineno-0-207"></a><span class="sd">        indices: Optional indices into the test data. This argument is</span>
-</span><span id="__span-0-208"><a id="__codelineno-0-208" name="__codelineno-0-208"></a><span class="sd">            unused left for compatibility with</span>
-</span><span id="__span-0-209"><a id="__codelineno-0-209" name="__codelineno-0-209"></a><span class="sd">            [get_training_data()][pydvl.utils.dataset.Dataset.get_training_data].</span>
-</span><span id="__span-0-210"><a id="__codelineno-0-210" name="__codelineno-0-210"></a>
-</span><span id="__span-0-211"><a id="__codelineno-0-211" name="__codelineno-0-211"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-212"><a id="__codelineno-0-212" name="__codelineno-0-212"></a><span class="sd">        The entire test data.</span>
-</span><span id="__span-0-213"><a id="__codelineno-0-213" name="__codelineno-0-213"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-214"><a id="__codelineno-0-214" name="__codelineno-0-214"></a>    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">x_test</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">y_test</span>
+</span><span id="__span-0-185"><a id="__codelineno-0-185" name="__codelineno-0-185"></a><span class="sd">    For example, the following snippet shows how one could go about</span>
+</span><span id="__span-0-186"><a id="__codelineno-0-186" name="__codelineno-0-186"></a><span class="sd">    mapping the training data indices into test data indices</span>
+</span><span id="__span-0-187"><a id="__codelineno-0-187" name="__codelineno-0-187"></a><span class="sd">    inside [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data]:</span>
+</span><span id="__span-0-188"><a id="__codelineno-0-188" name="__codelineno-0-188"></a>
+</span><span id="__span-0-189"><a id="__codelineno-0-189" name="__codelineno-0-189"></a><span class="sd">    ??? Example</span>
+</span><span id="__span-0-190"><a id="__codelineno-0-190" name="__codelineno-0-190"></a><span class="sd">        ```pycon</span>
+</span><span id="__span-0-191"><a id="__codelineno-0-191" name="__codelineno-0-191"></a><span class="sd">        &gt;&gt;&gt; from pydvl.utils import Dataset</span>
+</span><span id="__span-0-192"><a id="__codelineno-0-192" name="__codelineno-0-192"></a><span class="sd">        &gt;&gt;&gt; import numpy as np</span>
+</span><span id="__span-0-193"><a id="__codelineno-0-193" name="__codelineno-0-193"></a><span class="sd">        &gt;&gt;&gt; class DatasetWithTestDataIndices(Dataset):</span>
+</span><span id="__span-0-194"><a id="__codelineno-0-194" name="__codelineno-0-194"></a><span class="sd">        ...    def get_test_data(self, indices=None):</span>
+</span><span id="__span-0-195"><a id="__codelineno-0-195" name="__codelineno-0-195"></a><span class="sd">        ...        if indices is None:</span>
+</span><span id="__span-0-196"><a id="__codelineno-0-196" name="__codelineno-0-196"></a><span class="sd">        ...            return self.x_test, self.y_test</span>
+</span><span id="__span-0-197"><a id="__codelineno-0-197" name="__codelineno-0-197"></a><span class="sd">        ...        fraction = len(list(indices)) / len(self)</span>
+</span><span id="__span-0-198"><a id="__codelineno-0-198" name="__codelineno-0-198"></a><span class="sd">        ...        mapped_indices = len(self.x_test) / len(self) * np.asarray(indices)</span>
+</span><span id="__span-0-199"><a id="__codelineno-0-199" name="__codelineno-0-199"></a><span class="sd">        ...        mapped_indices = np.unique(mapped_indices.astype(int))</span>
+</span><span id="__span-0-200"><a id="__codelineno-0-200" name="__codelineno-0-200"></a><span class="sd">        ...        return self.x_test[mapped_indices], self.y_test[mapped_indices]</span>
+</span><span id="__span-0-201"><a id="__codelineno-0-201" name="__codelineno-0-201"></a><span class="sd">        ...</span>
+</span><span id="__span-0-202"><a id="__codelineno-0-202" name="__codelineno-0-202"></a><span class="sd">        &gt;&gt;&gt; X = np.random.rand(100, 10)</span>
+</span><span id="__span-0-203"><a id="__codelineno-0-203" name="__codelineno-0-203"></a><span class="sd">        &gt;&gt;&gt; y = np.random.randint(0, 2, 100)</span>
+</span><span id="__span-0-204"><a id="__codelineno-0-204" name="__codelineno-0-204"></a><span class="sd">        &gt;&gt;&gt; dataset = DatasetWithTestDataIndices.from_arrays(X, y)</span>
+</span><span id="__span-0-205"><a id="__codelineno-0-205" name="__codelineno-0-205"></a><span class="sd">        &gt;&gt;&gt; indices = np.random.choice(dataset.indices, 30, replace=False)</span>
+</span><span id="__span-0-206"><a id="__codelineno-0-206" name="__codelineno-0-206"></a><span class="sd">        &gt;&gt;&gt; _ = dataset.get_training_data(indices)</span>
+</span><span id="__span-0-207"><a id="__codelineno-0-207" name="__codelineno-0-207"></a><span class="sd">        &gt;&gt;&gt; _ = dataset.get_test_data(indices)</span>
+</span><span id="__span-0-208"><a id="__codelineno-0-208" name="__codelineno-0-208"></a><span class="sd">        ```</span>
+</span><span id="__span-0-209"><a id="__codelineno-0-209" name="__codelineno-0-209"></a>
+</span><span id="__span-0-210"><a id="__codelineno-0-210" name="__codelineno-0-210"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-211"><a id="__codelineno-0-211" name="__codelineno-0-211"></a><span class="sd">        indices: Optional indices into the test data. This argument is</span>
+</span><span id="__span-0-212"><a id="__codelineno-0-212" name="__codelineno-0-212"></a><span class="sd">            unused left for compatibility with</span>
+</span><span id="__span-0-213"><a id="__codelineno-0-213" name="__codelineno-0-213"></a><span class="sd">            [get_training_data()][pydvl.utils.dataset.Dataset.get_training_data].</span>
+</span><span id="__span-0-214"><a id="__codelineno-0-214" name="__codelineno-0-214"></a>
+</span><span id="__span-0-215"><a id="__codelineno-0-215" name="__codelineno-0-215"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-216"><a id="__codelineno-0-216" name="__codelineno-0-216"></a><span class="sd">        The entire test data.</span>
+</span><span id="__span-0-217"><a id="__codelineno-0-217" name="__codelineno-0-217"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-218"><a id="__codelineno-0-218" name="__codelineno-0-218"></a>    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">x_test</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">y_test</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3724,11 +3732,7 @@ <h3 id="pydvl.utils.dataset.Dataset.from_sklearn" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/dataset.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-249">249</a></span>
-<span class="normal"><a href="#__codelineno-0-250">250</a></span>
-<span class="normal"><a href="#__codelineno-0-251">251</a></span>
-<span class="normal"><a href="#__codelineno-0-252">252</a></span>
-<span class="normal"><a href="#__codelineno-0-253">253</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-253">253</a></span>
 <span class="normal"><a href="#__codelineno-0-254">254</a></span>
 <span class="normal"><a href="#__codelineno-0-255">255</a></span>
 <span class="normal"><a href="#__codelineno-0-256">256</a></span>
@@ -3782,65 +3786,69 @@ <h3 id="pydvl.utils.dataset.Dataset.from_sklearn" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-304">304</a></span>
 <span class="normal"><a href="#__codelineno-0-305">305</a></span>
 <span class="normal"><a href="#__codelineno-0-306">306</a></span>
-<span class="normal"><a href="#__codelineno-0-307">307</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-249"><a id="__codelineno-0-249" name="__codelineno-0-249"></a><span class="nd">@classmethod</span>
-</span><span id="__span-0-250"><a id="__codelineno-0-250" name="__codelineno-0-250"></a><span class="k">def</span> <span class="nf">from_sklearn</span><span class="p">(</span>
-</span><span id="__span-0-251"><a id="__codelineno-0-251" name="__codelineno-0-251"></a>    <span class="bp">cls</span><span class="p">,</span>
-</span><span id="__span-0-252"><a id="__codelineno-0-252" name="__codelineno-0-252"></a>    <span class="n">data</span><span class="p">:</span> <span class="n">Bunch</span><span class="p">,</span>
-</span><span id="__span-0-253"><a id="__codelineno-0-253" name="__codelineno-0-253"></a>    <span class="n">train_size</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.8</span><span class="p">,</span>
-</span><span id="__span-0-254"><a id="__codelineno-0-254" name="__codelineno-0-254"></a>    <span class="n">random_state</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-255"><a id="__codelineno-0-255" name="__codelineno-0-255"></a>    <span class="n">stratify_by_target</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-</span><span id="__span-0-256"><a id="__codelineno-0-256" name="__codelineno-0-256"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-</span><span id="__span-0-257"><a id="__codelineno-0-257" name="__codelineno-0-257"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Dataset&quot;</span><span class="p">:</span>
-</span><span id="__span-0-258"><a id="__codelineno-0-258" name="__codelineno-0-258"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Constructs a [Dataset][pydvl.utils.Dataset] object from a</span>
-</span><span id="__span-0-259"><a id="__codelineno-0-259" name="__codelineno-0-259"></a><span class="sd">    [sklearn.utils.Bunch][sklearn.utils.Bunch], as returned by the `load_*`</span>
-</span><span id="__span-0-260"><a id="__codelineno-0-260" name="__codelineno-0-260"></a><span class="sd">    functions in [scikit-learn toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html).</span>
-</span><span id="__span-0-261"><a id="__codelineno-0-261" name="__codelineno-0-261"></a>
-</span><span id="__span-0-262"><a id="__codelineno-0-262" name="__codelineno-0-262"></a><span class="sd">    ??? Example</span>
-</span><span id="__span-0-263"><a id="__codelineno-0-263" name="__codelineno-0-263"></a><span class="sd">        ```pycon</span>
-</span><span id="__span-0-264"><a id="__codelineno-0-264" name="__codelineno-0-264"></a><span class="sd">        &gt;&gt;&gt; from pydvl.utils import Dataset</span>
-</span><span id="__span-0-265"><a id="__codelineno-0-265" name="__codelineno-0-265"></a><span class="sd">        &gt;&gt;&gt; from sklearn.datasets import load_boston</span>
-</span><span id="__span-0-266"><a id="__codelineno-0-266" name="__codelineno-0-266"></a><span class="sd">        &gt;&gt;&gt; dataset = Dataset.from_sklearn(load_boston())</span>
-</span><span id="__span-0-267"><a id="__codelineno-0-267" name="__codelineno-0-267"></a><span class="sd">        ```</span>
-</span><span id="__span-0-268"><a id="__codelineno-0-268" name="__codelineno-0-268"></a>
-</span><span id="__span-0-269"><a id="__codelineno-0-269" name="__codelineno-0-269"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-270"><a id="__codelineno-0-270" name="__codelineno-0-270"></a><span class="sd">        data: scikit-learn Bunch object. The following attributes are supported:</span>
-</span><span id="__span-0-271"><a id="__codelineno-0-271" name="__codelineno-0-271"></a>
-</span><span id="__span-0-272"><a id="__codelineno-0-272" name="__codelineno-0-272"></a><span class="sd">            - `data`: covariates.</span>
-</span><span id="__span-0-273"><a id="__codelineno-0-273" name="__codelineno-0-273"></a><span class="sd">            - `target`: target variables (labels).</span>
-</span><span id="__span-0-274"><a id="__codelineno-0-274" name="__codelineno-0-274"></a><span class="sd">            - `feature_names` (**optional**): the feature names.</span>
-</span><span id="__span-0-275"><a id="__codelineno-0-275" name="__codelineno-0-275"></a><span class="sd">            - `target_names` (**optional**): the target names.</span>
-</span><span id="__span-0-276"><a id="__codelineno-0-276" name="__codelineno-0-276"></a><span class="sd">            - `DESCR` (**optional**): a description.</span>
-</span><span id="__span-0-277"><a id="__codelineno-0-277" name="__codelineno-0-277"></a><span class="sd">        train_size: size of the training dataset. Used in `train_test_split`</span>
-</span><span id="__span-0-278"><a id="__codelineno-0-278" name="__codelineno-0-278"></a><span class="sd">        random_state: seed for train / test split</span>
-</span><span id="__span-0-279"><a id="__codelineno-0-279" name="__codelineno-0-279"></a><span class="sd">        stratify_by_target: If `True`, data is split in a stratified</span>
-</span><span id="__span-0-280"><a id="__codelineno-0-280" name="__codelineno-0-280"></a><span class="sd">            fashion, using the target variable as labels. Read more in</span>
-</span><span id="__span-0-281"><a id="__codelineno-0-281" name="__codelineno-0-281"></a><span class="sd">            [scikit-learn&#39;s user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).</span>
-</span><span id="__span-0-282"><a id="__codelineno-0-282" name="__codelineno-0-282"></a><span class="sd">        kwargs: Additional keyword arguments to pass to the</span>
-</span><span id="__span-0-283"><a id="__codelineno-0-283" name="__codelineno-0-283"></a><span class="sd">            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `is_multi_output`.</span>
-</span><span id="__span-0-284"><a id="__codelineno-0-284" name="__codelineno-0-284"></a>
-</span><span id="__span-0-285"><a id="__codelineno-0-285" name="__codelineno-0-285"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-286"><a id="__codelineno-0-286" name="__codelineno-0-286"></a><span class="sd">        Object with the sklearn dataset</span>
-</span><span id="__span-0-287"><a id="__codelineno-0-287" name="__codelineno-0-287"></a>
-</span><span id="__span-0-288"><a id="__codelineno-0-288" name="__codelineno-0-288"></a><span class="sd">    !!! tip &quot;Changed in version 0.6.0&quot;</span>
-</span><span id="__span-0-289"><a id="__codelineno-0-289" name="__codelineno-0-289"></a><span class="sd">        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.</span>
-</span><span id="__span-0-290"><a id="__codelineno-0-290" name="__codelineno-0-290"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-291"><a id="__codelineno-0-291" name="__codelineno-0-291"></a>    <span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span>
-</span><span id="__span-0-292"><a id="__codelineno-0-292" name="__codelineno-0-292"></a>        <span class="n">data</span><span class="o">.</span><span class="n">data</span><span class="p">,</span>
-</span><span id="__span-0-293"><a id="__codelineno-0-293" name="__codelineno-0-293"></a>        <span class="n">data</span><span class="o">.</span><span class="n">target</span><span class="p">,</span>
-</span><span id="__span-0-294"><a id="__codelineno-0-294" name="__codelineno-0-294"></a>        <span class="n">train_size</span><span class="o">=</span><span class="n">train_size</span><span class="p">,</span>
-</span><span id="__span-0-295"><a id="__codelineno-0-295" name="__codelineno-0-295"></a>        <span class="n">random_state</span><span class="o">=</span><span class="n">random_state</span><span class="p">,</span>
-</span><span id="__span-0-296"><a id="__codelineno-0-296" name="__codelineno-0-296"></a>        <span class="n">stratify</span><span class="o">=</span><span class="n">data</span><span class="o">.</span><span class="n">target</span> <span class="k">if</span> <span class="n">stratify_by_target</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-297"><a id="__codelineno-0-297" name="__codelineno-0-297"></a>    <span class="p">)</span>
-</span><span id="__span-0-298"><a id="__codelineno-0-298" name="__codelineno-0-298"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span>
-</span><span id="__span-0-299"><a id="__codelineno-0-299" name="__codelineno-0-299"></a>        <span class="n">x_train</span><span class="p">,</span>
-</span><span id="__span-0-300"><a id="__codelineno-0-300" name="__codelineno-0-300"></a>        <span class="n">y_train</span><span class="p">,</span>
-</span><span id="__span-0-301"><a id="__codelineno-0-301" name="__codelineno-0-301"></a>        <span class="n">x_test</span><span class="p">,</span>
-</span><span id="__span-0-302"><a id="__codelineno-0-302" name="__codelineno-0-302"></a>        <span class="n">y_test</span><span class="p">,</span>
-</span><span id="__span-0-303"><a id="__codelineno-0-303" name="__codelineno-0-303"></a>        <span class="n">feature_names</span><span class="o">=</span><span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;feature_names&quot;</span><span class="p">),</span>
-</span><span id="__span-0-304"><a id="__codelineno-0-304" name="__codelineno-0-304"></a>        <span class="n">target_names</span><span class="o">=</span><span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;target_names&quot;</span><span class="p">),</span>
-</span><span id="__span-0-305"><a id="__codelineno-0-305" name="__codelineno-0-305"></a>        <span class="n">description</span><span class="o">=</span><span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;DESCR&quot;</span><span class="p">),</span>
-</span><span id="__span-0-306"><a id="__codelineno-0-306" name="__codelineno-0-306"></a>        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-</span><span id="__span-0-307"><a id="__codelineno-0-307" name="__codelineno-0-307"></a>    <span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-307">307</a></span>
+<span class="normal"><a href="#__codelineno-0-308">308</a></span>
+<span class="normal"><a href="#__codelineno-0-309">309</a></span>
+<span class="normal"><a href="#__codelineno-0-310">310</a></span>
+<span class="normal"><a href="#__codelineno-0-311">311</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-253"><a id="__codelineno-0-253" name="__codelineno-0-253"></a><span class="nd">@classmethod</span>
+</span><span id="__span-0-254"><a id="__codelineno-0-254" name="__codelineno-0-254"></a><span class="k">def</span> <span class="nf">from_sklearn</span><span class="p">(</span>
+</span><span id="__span-0-255"><a id="__codelineno-0-255" name="__codelineno-0-255"></a>    <span class="bp">cls</span><span class="p">,</span>
+</span><span id="__span-0-256"><a id="__codelineno-0-256" name="__codelineno-0-256"></a>    <span class="n">data</span><span class="p">:</span> <span class="n">Bunch</span><span class="p">,</span>
+</span><span id="__span-0-257"><a id="__codelineno-0-257" name="__codelineno-0-257"></a>    <span class="n">train_size</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.8</span><span class="p">,</span>
+</span><span id="__span-0-258"><a id="__codelineno-0-258" name="__codelineno-0-258"></a>    <span class="n">random_state</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-259"><a id="__codelineno-0-259" name="__codelineno-0-259"></a>    <span class="n">stratify_by_target</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+</span><span id="__span-0-260"><a id="__codelineno-0-260" name="__codelineno-0-260"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+</span><span id="__span-0-261"><a id="__codelineno-0-261" name="__codelineno-0-261"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Dataset&quot;</span><span class="p">:</span>
+</span><span id="__span-0-262"><a id="__codelineno-0-262" name="__codelineno-0-262"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Constructs a [Dataset][pydvl.utils.Dataset] object from a</span>
+</span><span id="__span-0-263"><a id="__codelineno-0-263" name="__codelineno-0-263"></a><span class="sd">    [sklearn.utils.Bunch][], as returned by the `load_*`</span>
+</span><span id="__span-0-264"><a id="__codelineno-0-264" name="__codelineno-0-264"></a><span class="sd">    functions in [scikit-learn toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html).</span>
+</span><span id="__span-0-265"><a id="__codelineno-0-265" name="__codelineno-0-265"></a>
+</span><span id="__span-0-266"><a id="__codelineno-0-266" name="__codelineno-0-266"></a><span class="sd">    ??? Example</span>
+</span><span id="__span-0-267"><a id="__codelineno-0-267" name="__codelineno-0-267"></a><span class="sd">        ```pycon</span>
+</span><span id="__span-0-268"><a id="__codelineno-0-268" name="__codelineno-0-268"></a><span class="sd">        &gt;&gt;&gt; from pydvl.utils import Dataset</span>
+</span><span id="__span-0-269"><a id="__codelineno-0-269" name="__codelineno-0-269"></a><span class="sd">        &gt;&gt;&gt; from sklearn.datasets import load_boston</span>
+</span><span id="__span-0-270"><a id="__codelineno-0-270" name="__codelineno-0-270"></a><span class="sd">        &gt;&gt;&gt; dataset = Dataset.from_sklearn(load_boston())</span>
+</span><span id="__span-0-271"><a id="__codelineno-0-271" name="__codelineno-0-271"></a><span class="sd">        ```</span>
+</span><span id="__span-0-272"><a id="__codelineno-0-272" name="__codelineno-0-272"></a>
+</span><span id="__span-0-273"><a id="__codelineno-0-273" name="__codelineno-0-273"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-274"><a id="__codelineno-0-274" name="__codelineno-0-274"></a><span class="sd">        data: scikit-learn Bunch object. The following attributes are supported:</span>
+</span><span id="__span-0-275"><a id="__codelineno-0-275" name="__codelineno-0-275"></a>
+</span><span id="__span-0-276"><a id="__codelineno-0-276" name="__codelineno-0-276"></a><span class="sd">            - `data`: covariates.</span>
+</span><span id="__span-0-277"><a id="__codelineno-0-277" name="__codelineno-0-277"></a><span class="sd">            - `target`: target variables (labels).</span>
+</span><span id="__span-0-278"><a id="__codelineno-0-278" name="__codelineno-0-278"></a><span class="sd">            - `feature_names` (**optional**): the feature names.</span>
+</span><span id="__span-0-279"><a id="__codelineno-0-279" name="__codelineno-0-279"></a><span class="sd">            - `target_names` (**optional**): the target names.</span>
+</span><span id="__span-0-280"><a id="__codelineno-0-280" name="__codelineno-0-280"></a><span class="sd">            - `DESCR` (**optional**): a description.</span>
+</span><span id="__span-0-281"><a id="__codelineno-0-281" name="__codelineno-0-281"></a><span class="sd">        train_size: size of the training dataset. Used in `train_test_split`</span>
+</span><span id="__span-0-282"><a id="__codelineno-0-282" name="__codelineno-0-282"></a><span class="sd">        random_state: seed for train / test split</span>
+</span><span id="__span-0-283"><a id="__codelineno-0-283" name="__codelineno-0-283"></a><span class="sd">        stratify_by_target: If `True`, data is split in a stratified</span>
+</span><span id="__span-0-284"><a id="__codelineno-0-284" name="__codelineno-0-284"></a><span class="sd">            fashion, using the target variable as labels. Read more in</span>
+</span><span id="__span-0-285"><a id="__codelineno-0-285" name="__codelineno-0-285"></a><span class="sd">            [scikit-learn&#39;s user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).</span>
+</span><span id="__span-0-286"><a id="__codelineno-0-286" name="__codelineno-0-286"></a><span class="sd">        kwargs: Additional keyword arguments to pass to the</span>
+</span><span id="__span-0-287"><a id="__codelineno-0-287" name="__codelineno-0-287"></a><span class="sd">            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `is_multi_output`.</span>
+</span><span id="__span-0-288"><a id="__codelineno-0-288" name="__codelineno-0-288"></a>
+</span><span id="__span-0-289"><a id="__codelineno-0-289" name="__codelineno-0-289"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-290"><a id="__codelineno-0-290" name="__codelineno-0-290"></a><span class="sd">        Object with the sklearn dataset</span>
+</span><span id="__span-0-291"><a id="__codelineno-0-291" name="__codelineno-0-291"></a>
+</span><span id="__span-0-292"><a id="__codelineno-0-292" name="__codelineno-0-292"></a><span class="sd">    !!! tip &quot;Changed in version 0.6.0&quot;</span>
+</span><span id="__span-0-293"><a id="__codelineno-0-293" name="__codelineno-0-293"></a><span class="sd">        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.</span>
+</span><span id="__span-0-294"><a id="__codelineno-0-294" name="__codelineno-0-294"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-295"><a id="__codelineno-0-295" name="__codelineno-0-295"></a>    <span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span>
+</span><span id="__span-0-296"><a id="__codelineno-0-296" name="__codelineno-0-296"></a>        <span class="n">data</span><span class="o">.</span><span class="n">data</span><span class="p">,</span>
+</span><span id="__span-0-297"><a id="__codelineno-0-297" name="__codelineno-0-297"></a>        <span class="n">data</span><span class="o">.</span><span class="n">target</span><span class="p">,</span>
+</span><span id="__span-0-298"><a id="__codelineno-0-298" name="__codelineno-0-298"></a>        <span class="n">train_size</span><span class="o">=</span><span class="n">train_size</span><span class="p">,</span>
+</span><span id="__span-0-299"><a id="__codelineno-0-299" name="__codelineno-0-299"></a>        <span class="n">random_state</span><span class="o">=</span><span class="n">random_state</span><span class="p">,</span>
+</span><span id="__span-0-300"><a id="__codelineno-0-300" name="__codelineno-0-300"></a>        <span class="n">stratify</span><span class="o">=</span><span class="n">data</span><span class="o">.</span><span class="n">target</span> <span class="k">if</span> <span class="n">stratify_by_target</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-301"><a id="__codelineno-0-301" name="__codelineno-0-301"></a>    <span class="p">)</span>
+</span><span id="__span-0-302"><a id="__codelineno-0-302" name="__codelineno-0-302"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span>
+</span><span id="__span-0-303"><a id="__codelineno-0-303" name="__codelineno-0-303"></a>        <span class="n">x_train</span><span class="p">,</span>
+</span><span id="__span-0-304"><a id="__codelineno-0-304" name="__codelineno-0-304"></a>        <span class="n">y_train</span><span class="p">,</span>
+</span><span id="__span-0-305"><a id="__codelineno-0-305" name="__codelineno-0-305"></a>        <span class="n">x_test</span><span class="p">,</span>
+</span><span id="__span-0-306"><a id="__codelineno-0-306" name="__codelineno-0-306"></a>        <span class="n">y_test</span><span class="p">,</span>
+</span><span id="__span-0-307"><a id="__codelineno-0-307" name="__codelineno-0-307"></a>        <span class="n">feature_names</span><span class="o">=</span><span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;feature_names&quot;</span><span class="p">),</span>
+</span><span id="__span-0-308"><a id="__codelineno-0-308" name="__codelineno-0-308"></a>        <span class="n">target_names</span><span class="o">=</span><span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;target_names&quot;</span><span class="p">),</span>
+</span><span id="__span-0-309"><a id="__codelineno-0-309" name="__codelineno-0-309"></a>        <span class="n">description</span><span class="o">=</span><span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;DESCR&quot;</span><span class="p">),</span>
+</span><span id="__span-0-310"><a id="__codelineno-0-310" name="__codelineno-0-310"></a>        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+</span><span id="__span-0-311"><a id="__codelineno-0-311" name="__codelineno-0-311"></a>    <span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -4023,11 +4031,7 @@ <h3 id="pydvl.utils.dataset.Dataset.from_arrays" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/dataset.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-309">309</a></span>
-<span class="normal"><a href="#__codelineno-0-310">310</a></span>
-<span class="normal"><a href="#__codelineno-0-311">311</a></span>
-<span class="normal"><a href="#__codelineno-0-312">312</a></span>
-<span class="normal"><a href="#__codelineno-0-313">313</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-313">313</a></span>
 <span class="normal"><a href="#__codelineno-0-314">314</a></span>
 <span class="normal"><a href="#__codelineno-0-315">315</a></span>
 <span class="normal"><a href="#__codelineno-0-316">316</a></span>
@@ -4071,55 +4075,59 @@ <h3 id="pydvl.utils.dataset.Dataset.from_arrays" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-354">354</a></span>
 <span class="normal"><a href="#__codelineno-0-355">355</a></span>
 <span class="normal"><a href="#__codelineno-0-356">356</a></span>
-<span class="normal"><a href="#__codelineno-0-357">357</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-309"><a id="__codelineno-0-309" name="__codelineno-0-309"></a><span class="nd">@classmethod</span>
-</span><span id="__span-0-310"><a id="__codelineno-0-310" name="__codelineno-0-310"></a><span class="k">def</span> <span class="nf">from_arrays</span><span class="p">(</span>
-</span><span id="__span-0-311"><a id="__codelineno-0-311" name="__codelineno-0-311"></a>    <span class="bp">cls</span><span class="p">,</span>
-</span><span id="__span-0-312"><a id="__codelineno-0-312" name="__codelineno-0-312"></a>    <span class="n">X</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span>
-</span><span id="__span-0-313"><a id="__codelineno-0-313" name="__codelineno-0-313"></a>    <span class="n">y</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span>
-</span><span id="__span-0-314"><a id="__codelineno-0-314" name="__codelineno-0-314"></a>    <span class="n">train_size</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.8</span><span class="p">,</span>
-</span><span id="__span-0-315"><a id="__codelineno-0-315" name="__codelineno-0-315"></a>    <span class="n">random_state</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-316"><a id="__codelineno-0-316" name="__codelineno-0-316"></a>    <span class="n">stratify_by_target</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-</span><span id="__span-0-317"><a id="__codelineno-0-317" name="__codelineno-0-317"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-</span><span id="__span-0-318"><a id="__codelineno-0-318" name="__codelineno-0-318"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Dataset&quot;</span><span class="p">:</span>
-</span><span id="__span-0-319"><a id="__codelineno-0-319" name="__codelineno-0-319"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Constructs a [Dataset][pydvl.utils.Dataset] object from X and y numpy arrays  as</span>
-</span><span id="__span-0-320"><a id="__codelineno-0-320" name="__codelineno-0-320"></a><span class="sd">    returned by the `make_*` functions in [sklearn generated datasets](https://scikit-learn.org/stable/datasets/sample_generators.html).</span>
-</span><span id="__span-0-321"><a id="__codelineno-0-321" name="__codelineno-0-321"></a>
-</span><span id="__span-0-322"><a id="__codelineno-0-322" name="__codelineno-0-322"></a><span class="sd">    ??? Example</span>
-</span><span id="__span-0-323"><a id="__codelineno-0-323" name="__codelineno-0-323"></a><span class="sd">        ```pycon</span>
-</span><span id="__span-0-324"><a id="__codelineno-0-324" name="__codelineno-0-324"></a><span class="sd">        &gt;&gt;&gt; from pydvl.utils import Dataset</span>
-</span><span id="__span-0-325"><a id="__codelineno-0-325" name="__codelineno-0-325"></a><span class="sd">        &gt;&gt;&gt; from sklearn.datasets import make_regression</span>
-</span><span id="__span-0-326"><a id="__codelineno-0-326" name="__codelineno-0-326"></a><span class="sd">        &gt;&gt;&gt; X, y = make_regression()</span>
-</span><span id="__span-0-327"><a id="__codelineno-0-327" name="__codelineno-0-327"></a><span class="sd">        &gt;&gt;&gt; dataset = Dataset.from_arrays(X, y)</span>
-</span><span id="__span-0-328"><a id="__codelineno-0-328" name="__codelineno-0-328"></a><span class="sd">        ```</span>
-</span><span id="__span-0-329"><a id="__codelineno-0-329" name="__codelineno-0-329"></a>
-</span><span id="__span-0-330"><a id="__codelineno-0-330" name="__codelineno-0-330"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-331"><a id="__codelineno-0-331" name="__codelineno-0-331"></a><span class="sd">        X: numpy array of shape (n_samples, n_features)</span>
-</span><span id="__span-0-332"><a id="__codelineno-0-332" name="__codelineno-0-332"></a><span class="sd">        y: numpy array of shape (n_samples,)</span>
-</span><span id="__span-0-333"><a id="__codelineno-0-333" name="__codelineno-0-333"></a><span class="sd">        train_size: size of the training dataset. Used in `train_test_split`</span>
-</span><span id="__span-0-334"><a id="__codelineno-0-334" name="__codelineno-0-334"></a><span class="sd">        random_state: seed for train / test split</span>
-</span><span id="__span-0-335"><a id="__codelineno-0-335" name="__codelineno-0-335"></a><span class="sd">        stratify_by_target: If `True`, data is split in a stratified fashion,</span>
-</span><span id="__span-0-336"><a id="__codelineno-0-336" name="__codelineno-0-336"></a><span class="sd">            using the y variable as labels. Read more in [sklearn&#39;s user</span>
-</span><span id="__span-0-337"><a id="__codelineno-0-337" name="__codelineno-0-337"></a><span class="sd">            guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).</span>
-</span><span id="__span-0-338"><a id="__codelineno-0-338" name="__codelineno-0-338"></a><span class="sd">        kwargs: Additional keyword arguments to pass to the</span>
-</span><span id="__span-0-339"><a id="__codelineno-0-339" name="__codelineno-0-339"></a><span class="sd">            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `feature_names`</span>
-</span><span id="__span-0-340"><a id="__codelineno-0-340" name="__codelineno-0-340"></a><span class="sd">            or `target_names`.</span>
-</span><span id="__span-0-341"><a id="__codelineno-0-341" name="__codelineno-0-341"></a>
-</span><span id="__span-0-342"><a id="__codelineno-0-342" name="__codelineno-0-342"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-343"><a id="__codelineno-0-343" name="__codelineno-0-343"></a><span class="sd">        Object with the passed X and y arrays split across training and test sets.</span>
-</span><span id="__span-0-344"><a id="__codelineno-0-344" name="__codelineno-0-344"></a>
-</span><span id="__span-0-345"><a id="__codelineno-0-345" name="__codelineno-0-345"></a><span class="sd">    !!! tip &quot;New in version 0.4.0&quot;</span>
-</span><span id="__span-0-346"><a id="__codelineno-0-346" name="__codelineno-0-346"></a>
-</span><span id="__span-0-347"><a id="__codelineno-0-347" name="__codelineno-0-347"></a><span class="sd">    !!! tip &quot;Changed in version 0.6.0&quot;</span>
-</span><span id="__span-0-348"><a id="__codelineno-0-348" name="__codelineno-0-348"></a><span class="sd">        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.</span>
-</span><span id="__span-0-349"><a id="__codelineno-0-349" name="__codelineno-0-349"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-350"><a id="__codelineno-0-350" name="__codelineno-0-350"></a>    <span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span>
-</span><span id="__span-0-351"><a id="__codelineno-0-351" name="__codelineno-0-351"></a>        <span class="n">X</span><span class="p">,</span>
-</span><span id="__span-0-352"><a id="__codelineno-0-352" name="__codelineno-0-352"></a>        <span class="n">y</span><span class="p">,</span>
-</span><span id="__span-0-353"><a id="__codelineno-0-353" name="__codelineno-0-353"></a>        <span class="n">train_size</span><span class="o">=</span><span class="n">train_size</span><span class="p">,</span>
-</span><span id="__span-0-354"><a id="__codelineno-0-354" name="__codelineno-0-354"></a>        <span class="n">random_state</span><span class="o">=</span><span class="n">random_state</span><span class="p">,</span>
-</span><span id="__span-0-355"><a id="__codelineno-0-355" name="__codelineno-0-355"></a>        <span class="n">stratify</span><span class="o">=</span><span class="n">y</span> <span class="k">if</span> <span class="n">stratify_by_target</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-356"><a id="__codelineno-0-356" name="__codelineno-0-356"></a>    <span class="p">)</span>
-</span><span id="__span-0-357"><a id="__codelineno-0-357" name="__codelineno-0-357"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-357">357</a></span>
+<span class="normal"><a href="#__codelineno-0-358">358</a></span>
+<span class="normal"><a href="#__codelineno-0-359">359</a></span>
+<span class="normal"><a href="#__codelineno-0-360">360</a></span>
+<span class="normal"><a href="#__codelineno-0-361">361</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-313"><a id="__codelineno-0-313" name="__codelineno-0-313"></a><span class="nd">@classmethod</span>
+</span><span id="__span-0-314"><a id="__codelineno-0-314" name="__codelineno-0-314"></a><span class="k">def</span> <span class="nf">from_arrays</span><span class="p">(</span>
+</span><span id="__span-0-315"><a id="__codelineno-0-315" name="__codelineno-0-315"></a>    <span class="bp">cls</span><span class="p">,</span>
+</span><span id="__span-0-316"><a id="__codelineno-0-316" name="__codelineno-0-316"></a>    <span class="n">X</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span>
+</span><span id="__span-0-317"><a id="__codelineno-0-317" name="__codelineno-0-317"></a>    <span class="n">y</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span>
+</span><span id="__span-0-318"><a id="__codelineno-0-318" name="__codelineno-0-318"></a>    <span class="n">train_size</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.8</span><span class="p">,</span>
+</span><span id="__span-0-319"><a id="__codelineno-0-319" name="__codelineno-0-319"></a>    <span class="n">random_state</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-320"><a id="__codelineno-0-320" name="__codelineno-0-320"></a>    <span class="n">stratify_by_target</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+</span><span id="__span-0-321"><a id="__codelineno-0-321" name="__codelineno-0-321"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+</span><span id="__span-0-322"><a id="__codelineno-0-322" name="__codelineno-0-322"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Dataset&quot;</span><span class="p">:</span>
+</span><span id="__span-0-323"><a id="__codelineno-0-323" name="__codelineno-0-323"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Constructs a [Dataset][pydvl.utils.Dataset] object from X and y numpy arrays  as</span>
+</span><span id="__span-0-324"><a id="__codelineno-0-324" name="__codelineno-0-324"></a><span class="sd">    returned by the `make_*` functions in [sklearn generated datasets](https://scikit-learn.org/stable/datasets/sample_generators.html).</span>
+</span><span id="__span-0-325"><a id="__codelineno-0-325" name="__codelineno-0-325"></a>
+</span><span id="__span-0-326"><a id="__codelineno-0-326" name="__codelineno-0-326"></a><span class="sd">    ??? Example</span>
+</span><span id="__span-0-327"><a id="__codelineno-0-327" name="__codelineno-0-327"></a><span class="sd">        ```pycon</span>
+</span><span id="__span-0-328"><a id="__codelineno-0-328" name="__codelineno-0-328"></a><span class="sd">        &gt;&gt;&gt; from pydvl.utils import Dataset</span>
+</span><span id="__span-0-329"><a id="__codelineno-0-329" name="__codelineno-0-329"></a><span class="sd">        &gt;&gt;&gt; from sklearn.datasets import make_regression</span>
+</span><span id="__span-0-330"><a id="__codelineno-0-330" name="__codelineno-0-330"></a><span class="sd">        &gt;&gt;&gt; X, y = make_regression()</span>
+</span><span id="__span-0-331"><a id="__codelineno-0-331" name="__codelineno-0-331"></a><span class="sd">        &gt;&gt;&gt; dataset = Dataset.from_arrays(X, y)</span>
+</span><span id="__span-0-332"><a id="__codelineno-0-332" name="__codelineno-0-332"></a><span class="sd">        ```</span>
+</span><span id="__span-0-333"><a id="__codelineno-0-333" name="__codelineno-0-333"></a>
+</span><span id="__span-0-334"><a id="__codelineno-0-334" name="__codelineno-0-334"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-335"><a id="__codelineno-0-335" name="__codelineno-0-335"></a><span class="sd">        X: numpy array of shape (n_samples, n_features)</span>
+</span><span id="__span-0-336"><a id="__codelineno-0-336" name="__codelineno-0-336"></a><span class="sd">        y: numpy array of shape (n_samples,)</span>
+</span><span id="__span-0-337"><a id="__codelineno-0-337" name="__codelineno-0-337"></a><span class="sd">        train_size: size of the training dataset. Used in `train_test_split`</span>
+</span><span id="__span-0-338"><a id="__codelineno-0-338" name="__codelineno-0-338"></a><span class="sd">        random_state: seed for train / test split</span>
+</span><span id="__span-0-339"><a id="__codelineno-0-339" name="__codelineno-0-339"></a><span class="sd">        stratify_by_target: If `True`, data is split in a stratified fashion,</span>
+</span><span id="__span-0-340"><a id="__codelineno-0-340" name="__codelineno-0-340"></a><span class="sd">            using the y variable as labels. Read more in [sklearn&#39;s user</span>
+</span><span id="__span-0-341"><a id="__codelineno-0-341" name="__codelineno-0-341"></a><span class="sd">            guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).</span>
+</span><span id="__span-0-342"><a id="__codelineno-0-342" name="__codelineno-0-342"></a><span class="sd">        kwargs: Additional keyword arguments to pass to the</span>
+</span><span id="__span-0-343"><a id="__codelineno-0-343" name="__codelineno-0-343"></a><span class="sd">            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `feature_names`</span>
+</span><span id="__span-0-344"><a id="__codelineno-0-344" name="__codelineno-0-344"></a><span class="sd">            or `target_names`.</span>
+</span><span id="__span-0-345"><a id="__codelineno-0-345" name="__codelineno-0-345"></a>
+</span><span id="__span-0-346"><a id="__codelineno-0-346" name="__codelineno-0-346"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-347"><a id="__codelineno-0-347" name="__codelineno-0-347"></a><span class="sd">        Object with the passed X and y arrays split across training and test sets.</span>
+</span><span id="__span-0-348"><a id="__codelineno-0-348" name="__codelineno-0-348"></a>
+</span><span id="__span-0-349"><a id="__codelineno-0-349" name="__codelineno-0-349"></a><span class="sd">    !!! tip &quot;New in version 0.4.0&quot;</span>
+</span><span id="__span-0-350"><a id="__codelineno-0-350" name="__codelineno-0-350"></a>
+</span><span id="__span-0-351"><a id="__codelineno-0-351" name="__codelineno-0-351"></a><span class="sd">    !!! tip &quot;Changed in version 0.6.0&quot;</span>
+</span><span id="__span-0-352"><a id="__codelineno-0-352" name="__codelineno-0-352"></a><span class="sd">        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.</span>
+</span><span id="__span-0-353"><a id="__codelineno-0-353" name="__codelineno-0-353"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-354"><a id="__codelineno-0-354" name="__codelineno-0-354"></a>    <span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span>
+</span><span id="__span-0-355"><a id="__codelineno-0-355" name="__codelineno-0-355"></a>        <span class="n">X</span><span class="p">,</span>
+</span><span id="__span-0-356"><a id="__codelineno-0-356" name="__codelineno-0-356"></a>        <span class="n">y</span><span class="p">,</span>
+</span><span id="__span-0-357"><a id="__codelineno-0-357" name="__codelineno-0-357"></a>        <span class="n">train_size</span><span class="o">=</span><span class="n">train_size</span><span class="p">,</span>
+</span><span id="__span-0-358"><a id="__codelineno-0-358" name="__codelineno-0-358"></a>        <span class="n">random_state</span><span class="o">=</span><span class="n">random_state</span><span class="p">,</span>
+</span><span id="__span-0-359"><a id="__codelineno-0-359" name="__codelineno-0-359"></a>        <span class="n">stratify</span><span class="o">=</span><span class="n">y</span> <span class="k">if</span> <span class="n">stratify_by_target</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-360"><a id="__codelineno-0-360" name="__codelineno-0-360"></a>    <span class="p">)</span>
+</span><span id="__span-0-361"><a id="__codelineno-0-361" name="__codelineno-0-361"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -4173,7 +4181,7 @@ <h2 id="pydvl.utils.dataset.GroupedDataset" class="doc doc-heading">
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-external" title="numpy.ndarray" href="https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html#numpy.ndarray">ndarray</a></code>
+                    <code><a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a></code>
                 </span>
             </p>
           </td>
@@ -4187,7 +4195,7 @@ <h2 id="pydvl.utils.dataset.GroupedDataset" class="doc doc-heading">
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-external" title="numpy.ndarray" href="https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html#numpy.ndarray">ndarray</a></code>
+                    <code><a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a></code>
                 </span>
             </p>
           </td>
@@ -4201,7 +4209,7 @@ <h2 id="pydvl.utils.dataset.GroupedDataset" class="doc doc-heading">
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-external" title="numpy.ndarray" href="https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html#numpy.ndarray">ndarray</a></code>
+                    <code><a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a></code>
                 </span>
             </p>
           </td>
@@ -4215,7 +4223,7 @@ <h2 id="pydvl.utils.dataset.GroupedDataset" class="doc doc-heading">
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-external" title="numpy.ndarray" href="https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html#numpy.ndarray">ndarray</a></code>
+                    <code><a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a></code>
                 </span>
             </p>
           </td>
@@ -4335,11 +4343,7 @@ <h2 id="pydvl.utils.dataset.GroupedDataset" class="doc doc-heading">
 
               <details class="quote">
                 <summary>Source code in <code>src/pydvl/utils/dataset.py</code></summary>
-                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-361">361</a></span>
-<span class="normal"><a href="#__codelineno-0-362">362</a></span>
-<span class="normal"><a href="#__codelineno-0-363">363</a></span>
-<span class="normal"><a href="#__codelineno-0-364">364</a></span>
-<span class="normal"><a href="#__codelineno-0-365">365</a></span>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-365">365</a></span>
 <span class="normal"><a href="#__codelineno-0-366">366</a></span>
 <span class="normal"><a href="#__codelineno-0-367">367</a></span>
 <span class="normal"><a href="#__codelineno-0-368">368</a></span>
@@ -4401,73 +4405,81 @@ <h2 id="pydvl.utils.dataset.GroupedDataset" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-424">424</a></span>
 <span class="normal"><a href="#__codelineno-0-425">425</a></span>
 <span class="normal"><a href="#__codelineno-0-426">426</a></span>
-<span class="normal"><a href="#__codelineno-0-427">427</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-361"><a id="__codelineno-0-361" name="__codelineno-0-361"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-</span><span id="__span-0-362"><a id="__codelineno-0-362" name="__codelineno-0-362"></a>    <span class="bp">self</span><span class="p">,</span>
-</span><span id="__span-0-363"><a id="__codelineno-0-363" name="__codelineno-0-363"></a>    <span class="n">x_train</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
-</span><span id="__span-0-364"><a id="__codelineno-0-364" name="__codelineno-0-364"></a>    <span class="n">y_train</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
-</span><span id="__span-0-365"><a id="__codelineno-0-365" name="__codelineno-0-365"></a>    <span class="n">x_test</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
-</span><span id="__span-0-366"><a id="__codelineno-0-366" name="__codelineno-0-366"></a>    <span class="n">y_test</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
-</span><span id="__span-0-367"><a id="__codelineno-0-367" name="__codelineno-0-367"></a>    <span class="n">data_groups</span><span class="p">:</span> <span class="n">Sequence</span><span class="p">,</span>
-</span><span id="__span-0-368"><a id="__codelineno-0-368" name="__codelineno-0-368"></a>    <span class="n">feature_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-369"><a id="__codelineno-0-369" name="__codelineno-0-369"></a>    <span class="n">target_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-370"><a id="__codelineno-0-370" name="__codelineno-0-370"></a>    <span class="n">group_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-371"><a id="__codelineno-0-371" name="__codelineno-0-371"></a>    <span class="n">description</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-372"><a id="__codelineno-0-372" name="__codelineno-0-372"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-</span><span id="__span-0-373"><a id="__codelineno-0-373" name="__codelineno-0-373"></a><span class="p">):</span>
-</span><span id="__span-0-374"><a id="__codelineno-0-374" name="__codelineno-0-374"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Class for grouping datasets.</span>
-</span><span id="__span-0-375"><a id="__codelineno-0-375" name="__codelineno-0-375"></a>
-</span><span id="__span-0-376"><a id="__codelineno-0-376" name="__codelineno-0-376"></a><span class="sd">    Used for calculating Shapley values of subsets of the data considered</span>
-</span><span id="__span-0-377"><a id="__codelineno-0-377" name="__codelineno-0-377"></a><span class="sd">    as logical units. For instance, one can group by value of a categorical</span>
-</span><span id="__span-0-378"><a id="__codelineno-0-378" name="__codelineno-0-378"></a><span class="sd">    feature, by bin into which a continuous feature falls, or by label.</span>
+<span class="normal"><a href="#__codelineno-0-427">427</a></span>
+<span class="normal"><a href="#__codelineno-0-428">428</a></span>
+<span class="normal"><a href="#__codelineno-0-429">429</a></span>
+<span class="normal"><a href="#__codelineno-0-430">430</a></span>
+<span class="normal"><a href="#__codelineno-0-431">431</a></span>
+<span class="normal"><a href="#__codelineno-0-432">432</a></span>
+<span class="normal"><a href="#__codelineno-0-433">433</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-365"><a id="__codelineno-0-365" name="__codelineno-0-365"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+</span><span id="__span-0-366"><a id="__codelineno-0-366" name="__codelineno-0-366"></a>    <span class="bp">self</span><span class="p">,</span>
+</span><span id="__span-0-367"><a id="__codelineno-0-367" name="__codelineno-0-367"></a>    <span class="n">x_train</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span>
+</span><span id="__span-0-368"><a id="__codelineno-0-368" name="__codelineno-0-368"></a>    <span class="n">y_train</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span>
+</span><span id="__span-0-369"><a id="__codelineno-0-369" name="__codelineno-0-369"></a>    <span class="n">x_test</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span>
+</span><span id="__span-0-370"><a id="__codelineno-0-370" name="__codelineno-0-370"></a>    <span class="n">y_test</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span>
+</span><span id="__span-0-371"><a id="__codelineno-0-371" name="__codelineno-0-371"></a>    <span class="n">data_groups</span><span class="p">:</span> <span class="n">Sequence</span><span class="p">,</span>
+</span><span id="__span-0-372"><a id="__codelineno-0-372" name="__codelineno-0-372"></a>    <span class="n">feature_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-373"><a id="__codelineno-0-373" name="__codelineno-0-373"></a>    <span class="n">target_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-374"><a id="__codelineno-0-374" name="__codelineno-0-374"></a>    <span class="n">group_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-375"><a id="__codelineno-0-375" name="__codelineno-0-375"></a>    <span class="n">description</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-376"><a id="__codelineno-0-376" name="__codelineno-0-376"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+</span><span id="__span-0-377"><a id="__codelineno-0-377" name="__codelineno-0-377"></a><span class="p">):</span>
+</span><span id="__span-0-378"><a id="__codelineno-0-378" name="__codelineno-0-378"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Class for grouping datasets.</span>
 </span><span id="__span-0-379"><a id="__codelineno-0-379" name="__codelineno-0-379"></a>
-</span><span id="__span-0-380"><a id="__codelineno-0-380" name="__codelineno-0-380"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-381"><a id="__codelineno-0-381" name="__codelineno-0-381"></a><span class="sd">        x_train: training data</span>
-</span><span id="__span-0-382"><a id="__codelineno-0-382" name="__codelineno-0-382"></a><span class="sd">        y_train: labels of training data</span>
-</span><span id="__span-0-383"><a id="__codelineno-0-383" name="__codelineno-0-383"></a><span class="sd">        x_test: test data</span>
-</span><span id="__span-0-384"><a id="__codelineno-0-384" name="__codelineno-0-384"></a><span class="sd">        y_test: labels of test data</span>
-</span><span id="__span-0-385"><a id="__codelineno-0-385" name="__codelineno-0-385"></a><span class="sd">        data_groups: Iterable of the same length as `x_train` containing</span>
-</span><span id="__span-0-386"><a id="__codelineno-0-386" name="__codelineno-0-386"></a><span class="sd">            a group label for each training data point. The label can be of any</span>
-</span><span id="__span-0-387"><a id="__codelineno-0-387" name="__codelineno-0-387"></a><span class="sd">            type, e.g. `str` or `int`. Data points with the same label will</span>
-</span><span id="__span-0-388"><a id="__codelineno-0-388" name="__codelineno-0-388"></a><span class="sd">            then be grouped by this object and considered as one for effects of</span>
-</span><span id="__span-0-389"><a id="__codelineno-0-389" name="__codelineno-0-389"></a><span class="sd">            valuation.</span>
-</span><span id="__span-0-390"><a id="__codelineno-0-390" name="__codelineno-0-390"></a><span class="sd">        feature_names: names of the covariates&#39; features.</span>
-</span><span id="__span-0-391"><a id="__codelineno-0-391" name="__codelineno-0-391"></a><span class="sd">        target_names: names of the labels or targets y</span>
-</span><span id="__span-0-392"><a id="__codelineno-0-392" name="__codelineno-0-392"></a><span class="sd">        group_names: names of the groups. If not provided, the labels</span>
-</span><span id="__span-0-393"><a id="__codelineno-0-393" name="__codelineno-0-393"></a><span class="sd">            from `data_groups` will be used.</span>
-</span><span id="__span-0-394"><a id="__codelineno-0-394" name="__codelineno-0-394"></a><span class="sd">        description: A textual description of the dataset</span>
-</span><span id="__span-0-395"><a id="__codelineno-0-395" name="__codelineno-0-395"></a><span class="sd">        kwargs: Additional keyword arguments to pass to the</span>
-</span><span id="__span-0-396"><a id="__codelineno-0-396" name="__codelineno-0-396"></a><span class="sd">            [Dataset][pydvl.utils.Dataset] constructor.</span>
-</span><span id="__span-0-397"><a id="__codelineno-0-397" name="__codelineno-0-397"></a>
-</span><span id="__span-0-398"><a id="__codelineno-0-398" name="__codelineno-0-398"></a><span class="sd">    !!! tip &quot;Changed in version 0.6.0&quot;</span>
-</span><span id="__span-0-399"><a id="__codelineno-0-399" name="__codelineno-0-399"></a><span class="sd">    Added `group_names` and forwarding of `kwargs`</span>
-</span><span id="__span-0-400"><a id="__codelineno-0-400" name="__codelineno-0-400"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-401"><a id="__codelineno-0-401" name="__codelineno-0-401"></a>    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-</span><span id="__span-0-402"><a id="__codelineno-0-402" name="__codelineno-0-402"></a>        <span class="n">x_train</span><span class="o">=</span><span class="n">x_train</span><span class="p">,</span>
-</span><span id="__span-0-403"><a id="__codelineno-0-403" name="__codelineno-0-403"></a>        <span class="n">y_train</span><span class="o">=</span><span class="n">y_train</span><span class="p">,</span>
-</span><span id="__span-0-404"><a id="__codelineno-0-404" name="__codelineno-0-404"></a>        <span class="n">x_test</span><span class="o">=</span><span class="n">x_test</span><span class="p">,</span>
-</span><span id="__span-0-405"><a id="__codelineno-0-405" name="__codelineno-0-405"></a>        <span class="n">y_test</span><span class="o">=</span><span class="n">y_test</span><span class="p">,</span>
-</span><span id="__span-0-406"><a id="__codelineno-0-406" name="__codelineno-0-406"></a>        <span class="n">feature_names</span><span class="o">=</span><span class="n">feature_names</span><span class="p">,</span>
-</span><span id="__span-0-407"><a id="__codelineno-0-407" name="__codelineno-0-407"></a>        <span class="n">target_names</span><span class="o">=</span><span class="n">target_names</span><span class="p">,</span>
-</span><span id="__span-0-408"><a id="__codelineno-0-408" name="__codelineno-0-408"></a>        <span class="n">description</span><span class="o">=</span><span class="n">description</span><span class="p">,</span>
-</span><span id="__span-0-409"><a id="__codelineno-0-409" name="__codelineno-0-409"></a>        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-</span><span id="__span-0-410"><a id="__codelineno-0-410" name="__codelineno-0-410"></a>    <span class="p">)</span>
-</span><span id="__span-0-411"><a id="__codelineno-0-411" name="__codelineno-0-411"></a>
-</span><span id="__span-0-412"><a id="__codelineno-0-412" name="__codelineno-0-412"></a>    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">data_groups</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">x_train</span><span class="p">):</span>
-</span><span id="__span-0-413"><a id="__codelineno-0-413" name="__codelineno-0-413"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-</span><span id="__span-0-414"><a id="__codelineno-0-414" name="__codelineno-0-414"></a>            <span class="sa">f</span><span class="s2">&quot;data_groups and x_train must have the same length.&quot;</span>
-</span><span id="__span-0-415"><a id="__codelineno-0-415" name="__codelineno-0-415"></a>            <span class="sa">f</span><span class="s2">&quot;Instead got </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">data_groups</span><span class="p">)</span><span class="si">=}</span><span class="s2"> and </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">x_train</span><span class="p">)</span><span class="si">=}</span><span class="s2">&quot;</span>
-</span><span id="__span-0-416"><a id="__codelineno-0-416" name="__codelineno-0-416"></a>        <span class="p">)</span>
-</span><span id="__span-0-417"><a id="__codelineno-0-417" name="__codelineno-0-417"></a>
-</span><span id="__span-0-418"><a id="__codelineno-0-418" name="__codelineno-0-418"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">groups</span><span class="p">:</span> <span class="n">OrderedDict</span><span class="p">[</span><span class="n">Any</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span>
-</span><span id="__span-0-419"><a id="__codelineno-0-419" name="__codelineno-0-419"></a>        <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="p">[]</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">set</span><span class="p">(</span><span class="n">data_groups</span><span class="p">)}</span>
-</span><span id="__span-0-420"><a id="__codelineno-0-420" name="__codelineno-0-420"></a>    <span class="p">)</span>
-</span><span id="__span-0-421"><a id="__codelineno-0-421" name="__codelineno-0-421"></a>    <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">group</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">data_groups</span><span class="p">):</span>
-</span><span id="__span-0-422"><a id="__codelineno-0-422" name="__codelineno-0-422"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">groups</span><span class="p">[</span><span class="n">group</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">idx</span><span class="p">)</span>
-</span><span id="__span-0-423"><a id="__codelineno-0-423" name="__codelineno-0-423"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">group_items</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">groups</span><span class="o">.</span><span class="n">items</span><span class="p">())</span>
-</span><span id="__span-0-424"><a id="__codelineno-0-424" name="__codelineno-0-424"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">groups</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span>
-</span><span id="__span-0-425"><a id="__codelineno-0-425" name="__codelineno-0-425"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_data_names</span> <span class="o">=</span> <span class="p">(</span>
-</span><span id="__span-0-426"><a id="__codelineno-0-426" name="__codelineno-0-426"></a>        <span class="n">group_names</span> <span class="k">if</span> <span class="n">group_names</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">groups</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
-</span><span id="__span-0-427"><a id="__codelineno-0-427" name="__codelineno-0-427"></a>    <span class="p">)</span>
+</span><span id="__span-0-380"><a id="__codelineno-0-380" name="__codelineno-0-380"></a><span class="sd">    Used for calculating Shapley values of subsets of the data considered</span>
+</span><span id="__span-0-381"><a id="__codelineno-0-381" name="__codelineno-0-381"></a><span class="sd">    as logical units. For instance, one can group by value of a categorical</span>
+</span><span id="__span-0-382"><a id="__codelineno-0-382" name="__codelineno-0-382"></a><span class="sd">    feature, by bin into which a continuous feature falls, or by label.</span>
+</span><span id="__span-0-383"><a id="__codelineno-0-383" name="__codelineno-0-383"></a>
+</span><span id="__span-0-384"><a id="__codelineno-0-384" name="__codelineno-0-384"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-385"><a id="__codelineno-0-385" name="__codelineno-0-385"></a><span class="sd">        x_train: training data</span>
+</span><span id="__span-0-386"><a id="__codelineno-0-386" name="__codelineno-0-386"></a><span class="sd">        y_train: labels of training data</span>
+</span><span id="__span-0-387"><a id="__codelineno-0-387" name="__codelineno-0-387"></a><span class="sd">        x_test: test data</span>
+</span><span id="__span-0-388"><a id="__codelineno-0-388" name="__codelineno-0-388"></a><span class="sd">        y_test: labels of test data</span>
+</span><span id="__span-0-389"><a id="__codelineno-0-389" name="__codelineno-0-389"></a><span class="sd">        data_groups: Iterable of the same length as `x_train` containing</span>
+</span><span id="__span-0-390"><a id="__codelineno-0-390" name="__codelineno-0-390"></a><span class="sd">            a group label for each training data point. The label can be of any</span>
+</span><span id="__span-0-391"><a id="__codelineno-0-391" name="__codelineno-0-391"></a><span class="sd">            type, e.g. `str` or `int`. Data points with the same label will</span>
+</span><span id="__span-0-392"><a id="__codelineno-0-392" name="__codelineno-0-392"></a><span class="sd">            then be grouped by this object and considered as one for effects of</span>
+</span><span id="__span-0-393"><a id="__codelineno-0-393" name="__codelineno-0-393"></a><span class="sd">            valuation.</span>
+</span><span id="__span-0-394"><a id="__codelineno-0-394" name="__codelineno-0-394"></a><span class="sd">        feature_names: names of the covariates&#39; features.</span>
+</span><span id="__span-0-395"><a id="__codelineno-0-395" name="__codelineno-0-395"></a><span class="sd">        target_names: names of the labels or targets y</span>
+</span><span id="__span-0-396"><a id="__codelineno-0-396" name="__codelineno-0-396"></a><span class="sd">        group_names: names of the groups. If not provided, the labels</span>
+</span><span id="__span-0-397"><a id="__codelineno-0-397" name="__codelineno-0-397"></a><span class="sd">            from `data_groups` will be used.</span>
+</span><span id="__span-0-398"><a id="__codelineno-0-398" name="__codelineno-0-398"></a><span class="sd">        description: A textual description of the dataset</span>
+</span><span id="__span-0-399"><a id="__codelineno-0-399" name="__codelineno-0-399"></a><span class="sd">        kwargs: Additional keyword arguments to pass to the</span>
+</span><span id="__span-0-400"><a id="__codelineno-0-400" name="__codelineno-0-400"></a><span class="sd">            [Dataset][pydvl.utils.Dataset] constructor.</span>
+</span><span id="__span-0-401"><a id="__codelineno-0-401" name="__codelineno-0-401"></a>
+</span><span id="__span-0-402"><a id="__codelineno-0-402" name="__codelineno-0-402"></a><span class="sd">    !!! tip &quot;Changed in version 0.6.0&quot;</span>
+</span><span id="__span-0-403"><a id="__codelineno-0-403" name="__codelineno-0-403"></a><span class="sd">    Added `group_names` and forwarding of `kwargs`</span>
+</span><span id="__span-0-404"><a id="__codelineno-0-404" name="__codelineno-0-404"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-405"><a id="__codelineno-0-405" name="__codelineno-0-405"></a>    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+</span><span id="__span-0-406"><a id="__codelineno-0-406" name="__codelineno-0-406"></a>        <span class="n">x_train</span><span class="o">=</span><span class="n">x_train</span><span class="p">,</span>
+</span><span id="__span-0-407"><a id="__codelineno-0-407" name="__codelineno-0-407"></a>        <span class="n">y_train</span><span class="o">=</span><span class="n">y_train</span><span class="p">,</span>
+</span><span id="__span-0-408"><a id="__codelineno-0-408" name="__codelineno-0-408"></a>        <span class="n">x_test</span><span class="o">=</span><span class="n">x_test</span><span class="p">,</span>
+</span><span id="__span-0-409"><a id="__codelineno-0-409" name="__codelineno-0-409"></a>        <span class="n">y_test</span><span class="o">=</span><span class="n">y_test</span><span class="p">,</span>
+</span><span id="__span-0-410"><a id="__codelineno-0-410" name="__codelineno-0-410"></a>        <span class="n">feature_names</span><span class="o">=</span><span class="n">feature_names</span><span class="p">,</span>
+</span><span id="__span-0-411"><a id="__codelineno-0-411" name="__codelineno-0-411"></a>        <span class="n">target_names</span><span class="o">=</span><span class="n">target_names</span><span class="p">,</span>
+</span><span id="__span-0-412"><a id="__codelineno-0-412" name="__codelineno-0-412"></a>        <span class="n">description</span><span class="o">=</span><span class="n">description</span><span class="p">,</span>
+</span><span id="__span-0-413"><a id="__codelineno-0-413" name="__codelineno-0-413"></a>        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+</span><span id="__span-0-414"><a id="__codelineno-0-414" name="__codelineno-0-414"></a>    <span class="p">)</span>
+</span><span id="__span-0-415"><a id="__codelineno-0-415" name="__codelineno-0-415"></a>
+</span><span id="__span-0-416"><a id="__codelineno-0-416" name="__codelineno-0-416"></a>    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">data_groups</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">x_train</span><span class="p">):</span>
+</span><span id="__span-0-417"><a id="__codelineno-0-417" name="__codelineno-0-417"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+</span><span id="__span-0-418"><a id="__codelineno-0-418" name="__codelineno-0-418"></a>            <span class="sa">f</span><span class="s2">&quot;data_groups and x_train must have the same length.&quot;</span>
+</span><span id="__span-0-419"><a id="__codelineno-0-419" name="__codelineno-0-419"></a>            <span class="sa">f</span><span class="s2">&quot;Instead got </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">data_groups</span><span class="p">)</span><span class="si">=}</span><span class="s2"> and </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">x_train</span><span class="p">)</span><span class="si">=}</span><span class="s2">&quot;</span>
+</span><span id="__span-0-420"><a id="__codelineno-0-420" name="__codelineno-0-420"></a>        <span class="p">)</span>
+</span><span id="__span-0-421"><a id="__codelineno-0-421" name="__codelineno-0-421"></a>
+</span><span id="__span-0-422"><a id="__codelineno-0-422" name="__codelineno-0-422"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">groups</span><span class="p">:</span> <span class="n">OrderedDict</span><span class="p">[</span><span class="n">Any</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span>
+</span><span id="__span-0-423"><a id="__codelineno-0-423" name="__codelineno-0-423"></a>        <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="p">[]</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">set</span><span class="p">(</span><span class="n">data_groups</span><span class="p">)}</span>
+</span><span id="__span-0-424"><a id="__codelineno-0-424" name="__codelineno-0-424"></a>    <span class="p">)</span>
+</span><span id="__span-0-425"><a id="__codelineno-0-425" name="__codelineno-0-425"></a>    <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">group</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">data_groups</span><span class="p">):</span>
+</span><span id="__span-0-426"><a id="__codelineno-0-426" name="__codelineno-0-426"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">groups</span><span class="p">[</span><span class="n">group</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">idx</span><span class="p">)</span>
+</span><span id="__span-0-427"><a id="__codelineno-0-427" name="__codelineno-0-427"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">group_items</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">groups</span><span class="o">.</span><span class="n">items</span><span class="p">())</span>
+</span><span id="__span-0-428"><a id="__codelineno-0-428" name="__codelineno-0-428"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">groups</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span>
+</span><span id="__span-0-429"><a id="__codelineno-0-429" name="__codelineno-0-429"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_data_names</span> <span class="o">=</span> <span class="p">(</span>
+</span><span id="__span-0-430"><a id="__codelineno-0-430" name="__codelineno-0-430"></a>        <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">group_names</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">object</span><span class="p">)</span>
+</span><span id="__span-0-431"><a id="__codelineno-0-431" name="__codelineno-0-431"></a>        <span class="k">if</span> <span class="n">group_names</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+</span><span id="__span-0-432"><a id="__codelineno-0-432" name="__codelineno-0-432"></a>        <span class="k">else</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">groups</span><span class="o">.</span><span class="n">keys</span><span class="p">()),</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">object</span><span class="p">)</span>
+</span><span id="__span-0-433"><a id="__codelineno-0-433" name="__codelineno-0-433"></a>    <span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
               </details>
 
@@ -4602,13 +4614,7 @@ <h3 id="pydvl.utils.dataset.GroupedDataset.get_training_data" class="doc doc-hea
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/dataset.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-444">444</a></span>
-<span class="normal"><a href="#__codelineno-0-445">445</a></span>
-<span class="normal"><a href="#__codelineno-0-446">446</a></span>
-<span class="normal"><a href="#__codelineno-0-447">447</a></span>
-<span class="normal"><a href="#__codelineno-0-448">448</a></span>
-<span class="normal"><a href="#__codelineno-0-449">449</a></span>
-<span class="normal"><a href="#__codelineno-0-450">450</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-450">450</a></span>
 <span class="normal"><a href="#__codelineno-0-451">451</a></span>
 <span class="normal"><a href="#__codelineno-0-452">452</a></span>
 <span class="normal"><a href="#__codelineno-0-453">453</a></span>
@@ -4619,24 +4625,30 @@ <h3 id="pydvl.utils.dataset.GroupedDataset.get_training_data" class="doc doc-hea
 <span class="normal"><a href="#__codelineno-0-458">458</a></span>
 <span class="normal"><a href="#__codelineno-0-459">459</a></span>
 <span class="normal"><a href="#__codelineno-0-460">460</a></span>
-<span class="normal"><a href="#__codelineno-0-461">461</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-444"><a id="__codelineno-0-444" name="__codelineno-0-444"></a><span class="k">def</span> <span class="nf">get_training_data</span><span class="p">(</span>
-</span><span id="__span-0-445"><a id="__codelineno-0-445" name="__codelineno-0-445"></a>    <span class="bp">self</span><span class="p">,</span> <span class="n">indices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Iterable</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
-</span><span id="__span-0-446"><a id="__codelineno-0-446" name="__codelineno-0-446"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">NDArray</span><span class="p">,</span> <span class="n">NDArray</span><span class="p">]:</span>
-</span><span id="__span-0-447"><a id="__codelineno-0-447" name="__codelineno-0-447"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the data and labels of all samples in the given groups.</span>
-</span><span id="__span-0-448"><a id="__codelineno-0-448" name="__codelineno-0-448"></a>
-</span><span id="__span-0-449"><a id="__codelineno-0-449" name="__codelineno-0-449"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-450"><a id="__codelineno-0-450" name="__codelineno-0-450"></a><span class="sd">        indices: group indices whose elements to return. If `None`,</span>
-</span><span id="__span-0-451"><a id="__codelineno-0-451" name="__codelineno-0-451"></a><span class="sd">            all data from all groups are returned.</span>
-</span><span id="__span-0-452"><a id="__codelineno-0-452" name="__codelineno-0-452"></a>
-</span><span id="__span-0-453"><a id="__codelineno-0-453" name="__codelineno-0-453"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-454"><a id="__codelineno-0-454" name="__codelineno-0-454"></a><span class="sd">        Tuple of training data x and labels y.</span>
-</span><span id="__span-0-455"><a id="__codelineno-0-455" name="__codelineno-0-455"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-456"><a id="__codelineno-0-456" name="__codelineno-0-456"></a>    <span class="k">if</span> <span class="n">indices</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-</span><span id="__span-0-457"><a id="__codelineno-0-457" name="__codelineno-0-457"></a>        <span class="n">indices</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">indices</span>
-</span><span id="__span-0-458"><a id="__codelineno-0-458" name="__codelineno-0-458"></a>    <span class="n">data_indices</span> <span class="o">=</span> <span class="p">[</span>
-</span><span id="__span-0-459"><a id="__codelineno-0-459" name="__codelineno-0-459"></a>        <span class="n">idx</span> <span class="k">for</span> <span class="n">group_id</span> <span class="ow">in</span> <span class="n">indices</span> <span class="k">for</span> <span class="n">idx</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">group_items</span><span class="p">[</span><span class="n">group_id</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
-</span><span id="__span-0-460"><a id="__codelineno-0-460" name="__codelineno-0-460"></a>    <span class="p">]</span>
-</span><span id="__span-0-461"><a id="__codelineno-0-461" name="__codelineno-0-461"></a>    <span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">get_training_data</span><span class="p">(</span><span class="n">data_indices</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-461">461</a></span>
+<span class="normal"><a href="#__codelineno-0-462">462</a></span>
+<span class="normal"><a href="#__codelineno-0-463">463</a></span>
+<span class="normal"><a href="#__codelineno-0-464">464</a></span>
+<span class="normal"><a href="#__codelineno-0-465">465</a></span>
+<span class="normal"><a href="#__codelineno-0-466">466</a></span>
+<span class="normal"><a href="#__codelineno-0-467">467</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-450"><a id="__codelineno-0-450" name="__codelineno-0-450"></a><span class="k">def</span> <span class="nf">get_training_data</span><span class="p">(</span>
+</span><span id="__span-0-451"><a id="__codelineno-0-451" name="__codelineno-0-451"></a>    <span class="bp">self</span><span class="p">,</span> <span class="n">indices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Iterable</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
+</span><span id="__span-0-452"><a id="__codelineno-0-452" name="__codelineno-0-452"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">NDArray</span><span class="p">,</span> <span class="n">NDArray</span><span class="p">]:</span>
+</span><span id="__span-0-453"><a id="__codelineno-0-453" name="__codelineno-0-453"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the data and labels of all samples in the given groups.</span>
+</span><span id="__span-0-454"><a id="__codelineno-0-454" name="__codelineno-0-454"></a>
+</span><span id="__span-0-455"><a id="__codelineno-0-455" name="__codelineno-0-455"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-456"><a id="__codelineno-0-456" name="__codelineno-0-456"></a><span class="sd">        indices: group indices whose elements to return. If `None`,</span>
+</span><span id="__span-0-457"><a id="__codelineno-0-457" name="__codelineno-0-457"></a><span class="sd">            all data from all groups are returned.</span>
+</span><span id="__span-0-458"><a id="__codelineno-0-458" name="__codelineno-0-458"></a>
+</span><span id="__span-0-459"><a id="__codelineno-0-459" name="__codelineno-0-459"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-460"><a id="__codelineno-0-460" name="__codelineno-0-460"></a><span class="sd">        Tuple of training data x and labels y.</span>
+</span><span id="__span-0-461"><a id="__codelineno-0-461" name="__codelineno-0-461"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-462"><a id="__codelineno-0-462" name="__codelineno-0-462"></a>    <span class="k">if</span> <span class="n">indices</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+</span><span id="__span-0-463"><a id="__codelineno-0-463" name="__codelineno-0-463"></a>        <span class="n">indices</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">indices</span>
+</span><span id="__span-0-464"><a id="__codelineno-0-464" name="__codelineno-0-464"></a>    <span class="n">data_indices</span> <span class="o">=</span> <span class="p">[</span>
+</span><span id="__span-0-465"><a id="__codelineno-0-465" name="__codelineno-0-465"></a>        <span class="n">idx</span> <span class="k">for</span> <span class="n">group_id</span> <span class="ow">in</span> <span class="n">indices</span> <span class="k">for</span> <span class="n">idx</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">group_items</span><span class="p">[</span><span class="n">group_id</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
+</span><span id="__span-0-466"><a id="__codelineno-0-466" name="__codelineno-0-466"></a>    <span class="p">]</span>
+</span><span id="__span-0-467"><a id="__codelineno-0-467" name="__codelineno-0-467"></a>    <span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">get_training_data</span><span class="p">(</span><span class="n">data_indices</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -4827,13 +4839,7 @@ <h3 id="pydvl.utils.dataset.GroupedDataset.from_sklearn" class="doc doc-heading"
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/dataset.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-463">463</a></span>
-<span class="normal"><a href="#__codelineno-0-464">464</a></span>
-<span class="normal"><a href="#__codelineno-0-465">465</a></span>
-<span class="normal"><a href="#__codelineno-0-466">466</a></span>
-<span class="normal"><a href="#__codelineno-0-467">467</a></span>
-<span class="normal"><a href="#__codelineno-0-468">468</a></span>
-<span class="normal"><a href="#__codelineno-0-469">469</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-469">469</a></span>
 <span class="normal"><a href="#__codelineno-0-470">470</a></span>
 <span class="normal"><a href="#__codelineno-0-471">471</a></span>
 <span class="normal"><a href="#__codelineno-0-472">472</a></span>
@@ -4890,70 +4896,76 @@ <h3 id="pydvl.utils.dataset.GroupedDataset.from_sklearn" class="doc doc-heading"
 <span class="normal"><a href="#__codelineno-0-523">523</a></span>
 <span class="normal"><a href="#__codelineno-0-524">524</a></span>
 <span class="normal"><a href="#__codelineno-0-525">525</a></span>
-<span class="normal"><a href="#__codelineno-0-526">526</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-463"><a id="__codelineno-0-463" name="__codelineno-0-463"></a><span class="nd">@classmethod</span>
-</span><span id="__span-0-464"><a id="__codelineno-0-464" name="__codelineno-0-464"></a><span class="k">def</span> <span class="nf">from_sklearn</span><span class="p">(</span>
-</span><span id="__span-0-465"><a id="__codelineno-0-465" name="__codelineno-0-465"></a>    <span class="bp">cls</span><span class="p">,</span>
-</span><span id="__span-0-466"><a id="__codelineno-0-466" name="__codelineno-0-466"></a>    <span class="n">data</span><span class="p">:</span> <span class="n">Bunch</span><span class="p">,</span>
-</span><span id="__span-0-467"><a id="__codelineno-0-467" name="__codelineno-0-467"></a>    <span class="n">train_size</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.8</span><span class="p">,</span>
-</span><span id="__span-0-468"><a id="__codelineno-0-468" name="__codelineno-0-468"></a>    <span class="n">random_state</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-469"><a id="__codelineno-0-469" name="__codelineno-0-469"></a>    <span class="n">stratify_by_target</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-</span><span id="__span-0-470"><a id="__codelineno-0-470" name="__codelineno-0-470"></a>    <span class="n">data_groups</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-471"><a id="__codelineno-0-471" name="__codelineno-0-471"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-</span><span id="__span-0-472"><a id="__codelineno-0-472" name="__codelineno-0-472"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;GroupedDataset&quot;</span><span class="p">:</span>
-</span><span id="__span-0-473"><a id="__codelineno-0-473" name="__codelineno-0-473"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Constructs a [GroupedDataset][pydvl.utils.GroupedDataset] object from a</span>
-</span><span id="__span-0-474"><a id="__codelineno-0-474" name="__codelineno-0-474"></a><span class="sd">    [sklearn.utils.Bunch][sklearn.utils.Bunch] as returned by the `load_*` functions in</span>
-</span><span id="__span-0-475"><a id="__codelineno-0-475" name="__codelineno-0-475"></a><span class="sd">    [scikit-learn toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html) and groups</span>
-</span><span id="__span-0-476"><a id="__codelineno-0-476" name="__codelineno-0-476"></a><span class="sd">    it.</span>
-</span><span id="__span-0-477"><a id="__codelineno-0-477" name="__codelineno-0-477"></a>
-</span><span id="__span-0-478"><a id="__codelineno-0-478" name="__codelineno-0-478"></a><span class="sd">    ??? Example</span>
-</span><span id="__span-0-479"><a id="__codelineno-0-479" name="__codelineno-0-479"></a><span class="sd">        ```pycon</span>
-</span><span id="__span-0-480"><a id="__codelineno-0-480" name="__codelineno-0-480"></a><span class="sd">        &gt;&gt;&gt; from sklearn.datasets import load_iris</span>
-</span><span id="__span-0-481"><a id="__codelineno-0-481" name="__codelineno-0-481"></a><span class="sd">        &gt;&gt;&gt; from pydvl.utils import GroupedDataset</span>
-</span><span id="__span-0-482"><a id="__codelineno-0-482" name="__codelineno-0-482"></a><span class="sd">        &gt;&gt;&gt; iris = load_iris()</span>
-</span><span id="__span-0-483"><a id="__codelineno-0-483" name="__codelineno-0-483"></a><span class="sd">        &gt;&gt;&gt; data_groups = iris.data[:, 0] // 0.5</span>
-</span><span id="__span-0-484"><a id="__codelineno-0-484" name="__codelineno-0-484"></a><span class="sd">        &gt;&gt;&gt; dataset = GroupedDataset.from_sklearn(iris, data_groups=data_groups)</span>
-</span><span id="__span-0-485"><a id="__codelineno-0-485" name="__codelineno-0-485"></a><span class="sd">        ```</span>
-</span><span id="__span-0-486"><a id="__codelineno-0-486" name="__codelineno-0-486"></a>
-</span><span id="__span-0-487"><a id="__codelineno-0-487" name="__codelineno-0-487"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-488"><a id="__codelineno-0-488" name="__codelineno-0-488"></a><span class="sd">        data: scikit-learn Bunch object. The following attributes are supported:</span>
-</span><span id="__span-0-489"><a id="__codelineno-0-489" name="__codelineno-0-489"></a>
-</span><span id="__span-0-490"><a id="__codelineno-0-490" name="__codelineno-0-490"></a><span class="sd">            - `data`: covariates.</span>
-</span><span id="__span-0-491"><a id="__codelineno-0-491" name="__codelineno-0-491"></a><span class="sd">            - `target`: target variables (labels).</span>
-</span><span id="__span-0-492"><a id="__codelineno-0-492" name="__codelineno-0-492"></a><span class="sd">            - `feature_names` (**optional**): the feature names.</span>
-</span><span id="__span-0-493"><a id="__codelineno-0-493" name="__codelineno-0-493"></a><span class="sd">            - `target_names` (**optional**): the target names.</span>
-</span><span id="__span-0-494"><a id="__codelineno-0-494" name="__codelineno-0-494"></a><span class="sd">            - `DESCR` (**optional**): a description.</span>
-</span><span id="__span-0-495"><a id="__codelineno-0-495" name="__codelineno-0-495"></a><span class="sd">        train_size: size of the training dataset. Used in `train_test_split`.</span>
-</span><span id="__span-0-496"><a id="__codelineno-0-496" name="__codelineno-0-496"></a><span class="sd">        random_state: seed for train / test split.</span>
-</span><span id="__span-0-497"><a id="__codelineno-0-497" name="__codelineno-0-497"></a><span class="sd">        stratify_by_target: If `True`, data is split in a stratified</span>
-</span><span id="__span-0-498"><a id="__codelineno-0-498" name="__codelineno-0-498"></a><span class="sd">            fashion, using the target variable as labels. Read more in</span>
-</span><span id="__span-0-499"><a id="__codelineno-0-499" name="__codelineno-0-499"></a><span class="sd">            [sklearn&#39;s user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).</span>
-</span><span id="__span-0-500"><a id="__codelineno-0-500" name="__codelineno-0-500"></a><span class="sd">        data_groups: an array holding the group index or name for each</span>
-</span><span id="__span-0-501"><a id="__codelineno-0-501" name="__codelineno-0-501"></a><span class="sd">            data point. The length of this array must be equal to the number of</span>
-</span><span id="__span-0-502"><a id="__codelineno-0-502" name="__codelineno-0-502"></a><span class="sd">            data points in the dataset.</span>
-</span><span id="__span-0-503"><a id="__codelineno-0-503" name="__codelineno-0-503"></a><span class="sd">        kwargs: Additional keyword arguments to pass to the</span>
-</span><span id="__span-0-504"><a id="__codelineno-0-504" name="__codelineno-0-504"></a><span class="sd">            [Dataset][pydvl.utils.Dataset] constructor.</span>
-</span><span id="__span-0-505"><a id="__codelineno-0-505" name="__codelineno-0-505"></a>
-</span><span id="__span-0-506"><a id="__codelineno-0-506" name="__codelineno-0-506"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-507"><a id="__codelineno-0-507" name="__codelineno-0-507"></a><span class="sd">        Dataset with the selected sklearn data</span>
-</span><span id="__span-0-508"><a id="__codelineno-0-508" name="__codelineno-0-508"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-509"><a id="__codelineno-0-509" name="__codelineno-0-509"></a>    <span class="k">if</span> <span class="n">data_groups</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-</span><span id="__span-0-510"><a id="__codelineno-0-510" name="__codelineno-0-510"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-</span><span id="__span-0-511"><a id="__codelineno-0-511" name="__codelineno-0-511"></a>            <span class="s2">&quot;data_groups must be provided when constructing a GroupedDataset&quot;</span>
-</span><span id="__span-0-512"><a id="__codelineno-0-512" name="__codelineno-0-512"></a>        <span class="p">)</span>
-</span><span id="__span-0-513"><a id="__codelineno-0-513" name="__codelineno-0-513"></a>
-</span><span id="__span-0-514"><a id="__codelineno-0-514" name="__codelineno-0-514"></a>    <span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="n">data_groups_train</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span>
-</span><span id="__span-0-515"><a id="__codelineno-0-515" name="__codelineno-0-515"></a>        <span class="n">data</span><span class="o">.</span><span class="n">data</span><span class="p">,</span>
-</span><span id="__span-0-516"><a id="__codelineno-0-516" name="__codelineno-0-516"></a>        <span class="n">data</span><span class="o">.</span><span class="n">target</span><span class="p">,</span>
-</span><span id="__span-0-517"><a id="__codelineno-0-517" name="__codelineno-0-517"></a>        <span class="n">data_groups</span><span class="p">,</span>
-</span><span id="__span-0-518"><a id="__codelineno-0-518" name="__codelineno-0-518"></a>        <span class="n">train_size</span><span class="o">=</span><span class="n">train_size</span><span class="p">,</span>
-</span><span id="__span-0-519"><a id="__codelineno-0-519" name="__codelineno-0-519"></a>        <span class="n">random_state</span><span class="o">=</span><span class="n">random_state</span><span class="p">,</span>
-</span><span id="__span-0-520"><a id="__codelineno-0-520" name="__codelineno-0-520"></a>        <span class="n">stratify</span><span class="o">=</span><span class="n">data</span><span class="o">.</span><span class="n">target</span> <span class="k">if</span> <span class="n">stratify_by_target</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-521"><a id="__codelineno-0-521" name="__codelineno-0-521"></a>    <span class="p">)</span>
-</span><span id="__span-0-522"><a id="__codelineno-0-522" name="__codelineno-0-522"></a>
-</span><span id="__span-0-523"><a id="__codelineno-0-523" name="__codelineno-0-523"></a>    <span class="n">dataset</span> <span class="o">=</span> <span class="n">Dataset</span><span class="p">(</span>
-</span><span id="__span-0-524"><a id="__codelineno-0-524" name="__codelineno-0-524"></a>        <span class="n">x_train</span><span class="o">=</span><span class="n">x_train</span><span class="p">,</span> <span class="n">y_train</span><span class="o">=</span><span class="n">y_train</span><span class="p">,</span> <span class="n">x_test</span><span class="o">=</span><span class="n">x_test</span><span class="p">,</span> <span class="n">y_test</span><span class="o">=</span><span class="n">y_test</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span>
-</span><span id="__span-0-525"><a id="__codelineno-0-525" name="__codelineno-0-525"></a>    <span class="p">)</span>
-</span><span id="__span-0-526"><a id="__codelineno-0-526" name="__codelineno-0-526"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">from_dataset</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">data_groups_train</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
+<span class="normal"><a href="#__codelineno-0-526">526</a></span>
+<span class="normal"><a href="#__codelineno-0-527">527</a></span>
+<span class="normal"><a href="#__codelineno-0-528">528</a></span>
+<span class="normal"><a href="#__codelineno-0-529">529</a></span>
+<span class="normal"><a href="#__codelineno-0-530">530</a></span>
+<span class="normal"><a href="#__codelineno-0-531">531</a></span>
+<span class="normal"><a href="#__codelineno-0-532">532</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-469"><a id="__codelineno-0-469" name="__codelineno-0-469"></a><span class="nd">@classmethod</span>
+</span><span id="__span-0-470"><a id="__codelineno-0-470" name="__codelineno-0-470"></a><span class="k">def</span> <span class="nf">from_sklearn</span><span class="p">(</span>
+</span><span id="__span-0-471"><a id="__codelineno-0-471" name="__codelineno-0-471"></a>    <span class="bp">cls</span><span class="p">,</span>
+</span><span id="__span-0-472"><a id="__codelineno-0-472" name="__codelineno-0-472"></a>    <span class="n">data</span><span class="p">:</span> <span class="n">Bunch</span><span class="p">,</span>
+</span><span id="__span-0-473"><a id="__codelineno-0-473" name="__codelineno-0-473"></a>    <span class="n">train_size</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.8</span><span class="p">,</span>
+</span><span id="__span-0-474"><a id="__codelineno-0-474" name="__codelineno-0-474"></a>    <span class="n">random_state</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-475"><a id="__codelineno-0-475" name="__codelineno-0-475"></a>    <span class="n">stratify_by_target</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+</span><span id="__span-0-476"><a id="__codelineno-0-476" name="__codelineno-0-476"></a>    <span class="n">data_groups</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-477"><a id="__codelineno-0-477" name="__codelineno-0-477"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+</span><span id="__span-0-478"><a id="__codelineno-0-478" name="__codelineno-0-478"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;GroupedDataset&quot;</span><span class="p">:</span>
+</span><span id="__span-0-479"><a id="__codelineno-0-479" name="__codelineno-0-479"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Constructs a [GroupedDataset][pydvl.utils.GroupedDataset] object from a</span>
+</span><span id="__span-0-480"><a id="__codelineno-0-480" name="__codelineno-0-480"></a><span class="sd">    [sklearn.utils.Bunch][sklearn.utils.Bunch] as returned by the `load_*` functions in</span>
+</span><span id="__span-0-481"><a id="__codelineno-0-481" name="__codelineno-0-481"></a><span class="sd">    [scikit-learn toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html) and groups</span>
+</span><span id="__span-0-482"><a id="__codelineno-0-482" name="__codelineno-0-482"></a><span class="sd">    it.</span>
+</span><span id="__span-0-483"><a id="__codelineno-0-483" name="__codelineno-0-483"></a>
+</span><span id="__span-0-484"><a id="__codelineno-0-484" name="__codelineno-0-484"></a><span class="sd">    ??? Example</span>
+</span><span id="__span-0-485"><a id="__codelineno-0-485" name="__codelineno-0-485"></a><span class="sd">        ```pycon</span>
+</span><span id="__span-0-486"><a id="__codelineno-0-486" name="__codelineno-0-486"></a><span class="sd">        &gt;&gt;&gt; from sklearn.datasets import load_iris</span>
+</span><span id="__span-0-487"><a id="__codelineno-0-487" name="__codelineno-0-487"></a><span class="sd">        &gt;&gt;&gt; from pydvl.utils import GroupedDataset</span>
+</span><span id="__span-0-488"><a id="__codelineno-0-488" name="__codelineno-0-488"></a><span class="sd">        &gt;&gt;&gt; iris = load_iris()</span>
+</span><span id="__span-0-489"><a id="__codelineno-0-489" name="__codelineno-0-489"></a><span class="sd">        &gt;&gt;&gt; data_groups = iris.data[:, 0] // 0.5</span>
+</span><span id="__span-0-490"><a id="__codelineno-0-490" name="__codelineno-0-490"></a><span class="sd">        &gt;&gt;&gt; dataset = GroupedDataset.from_sklearn(iris, data_groups=data_groups)</span>
+</span><span id="__span-0-491"><a id="__codelineno-0-491" name="__codelineno-0-491"></a><span class="sd">        ```</span>
+</span><span id="__span-0-492"><a id="__codelineno-0-492" name="__codelineno-0-492"></a>
+</span><span id="__span-0-493"><a id="__codelineno-0-493" name="__codelineno-0-493"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-494"><a id="__codelineno-0-494" name="__codelineno-0-494"></a><span class="sd">        data: scikit-learn Bunch object. The following attributes are supported:</span>
+</span><span id="__span-0-495"><a id="__codelineno-0-495" name="__codelineno-0-495"></a>
+</span><span id="__span-0-496"><a id="__codelineno-0-496" name="__codelineno-0-496"></a><span class="sd">            - `data`: covariates.</span>
+</span><span id="__span-0-497"><a id="__codelineno-0-497" name="__codelineno-0-497"></a><span class="sd">            - `target`: target variables (labels).</span>
+</span><span id="__span-0-498"><a id="__codelineno-0-498" name="__codelineno-0-498"></a><span class="sd">            - `feature_names` (**optional**): the feature names.</span>
+</span><span id="__span-0-499"><a id="__codelineno-0-499" name="__codelineno-0-499"></a><span class="sd">            - `target_names` (**optional**): the target names.</span>
+</span><span id="__span-0-500"><a id="__codelineno-0-500" name="__codelineno-0-500"></a><span class="sd">            - `DESCR` (**optional**): a description.</span>
+</span><span id="__span-0-501"><a id="__codelineno-0-501" name="__codelineno-0-501"></a><span class="sd">        train_size: size of the training dataset. Used in `train_test_split`.</span>
+</span><span id="__span-0-502"><a id="__codelineno-0-502" name="__codelineno-0-502"></a><span class="sd">        random_state: seed for train / test split.</span>
+</span><span id="__span-0-503"><a id="__codelineno-0-503" name="__codelineno-0-503"></a><span class="sd">        stratify_by_target: If `True`, data is split in a stratified</span>
+</span><span id="__span-0-504"><a id="__codelineno-0-504" name="__codelineno-0-504"></a><span class="sd">            fashion, using the target variable as labels. Read more in</span>
+</span><span id="__span-0-505"><a id="__codelineno-0-505" name="__codelineno-0-505"></a><span class="sd">            [sklearn&#39;s user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).</span>
+</span><span id="__span-0-506"><a id="__codelineno-0-506" name="__codelineno-0-506"></a><span class="sd">        data_groups: an array holding the group index or name for each</span>
+</span><span id="__span-0-507"><a id="__codelineno-0-507" name="__codelineno-0-507"></a><span class="sd">            data point. The length of this array must be equal to the number of</span>
+</span><span id="__span-0-508"><a id="__codelineno-0-508" name="__codelineno-0-508"></a><span class="sd">            data points in the dataset.</span>
+</span><span id="__span-0-509"><a id="__codelineno-0-509" name="__codelineno-0-509"></a><span class="sd">        kwargs: Additional keyword arguments to pass to the</span>
+</span><span id="__span-0-510"><a id="__codelineno-0-510" name="__codelineno-0-510"></a><span class="sd">            [Dataset][pydvl.utils.Dataset] constructor.</span>
+</span><span id="__span-0-511"><a id="__codelineno-0-511" name="__codelineno-0-511"></a>
+</span><span id="__span-0-512"><a id="__codelineno-0-512" name="__codelineno-0-512"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-513"><a id="__codelineno-0-513" name="__codelineno-0-513"></a><span class="sd">        Dataset with the selected sklearn data</span>
+</span><span id="__span-0-514"><a id="__codelineno-0-514" name="__codelineno-0-514"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-515"><a id="__codelineno-0-515" name="__codelineno-0-515"></a>    <span class="k">if</span> <span class="n">data_groups</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+</span><span id="__span-0-516"><a id="__codelineno-0-516" name="__codelineno-0-516"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+</span><span id="__span-0-517"><a id="__codelineno-0-517" name="__codelineno-0-517"></a>            <span class="s2">&quot;data_groups must be provided when constructing a GroupedDataset&quot;</span>
+</span><span id="__span-0-518"><a id="__codelineno-0-518" name="__codelineno-0-518"></a>        <span class="p">)</span>
+</span><span id="__span-0-519"><a id="__codelineno-0-519" name="__codelineno-0-519"></a>
+</span><span id="__span-0-520"><a id="__codelineno-0-520" name="__codelineno-0-520"></a>    <span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="n">data_groups_train</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span>
+</span><span id="__span-0-521"><a id="__codelineno-0-521" name="__codelineno-0-521"></a>        <span class="n">data</span><span class="o">.</span><span class="n">data</span><span class="p">,</span>
+</span><span id="__span-0-522"><a id="__codelineno-0-522" name="__codelineno-0-522"></a>        <span class="n">data</span><span class="o">.</span><span class="n">target</span><span class="p">,</span>
+</span><span id="__span-0-523"><a id="__codelineno-0-523" name="__codelineno-0-523"></a>        <span class="n">data_groups</span><span class="p">,</span>
+</span><span id="__span-0-524"><a id="__codelineno-0-524" name="__codelineno-0-524"></a>        <span class="n">train_size</span><span class="o">=</span><span class="n">train_size</span><span class="p">,</span>
+</span><span id="__span-0-525"><a id="__codelineno-0-525" name="__codelineno-0-525"></a>        <span class="n">random_state</span><span class="o">=</span><span class="n">random_state</span><span class="p">,</span>
+</span><span id="__span-0-526"><a id="__codelineno-0-526" name="__codelineno-0-526"></a>        <span class="n">stratify</span><span class="o">=</span><span class="n">data</span><span class="o">.</span><span class="n">target</span> <span class="k">if</span> <span class="n">stratify_by_target</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-527"><a id="__codelineno-0-527" name="__codelineno-0-527"></a>    <span class="p">)</span>
+</span><span id="__span-0-528"><a id="__codelineno-0-528" name="__codelineno-0-528"></a>
+</span><span id="__span-0-529"><a id="__codelineno-0-529" name="__codelineno-0-529"></a>    <span class="n">dataset</span> <span class="o">=</span> <span class="n">Dataset</span><span class="p">(</span>
+</span><span id="__span-0-530"><a id="__codelineno-0-530" name="__codelineno-0-530"></a>        <span class="n">x_train</span><span class="o">=</span><span class="n">x_train</span><span class="p">,</span> <span class="n">y_train</span><span class="o">=</span><span class="n">y_train</span><span class="p">,</span> <span class="n">x_test</span><span class="o">=</span><span class="n">x_test</span><span class="p">,</span> <span class="n">y_test</span><span class="o">=</span><span class="n">y_test</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span>
+</span><span id="__span-0-531"><a id="__codelineno-0-531" name="__codelineno-0-531"></a>    <span class="p">)</span>
+</span><span id="__span-0-532"><a id="__codelineno-0-532" name="__codelineno-0-532"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">from_dataset</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">data_groups_train</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -5165,13 +5177,7 @@ <h3 id="pydvl.utils.dataset.GroupedDataset.from_arrays" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/dataset.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-528">528</a></span>
-<span class="normal"><a href="#__codelineno-0-529">529</a></span>
-<span class="normal"><a href="#__codelineno-0-530">530</a></span>
-<span class="normal"><a href="#__codelineno-0-531">531</a></span>
-<span class="normal"><a href="#__codelineno-0-532">532</a></span>
-<span class="normal"><a href="#__codelineno-0-533">533</a></span>
-<span class="normal"><a href="#__codelineno-0-534">534</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-534">534</a></span>
 <span class="normal"><a href="#__codelineno-0-535">535</a></span>
 <span class="normal"><a href="#__codelineno-0-536">536</a></span>
 <span class="normal"><a href="#__codelineno-0-537">537</a></span>
@@ -5234,76 +5240,82 @@ <h3 id="pydvl.utils.dataset.GroupedDataset.from_arrays" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-594">594</a></span>
 <span class="normal"><a href="#__codelineno-0-595">595</a></span>
 <span class="normal"><a href="#__codelineno-0-596">596</a></span>
-<span class="normal"><a href="#__codelineno-0-597">597</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-528"><a id="__codelineno-0-528" name="__codelineno-0-528"></a><span class="nd">@classmethod</span>
-</span><span id="__span-0-529"><a id="__codelineno-0-529" name="__codelineno-0-529"></a><span class="k">def</span> <span class="nf">from_arrays</span><span class="p">(</span>
-</span><span id="__span-0-530"><a id="__codelineno-0-530" name="__codelineno-0-530"></a>    <span class="bp">cls</span><span class="p">,</span>
-</span><span id="__span-0-531"><a id="__codelineno-0-531" name="__codelineno-0-531"></a>    <span class="n">X</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span>
-</span><span id="__span-0-532"><a id="__codelineno-0-532" name="__codelineno-0-532"></a>    <span class="n">y</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span>
-</span><span id="__span-0-533"><a id="__codelineno-0-533" name="__codelineno-0-533"></a>    <span class="n">train_size</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.8</span><span class="p">,</span>
-</span><span id="__span-0-534"><a id="__codelineno-0-534" name="__codelineno-0-534"></a>    <span class="n">random_state</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-535"><a id="__codelineno-0-535" name="__codelineno-0-535"></a>    <span class="n">stratify_by_target</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-</span><span id="__span-0-536"><a id="__codelineno-0-536" name="__codelineno-0-536"></a>    <span class="n">data_groups</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-537"><a id="__codelineno-0-537" name="__codelineno-0-537"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-</span><span id="__span-0-538"><a id="__codelineno-0-538" name="__codelineno-0-538"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Dataset&quot;</span><span class="p">:</span>
-</span><span id="__span-0-539"><a id="__codelineno-0-539" name="__codelineno-0-539"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Constructs a [GroupedDataset][pydvl.utils.GroupedDataset] object from X and y numpy arrays</span>
-</span><span id="__span-0-540"><a id="__codelineno-0-540" name="__codelineno-0-540"></a><span class="sd">    as returned by the `make_*` functions in</span>
-</span><span id="__span-0-541"><a id="__codelineno-0-541" name="__codelineno-0-541"></a><span class="sd">    [scikit-learn generated datasets](https://scikit-learn.org/stable/datasets/sample_generators.html).</span>
-</span><span id="__span-0-542"><a id="__codelineno-0-542" name="__codelineno-0-542"></a>
-</span><span id="__span-0-543"><a id="__codelineno-0-543" name="__codelineno-0-543"></a><span class="sd">    ??? Example</span>
-</span><span id="__span-0-544"><a id="__codelineno-0-544" name="__codelineno-0-544"></a><span class="sd">        ```pycon</span>
-</span><span id="__span-0-545"><a id="__codelineno-0-545" name="__codelineno-0-545"></a><span class="sd">        &gt;&gt;&gt; from sklearn.datasets import make_classification</span>
-</span><span id="__span-0-546"><a id="__codelineno-0-546" name="__codelineno-0-546"></a><span class="sd">        &gt;&gt;&gt; from pydvl.utils import GroupedDataset</span>
-</span><span id="__span-0-547"><a id="__codelineno-0-547" name="__codelineno-0-547"></a><span class="sd">        &gt;&gt;&gt; X, y = make_classification(</span>
-</span><span id="__span-0-548"><a id="__codelineno-0-548" name="__codelineno-0-548"></a><span class="sd">        ...     n_samples=100,</span>
-</span><span id="__span-0-549"><a id="__codelineno-0-549" name="__codelineno-0-549"></a><span class="sd">        ...     n_features=4,</span>
-</span><span id="__span-0-550"><a id="__codelineno-0-550" name="__codelineno-0-550"></a><span class="sd">        ...     n_informative=2,</span>
-</span><span id="__span-0-551"><a id="__codelineno-0-551" name="__codelineno-0-551"></a><span class="sd">        ...     n_redundant=0,</span>
-</span><span id="__span-0-552"><a id="__codelineno-0-552" name="__codelineno-0-552"></a><span class="sd">        ...     random_state=0,</span>
-</span><span id="__span-0-553"><a id="__codelineno-0-553" name="__codelineno-0-553"></a><span class="sd">        ...     shuffle=False</span>
-</span><span id="__span-0-554"><a id="__codelineno-0-554" name="__codelineno-0-554"></a><span class="sd">        ... )</span>
-</span><span id="__span-0-555"><a id="__codelineno-0-555" name="__codelineno-0-555"></a><span class="sd">        &gt;&gt;&gt; data_groups = X[:, 0] // 0.5</span>
-</span><span id="__span-0-556"><a id="__codelineno-0-556" name="__codelineno-0-556"></a><span class="sd">        &gt;&gt;&gt; dataset = GroupedDataset.from_arrays(X, y, data_groups=data_groups)</span>
-</span><span id="__span-0-557"><a id="__codelineno-0-557" name="__codelineno-0-557"></a><span class="sd">        ```</span>
-</span><span id="__span-0-558"><a id="__codelineno-0-558" name="__codelineno-0-558"></a>
-</span><span id="__span-0-559"><a id="__codelineno-0-559" name="__codelineno-0-559"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-560"><a id="__codelineno-0-560" name="__codelineno-0-560"></a><span class="sd">        X: array of shape (n_samples, n_features)</span>
-</span><span id="__span-0-561"><a id="__codelineno-0-561" name="__codelineno-0-561"></a><span class="sd">        y: array of shape (n_samples,)</span>
-</span><span id="__span-0-562"><a id="__codelineno-0-562" name="__codelineno-0-562"></a><span class="sd">        train_size: size of the training dataset. Used in `train_test_split`.</span>
-</span><span id="__span-0-563"><a id="__codelineno-0-563" name="__codelineno-0-563"></a><span class="sd">        random_state: seed for train / test split.</span>
-</span><span id="__span-0-564"><a id="__codelineno-0-564" name="__codelineno-0-564"></a><span class="sd">        stratify_by_target: If `True`, data is split in a stratified</span>
-</span><span id="__span-0-565"><a id="__codelineno-0-565" name="__codelineno-0-565"></a><span class="sd">            fashion, using the y variable as labels. Read more in</span>
-</span><span id="__span-0-566"><a id="__codelineno-0-566" name="__codelineno-0-566"></a><span class="sd">            [sklearn&#39;s user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).</span>
-</span><span id="__span-0-567"><a id="__codelineno-0-567" name="__codelineno-0-567"></a><span class="sd">        data_groups: an array holding the group index or name for each data</span>
-</span><span id="__span-0-568"><a id="__codelineno-0-568" name="__codelineno-0-568"></a><span class="sd">            point. The length of this array must be equal to the number of</span>
-</span><span id="__span-0-569"><a id="__codelineno-0-569" name="__codelineno-0-569"></a><span class="sd">            data points in the dataset.</span>
-</span><span id="__span-0-570"><a id="__codelineno-0-570" name="__codelineno-0-570"></a><span class="sd">        kwargs: Additional keyword arguments that will be passed to the</span>
-</span><span id="__span-0-571"><a id="__codelineno-0-571" name="__codelineno-0-571"></a><span class="sd">            [Dataset][pydvl.utils.Dataset] constructor.</span>
-</span><span id="__span-0-572"><a id="__codelineno-0-572" name="__codelineno-0-572"></a>
-</span><span id="__span-0-573"><a id="__codelineno-0-573" name="__codelineno-0-573"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-574"><a id="__codelineno-0-574" name="__codelineno-0-574"></a><span class="sd">        Dataset with the passed X and y arrays split across training and</span>
-</span><span id="__span-0-575"><a id="__codelineno-0-575" name="__codelineno-0-575"></a><span class="sd">            test sets.</span>
-</span><span id="__span-0-576"><a id="__codelineno-0-576" name="__codelineno-0-576"></a>
-</span><span id="__span-0-577"><a id="__codelineno-0-577" name="__codelineno-0-577"></a><span class="sd">    !!! tip &quot;New in version 0.4.0&quot;</span>
+<span class="normal"><a href="#__codelineno-0-597">597</a></span>
+<span class="normal"><a href="#__codelineno-0-598">598</a></span>
+<span class="normal"><a href="#__codelineno-0-599">599</a></span>
+<span class="normal"><a href="#__codelineno-0-600">600</a></span>
+<span class="normal"><a href="#__codelineno-0-601">601</a></span>
+<span class="normal"><a href="#__codelineno-0-602">602</a></span>
+<span class="normal"><a href="#__codelineno-0-603">603</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-534"><a id="__codelineno-0-534" name="__codelineno-0-534"></a><span class="nd">@classmethod</span>
+</span><span id="__span-0-535"><a id="__codelineno-0-535" name="__codelineno-0-535"></a><span class="k">def</span> <span class="nf">from_arrays</span><span class="p">(</span>
+</span><span id="__span-0-536"><a id="__codelineno-0-536" name="__codelineno-0-536"></a>    <span class="bp">cls</span><span class="p">,</span>
+</span><span id="__span-0-537"><a id="__codelineno-0-537" name="__codelineno-0-537"></a>    <span class="n">X</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span>
+</span><span id="__span-0-538"><a id="__codelineno-0-538" name="__codelineno-0-538"></a>    <span class="n">y</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span>
+</span><span id="__span-0-539"><a id="__codelineno-0-539" name="__codelineno-0-539"></a>    <span class="n">train_size</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.8</span><span class="p">,</span>
+</span><span id="__span-0-540"><a id="__codelineno-0-540" name="__codelineno-0-540"></a>    <span class="n">random_state</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-541"><a id="__codelineno-0-541" name="__codelineno-0-541"></a>    <span class="n">stratify_by_target</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+</span><span id="__span-0-542"><a id="__codelineno-0-542" name="__codelineno-0-542"></a>    <span class="n">data_groups</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-543"><a id="__codelineno-0-543" name="__codelineno-0-543"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+</span><span id="__span-0-544"><a id="__codelineno-0-544" name="__codelineno-0-544"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Dataset&quot;</span><span class="p">:</span>
+</span><span id="__span-0-545"><a id="__codelineno-0-545" name="__codelineno-0-545"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Constructs a [GroupedDataset][pydvl.utils.GroupedDataset] object from X and y numpy arrays</span>
+</span><span id="__span-0-546"><a id="__codelineno-0-546" name="__codelineno-0-546"></a><span class="sd">    as returned by the `make_*` functions in</span>
+</span><span id="__span-0-547"><a id="__codelineno-0-547" name="__codelineno-0-547"></a><span class="sd">    [scikit-learn generated datasets](https://scikit-learn.org/stable/datasets/sample_generators.html).</span>
+</span><span id="__span-0-548"><a id="__codelineno-0-548" name="__codelineno-0-548"></a>
+</span><span id="__span-0-549"><a id="__codelineno-0-549" name="__codelineno-0-549"></a><span class="sd">    ??? Example</span>
+</span><span id="__span-0-550"><a id="__codelineno-0-550" name="__codelineno-0-550"></a><span class="sd">        ```pycon</span>
+</span><span id="__span-0-551"><a id="__codelineno-0-551" name="__codelineno-0-551"></a><span class="sd">        &gt;&gt;&gt; from sklearn.datasets import make_classification</span>
+</span><span id="__span-0-552"><a id="__codelineno-0-552" name="__codelineno-0-552"></a><span class="sd">        &gt;&gt;&gt; from pydvl.utils import GroupedDataset</span>
+</span><span id="__span-0-553"><a id="__codelineno-0-553" name="__codelineno-0-553"></a><span class="sd">        &gt;&gt;&gt; X, y = make_classification(</span>
+</span><span id="__span-0-554"><a id="__codelineno-0-554" name="__codelineno-0-554"></a><span class="sd">        ...     n_samples=100,</span>
+</span><span id="__span-0-555"><a id="__codelineno-0-555" name="__codelineno-0-555"></a><span class="sd">        ...     n_features=4,</span>
+</span><span id="__span-0-556"><a id="__codelineno-0-556" name="__codelineno-0-556"></a><span class="sd">        ...     n_informative=2,</span>
+</span><span id="__span-0-557"><a id="__codelineno-0-557" name="__codelineno-0-557"></a><span class="sd">        ...     n_redundant=0,</span>
+</span><span id="__span-0-558"><a id="__codelineno-0-558" name="__codelineno-0-558"></a><span class="sd">        ...     random_state=0,</span>
+</span><span id="__span-0-559"><a id="__codelineno-0-559" name="__codelineno-0-559"></a><span class="sd">        ...     shuffle=False</span>
+</span><span id="__span-0-560"><a id="__codelineno-0-560" name="__codelineno-0-560"></a><span class="sd">        ... )</span>
+</span><span id="__span-0-561"><a id="__codelineno-0-561" name="__codelineno-0-561"></a><span class="sd">        &gt;&gt;&gt; data_groups = X[:, 0] // 0.5</span>
+</span><span id="__span-0-562"><a id="__codelineno-0-562" name="__codelineno-0-562"></a><span class="sd">        &gt;&gt;&gt; dataset = GroupedDataset.from_arrays(X, y, data_groups=data_groups)</span>
+</span><span id="__span-0-563"><a id="__codelineno-0-563" name="__codelineno-0-563"></a><span class="sd">        ```</span>
+</span><span id="__span-0-564"><a id="__codelineno-0-564" name="__codelineno-0-564"></a>
+</span><span id="__span-0-565"><a id="__codelineno-0-565" name="__codelineno-0-565"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-566"><a id="__codelineno-0-566" name="__codelineno-0-566"></a><span class="sd">        X: array of shape (n_samples, n_features)</span>
+</span><span id="__span-0-567"><a id="__codelineno-0-567" name="__codelineno-0-567"></a><span class="sd">        y: array of shape (n_samples,)</span>
+</span><span id="__span-0-568"><a id="__codelineno-0-568" name="__codelineno-0-568"></a><span class="sd">        train_size: size of the training dataset. Used in `train_test_split`.</span>
+</span><span id="__span-0-569"><a id="__codelineno-0-569" name="__codelineno-0-569"></a><span class="sd">        random_state: seed for train / test split.</span>
+</span><span id="__span-0-570"><a id="__codelineno-0-570" name="__codelineno-0-570"></a><span class="sd">        stratify_by_target: If `True`, data is split in a stratified</span>
+</span><span id="__span-0-571"><a id="__codelineno-0-571" name="__codelineno-0-571"></a><span class="sd">            fashion, using the y variable as labels. Read more in</span>
+</span><span id="__span-0-572"><a id="__codelineno-0-572" name="__codelineno-0-572"></a><span class="sd">            [sklearn&#39;s user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).</span>
+</span><span id="__span-0-573"><a id="__codelineno-0-573" name="__codelineno-0-573"></a><span class="sd">        data_groups: an array holding the group index or name for each data</span>
+</span><span id="__span-0-574"><a id="__codelineno-0-574" name="__codelineno-0-574"></a><span class="sd">            point. The length of this array must be equal to the number of</span>
+</span><span id="__span-0-575"><a id="__codelineno-0-575" name="__codelineno-0-575"></a><span class="sd">            data points in the dataset.</span>
+</span><span id="__span-0-576"><a id="__codelineno-0-576" name="__codelineno-0-576"></a><span class="sd">        kwargs: Additional keyword arguments that will be passed to the</span>
+</span><span id="__span-0-577"><a id="__codelineno-0-577" name="__codelineno-0-577"></a><span class="sd">            [Dataset][pydvl.utils.Dataset] constructor.</span>
 </span><span id="__span-0-578"><a id="__codelineno-0-578" name="__codelineno-0-578"></a>
-</span><span id="__span-0-579"><a id="__codelineno-0-579" name="__codelineno-0-579"></a><span class="sd">    !!! tip &quot;Changed in version 0.6.0&quot;</span>
-</span><span id="__span-0-580"><a id="__codelineno-0-580" name="__codelineno-0-580"></a><span class="sd">        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.</span>
-</span><span id="__span-0-581"><a id="__codelineno-0-581" name="__codelineno-0-581"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-582"><a id="__codelineno-0-582" name="__codelineno-0-582"></a>    <span class="k">if</span> <span class="n">data_groups</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-</span><span id="__span-0-583"><a id="__codelineno-0-583" name="__codelineno-0-583"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-</span><span id="__span-0-584"><a id="__codelineno-0-584" name="__codelineno-0-584"></a>            <span class="s2">&quot;data_groups must be provided when constructing a GroupedDataset&quot;</span>
-</span><span id="__span-0-585"><a id="__codelineno-0-585" name="__codelineno-0-585"></a>        <span class="p">)</span>
-</span><span id="__span-0-586"><a id="__codelineno-0-586" name="__codelineno-0-586"></a>    <span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="n">data_groups_train</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span>
-</span><span id="__span-0-587"><a id="__codelineno-0-587" name="__codelineno-0-587"></a>        <span class="n">X</span><span class="p">,</span>
-</span><span id="__span-0-588"><a id="__codelineno-0-588" name="__codelineno-0-588"></a>        <span class="n">y</span><span class="p">,</span>
-</span><span id="__span-0-589"><a id="__codelineno-0-589" name="__codelineno-0-589"></a>        <span class="n">data_groups</span><span class="p">,</span>
-</span><span id="__span-0-590"><a id="__codelineno-0-590" name="__codelineno-0-590"></a>        <span class="n">train_size</span><span class="o">=</span><span class="n">train_size</span><span class="p">,</span>
-</span><span id="__span-0-591"><a id="__codelineno-0-591" name="__codelineno-0-591"></a>        <span class="n">random_state</span><span class="o">=</span><span class="n">random_state</span><span class="p">,</span>
-</span><span id="__span-0-592"><a id="__codelineno-0-592" name="__codelineno-0-592"></a>        <span class="n">stratify</span><span class="o">=</span><span class="n">y</span> <span class="k">if</span> <span class="n">stratify_by_target</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-593"><a id="__codelineno-0-593" name="__codelineno-0-593"></a>    <span class="p">)</span>
-</span><span id="__span-0-594"><a id="__codelineno-0-594" name="__codelineno-0-594"></a>    <span class="n">dataset</span> <span class="o">=</span> <span class="n">Dataset</span><span class="p">(</span>
-</span><span id="__span-0-595"><a id="__codelineno-0-595" name="__codelineno-0-595"></a>        <span class="n">x_train</span><span class="o">=</span><span class="n">x_train</span><span class="p">,</span> <span class="n">y_train</span><span class="o">=</span><span class="n">y_train</span><span class="p">,</span> <span class="n">x_test</span><span class="o">=</span><span class="n">x_test</span><span class="p">,</span> <span class="n">y_test</span><span class="o">=</span><span class="n">y_test</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span>
-</span><span id="__span-0-596"><a id="__codelineno-0-596" name="__codelineno-0-596"></a>    <span class="p">)</span>
-</span><span id="__span-0-597"><a id="__codelineno-0-597" name="__codelineno-0-597"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">from_dataset</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">data_groups_train</span><span class="p">)</span>
+</span><span id="__span-0-579"><a id="__codelineno-0-579" name="__codelineno-0-579"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-580"><a id="__codelineno-0-580" name="__codelineno-0-580"></a><span class="sd">        Dataset with the passed X and y arrays split across training and</span>
+</span><span id="__span-0-581"><a id="__codelineno-0-581" name="__codelineno-0-581"></a><span class="sd">            test sets.</span>
+</span><span id="__span-0-582"><a id="__codelineno-0-582" name="__codelineno-0-582"></a>
+</span><span id="__span-0-583"><a id="__codelineno-0-583" name="__codelineno-0-583"></a><span class="sd">    !!! tip &quot;New in version 0.4.0&quot;</span>
+</span><span id="__span-0-584"><a id="__codelineno-0-584" name="__codelineno-0-584"></a>
+</span><span id="__span-0-585"><a id="__codelineno-0-585" name="__codelineno-0-585"></a><span class="sd">    !!! tip &quot;Changed in version 0.6.0&quot;</span>
+</span><span id="__span-0-586"><a id="__codelineno-0-586" name="__codelineno-0-586"></a><span class="sd">        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.</span>
+</span><span id="__span-0-587"><a id="__codelineno-0-587" name="__codelineno-0-587"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-588"><a id="__codelineno-0-588" name="__codelineno-0-588"></a>    <span class="k">if</span> <span class="n">data_groups</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+</span><span id="__span-0-589"><a id="__codelineno-0-589" name="__codelineno-0-589"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+</span><span id="__span-0-590"><a id="__codelineno-0-590" name="__codelineno-0-590"></a>            <span class="s2">&quot;data_groups must be provided when constructing a GroupedDataset&quot;</span>
+</span><span id="__span-0-591"><a id="__codelineno-0-591" name="__codelineno-0-591"></a>        <span class="p">)</span>
+</span><span id="__span-0-592"><a id="__codelineno-0-592" name="__codelineno-0-592"></a>    <span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="n">data_groups_train</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span>
+</span><span id="__span-0-593"><a id="__codelineno-0-593" name="__codelineno-0-593"></a>        <span class="n">X</span><span class="p">,</span>
+</span><span id="__span-0-594"><a id="__codelineno-0-594" name="__codelineno-0-594"></a>        <span class="n">y</span><span class="p">,</span>
+</span><span id="__span-0-595"><a id="__codelineno-0-595" name="__codelineno-0-595"></a>        <span class="n">data_groups</span><span class="p">,</span>
+</span><span id="__span-0-596"><a id="__codelineno-0-596" name="__codelineno-0-596"></a>        <span class="n">train_size</span><span class="o">=</span><span class="n">train_size</span><span class="p">,</span>
+</span><span id="__span-0-597"><a id="__codelineno-0-597" name="__codelineno-0-597"></a>        <span class="n">random_state</span><span class="o">=</span><span class="n">random_state</span><span class="p">,</span>
+</span><span id="__span-0-598"><a id="__codelineno-0-598" name="__codelineno-0-598"></a>        <span class="n">stratify</span><span class="o">=</span><span class="n">y</span> <span class="k">if</span> <span class="n">stratify_by_target</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-599"><a id="__codelineno-0-599" name="__codelineno-0-599"></a>    <span class="p">)</span>
+</span><span id="__span-0-600"><a id="__codelineno-0-600" name="__codelineno-0-600"></a>    <span class="n">dataset</span> <span class="o">=</span> <span class="n">Dataset</span><span class="p">(</span>
+</span><span id="__span-0-601"><a id="__codelineno-0-601" name="__codelineno-0-601"></a>        <span class="n">x_train</span><span class="o">=</span><span class="n">x_train</span><span class="p">,</span> <span class="n">y_train</span><span class="o">=</span><span class="n">y_train</span><span class="p">,</span> <span class="n">x_test</span><span class="o">=</span><span class="n">x_test</span><span class="p">,</span> <span class="n">y_test</span><span class="o">=</span><span class="n">y_test</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span>
+</span><span id="__span-0-602"><a id="__codelineno-0-602" name="__codelineno-0-602"></a>    <span class="p">)</span>
+</span><span id="__span-0-603"><a id="__codelineno-0-603" name="__codelineno-0-603"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">from_dataset</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">data_groups_train</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -5413,13 +5425,7 @@ <h3 id="pydvl.utils.dataset.GroupedDataset.from_dataset" class="doc doc-heading"
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/dataset.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-599">599</a></span>
-<span class="normal"><a href="#__codelineno-0-600">600</a></span>
-<span class="normal"><a href="#__codelineno-0-601">601</a></span>
-<span class="normal"><a href="#__codelineno-0-602">602</a></span>
-<span class="normal"><a href="#__codelineno-0-603">603</a></span>
-<span class="normal"><a href="#__codelineno-0-604">604</a></span>
-<span class="normal"><a href="#__codelineno-0-605">605</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-605">605</a></span>
 <span class="normal"><a href="#__codelineno-0-606">606</a></span>
 <span class="normal"><a href="#__codelineno-0-607">607</a></span>
 <span class="normal"><a href="#__codelineno-0-608">608</a></span>
@@ -5450,44 +5456,50 @@ <h3 id="pydvl.utils.dataset.GroupedDataset.from_dataset" class="doc doc-heading"
 <span class="normal"><a href="#__codelineno-0-633">633</a></span>
 <span class="normal"><a href="#__codelineno-0-634">634</a></span>
 <span class="normal"><a href="#__codelineno-0-635">635</a></span>
-<span class="normal"><a href="#__codelineno-0-636">636</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-599"><a id="__codelineno-0-599" name="__codelineno-0-599"></a><span class="nd">@classmethod</span>
-</span><span id="__span-0-600"><a id="__codelineno-0-600" name="__codelineno-0-600"></a><span class="k">def</span> <span class="nf">from_dataset</span><span class="p">(</span>
-</span><span id="__span-0-601"><a id="__codelineno-0-601" name="__codelineno-0-601"></a>    <span class="bp">cls</span><span class="p">,</span> <span class="n">dataset</span><span class="p">:</span> <span class="n">Dataset</span><span class="p">,</span> <span class="n">data_groups</span><span class="p">:</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">Any</span><span class="p">]</span>
-</span><span id="__span-0-602"><a id="__codelineno-0-602" name="__codelineno-0-602"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;GroupedDataset&quot;</span><span class="p">:</span>
-</span><span id="__span-0-603"><a id="__codelineno-0-603" name="__codelineno-0-603"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Creates a [GroupedDataset][pydvl.utils.GroupedDataset] object from the data a</span>
-</span><span id="__span-0-604"><a id="__codelineno-0-604" name="__codelineno-0-604"></a><span class="sd">    [Dataset][pydvl.utils.Dataset] object and a mapping of data groups.</span>
-</span><span id="__span-0-605"><a id="__codelineno-0-605" name="__codelineno-0-605"></a>
-</span><span id="__span-0-606"><a id="__codelineno-0-606" name="__codelineno-0-606"></a><span class="sd">    ??? Example</span>
-</span><span id="__span-0-607"><a id="__codelineno-0-607" name="__codelineno-0-607"></a><span class="sd">        ```pycon</span>
-</span><span id="__span-0-608"><a id="__codelineno-0-608" name="__codelineno-0-608"></a><span class="sd">        &gt;&gt;&gt; import numpy as np</span>
-</span><span id="__span-0-609"><a id="__codelineno-0-609" name="__codelineno-0-609"></a><span class="sd">        &gt;&gt;&gt; from pydvl.utils import Dataset, GroupedDataset</span>
-</span><span id="__span-0-610"><a id="__codelineno-0-610" name="__codelineno-0-610"></a><span class="sd">        &gt;&gt;&gt; dataset = Dataset.from_arrays(</span>
-</span><span id="__span-0-611"><a id="__codelineno-0-611" name="__codelineno-0-611"></a><span class="sd">        ...     X=np.asarray([[1, 2], [3, 4], [5, 6], [7, 8]]),</span>
-</span><span id="__span-0-612"><a id="__codelineno-0-612" name="__codelineno-0-612"></a><span class="sd">        ...     y=np.asarray([0, 1, 0, 1]),</span>
-</span><span id="__span-0-613"><a id="__codelineno-0-613" name="__codelineno-0-613"></a><span class="sd">        ... )</span>
-</span><span id="__span-0-614"><a id="__codelineno-0-614" name="__codelineno-0-614"></a><span class="sd">        &gt;&gt;&gt; dataset = GroupedDataset.from_dataset(dataset, data_groups=[0, 0, 1, 1])</span>
-</span><span id="__span-0-615"><a id="__codelineno-0-615" name="__codelineno-0-615"></a><span class="sd">        ```</span>
-</span><span id="__span-0-616"><a id="__codelineno-0-616" name="__codelineno-0-616"></a>
-</span><span id="__span-0-617"><a id="__codelineno-0-617" name="__codelineno-0-617"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-618"><a id="__codelineno-0-618" name="__codelineno-0-618"></a><span class="sd">        dataset: The original data.</span>
-</span><span id="__span-0-619"><a id="__codelineno-0-619" name="__codelineno-0-619"></a><span class="sd">        data_groups: An array holding the group index or name for each data</span>
-</span><span id="__span-0-620"><a id="__codelineno-0-620" name="__codelineno-0-620"></a><span class="sd">            point. The length of this array must be equal to the number of</span>
-</span><span id="__span-0-621"><a id="__codelineno-0-621" name="__codelineno-0-621"></a><span class="sd">            data points in the dataset.</span>
+<span class="normal"><a href="#__codelineno-0-636">636</a></span>
+<span class="normal"><a href="#__codelineno-0-637">637</a></span>
+<span class="normal"><a href="#__codelineno-0-638">638</a></span>
+<span class="normal"><a href="#__codelineno-0-639">639</a></span>
+<span class="normal"><a href="#__codelineno-0-640">640</a></span>
+<span class="normal"><a href="#__codelineno-0-641">641</a></span>
+<span class="normal"><a href="#__codelineno-0-642">642</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-605"><a id="__codelineno-0-605" name="__codelineno-0-605"></a><span class="nd">@classmethod</span>
+</span><span id="__span-0-606"><a id="__codelineno-0-606" name="__codelineno-0-606"></a><span class="k">def</span> <span class="nf">from_dataset</span><span class="p">(</span>
+</span><span id="__span-0-607"><a id="__codelineno-0-607" name="__codelineno-0-607"></a>    <span class="bp">cls</span><span class="p">,</span> <span class="n">dataset</span><span class="p">:</span> <span class="n">Dataset</span><span class="p">,</span> <span class="n">data_groups</span><span class="p">:</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">Any</span><span class="p">]</span>
+</span><span id="__span-0-608"><a id="__codelineno-0-608" name="__codelineno-0-608"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;GroupedDataset&quot;</span><span class="p">:</span>
+</span><span id="__span-0-609"><a id="__codelineno-0-609" name="__codelineno-0-609"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Creates a [GroupedDataset][pydvl.utils.GroupedDataset] object from the data a</span>
+</span><span id="__span-0-610"><a id="__codelineno-0-610" name="__codelineno-0-610"></a><span class="sd">    [Dataset][pydvl.utils.Dataset] object and a mapping of data groups.</span>
+</span><span id="__span-0-611"><a id="__codelineno-0-611" name="__codelineno-0-611"></a>
+</span><span id="__span-0-612"><a id="__codelineno-0-612" name="__codelineno-0-612"></a><span class="sd">    ??? Example</span>
+</span><span id="__span-0-613"><a id="__codelineno-0-613" name="__codelineno-0-613"></a><span class="sd">        ```pycon</span>
+</span><span id="__span-0-614"><a id="__codelineno-0-614" name="__codelineno-0-614"></a><span class="sd">        &gt;&gt;&gt; import numpy as np</span>
+</span><span id="__span-0-615"><a id="__codelineno-0-615" name="__codelineno-0-615"></a><span class="sd">        &gt;&gt;&gt; from pydvl.utils import Dataset, GroupedDataset</span>
+</span><span id="__span-0-616"><a id="__codelineno-0-616" name="__codelineno-0-616"></a><span class="sd">        &gt;&gt;&gt; dataset = Dataset.from_arrays(</span>
+</span><span id="__span-0-617"><a id="__codelineno-0-617" name="__codelineno-0-617"></a><span class="sd">        ...     X=np.asarray([[1, 2], [3, 4], [5, 6], [7, 8]]),</span>
+</span><span id="__span-0-618"><a id="__codelineno-0-618" name="__codelineno-0-618"></a><span class="sd">        ...     y=np.asarray([0, 1, 0, 1]),</span>
+</span><span id="__span-0-619"><a id="__codelineno-0-619" name="__codelineno-0-619"></a><span class="sd">        ... )</span>
+</span><span id="__span-0-620"><a id="__codelineno-0-620" name="__codelineno-0-620"></a><span class="sd">        &gt;&gt;&gt; dataset = GroupedDataset.from_dataset(dataset, data_groups=[0, 0, 1, 1])</span>
+</span><span id="__span-0-621"><a id="__codelineno-0-621" name="__codelineno-0-621"></a><span class="sd">        ```</span>
 </span><span id="__span-0-622"><a id="__codelineno-0-622" name="__codelineno-0-622"></a>
-</span><span id="__span-0-623"><a id="__codelineno-0-623" name="__codelineno-0-623"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-624"><a id="__codelineno-0-624" name="__codelineno-0-624"></a><span class="sd">        A [GroupedDataset][pydvl.utils.GroupedDataset] with the initial</span>
-</span><span id="__span-0-625"><a id="__codelineno-0-625" name="__codelineno-0-625"></a><span class="sd">            [Dataset][pydvl.utils.Dataset] grouped by data_groups.</span>
-</span><span id="__span-0-626"><a id="__codelineno-0-626" name="__codelineno-0-626"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-627"><a id="__codelineno-0-627" name="__codelineno-0-627"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span>
-</span><span id="__span-0-628"><a id="__codelineno-0-628" name="__codelineno-0-628"></a>        <span class="n">x_train</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">x_train</span><span class="p">,</span>
-</span><span id="__span-0-629"><a id="__codelineno-0-629" name="__codelineno-0-629"></a>        <span class="n">y_train</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">y_train</span><span class="p">,</span>
-</span><span id="__span-0-630"><a id="__codelineno-0-630" name="__codelineno-0-630"></a>        <span class="n">x_test</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">x_test</span><span class="p">,</span>
-</span><span id="__span-0-631"><a id="__codelineno-0-631" name="__codelineno-0-631"></a>        <span class="n">y_test</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">y_test</span><span class="p">,</span>
-</span><span id="__span-0-632"><a id="__codelineno-0-632" name="__codelineno-0-632"></a>        <span class="n">data_groups</span><span class="o">=</span><span class="n">data_groups</span><span class="p">,</span>
-</span><span id="__span-0-633"><a id="__codelineno-0-633" name="__codelineno-0-633"></a>        <span class="n">feature_names</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">feature_names</span><span class="p">,</span>
-</span><span id="__span-0-634"><a id="__codelineno-0-634" name="__codelineno-0-634"></a>        <span class="n">target_names</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">target_names</span><span class="p">,</span>
-</span><span id="__span-0-635"><a id="__codelineno-0-635" name="__codelineno-0-635"></a>        <span class="n">description</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">description</span><span class="p">,</span>
-</span><span id="__span-0-636"><a id="__codelineno-0-636" name="__codelineno-0-636"></a>    <span class="p">)</span>
+</span><span id="__span-0-623"><a id="__codelineno-0-623" name="__codelineno-0-623"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-624"><a id="__codelineno-0-624" name="__codelineno-0-624"></a><span class="sd">        dataset: The original data.</span>
+</span><span id="__span-0-625"><a id="__codelineno-0-625" name="__codelineno-0-625"></a><span class="sd">        data_groups: An array holding the group index or name for each data</span>
+</span><span id="__span-0-626"><a id="__codelineno-0-626" name="__codelineno-0-626"></a><span class="sd">            point. The length of this array must be equal to the number of</span>
+</span><span id="__span-0-627"><a id="__codelineno-0-627" name="__codelineno-0-627"></a><span class="sd">            data points in the dataset.</span>
+</span><span id="__span-0-628"><a id="__codelineno-0-628" name="__codelineno-0-628"></a>
+</span><span id="__span-0-629"><a id="__codelineno-0-629" name="__codelineno-0-629"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-630"><a id="__codelineno-0-630" name="__codelineno-0-630"></a><span class="sd">        A [GroupedDataset][pydvl.utils.GroupedDataset] with the initial</span>
+</span><span id="__span-0-631"><a id="__codelineno-0-631" name="__codelineno-0-631"></a><span class="sd">            [Dataset][pydvl.utils.Dataset] grouped by data_groups.</span>
+</span><span id="__span-0-632"><a id="__codelineno-0-632" name="__codelineno-0-632"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-633"><a id="__codelineno-0-633" name="__codelineno-0-633"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span>
+</span><span id="__span-0-634"><a id="__codelineno-0-634" name="__codelineno-0-634"></a>        <span class="n">x_train</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">x_train</span><span class="p">,</span>
+</span><span id="__span-0-635"><a id="__codelineno-0-635" name="__codelineno-0-635"></a>        <span class="n">y_train</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">y_train</span><span class="p">,</span>
+</span><span id="__span-0-636"><a id="__codelineno-0-636" name="__codelineno-0-636"></a>        <span class="n">x_test</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">x_test</span><span class="p">,</span>
+</span><span id="__span-0-637"><a id="__codelineno-0-637" name="__codelineno-0-637"></a>        <span class="n">y_test</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">y_test</span><span class="p">,</span>
+</span><span id="__span-0-638"><a id="__codelineno-0-638" name="__codelineno-0-638"></a>        <span class="n">data_groups</span><span class="o">=</span><span class="n">data_groups</span><span class="p">,</span>
+</span><span id="__span-0-639"><a id="__codelineno-0-639" name="__codelineno-0-639"></a>        <span class="n">feature_names</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">feature_names</span><span class="p">,</span>
+</span><span id="__span-0-640"><a id="__codelineno-0-640" name="__codelineno-0-640"></a>        <span class="n">target_names</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">target_names</span><span class="p">,</span>
+</span><span id="__span-0-641"><a id="__codelineno-0-641" name="__codelineno-0-641"></a>        <span class="n">description</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">description</span><span class="p">,</span>
+</span><span id="__span-0-642"><a id="__codelineno-0-642" name="__codelineno-0-642"></a>    <span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -5516,11 +5528,11 @@ <h3 id="pydvl.utils.dataset.GroupedDataset.from_dataset" class="doc doc-heading"
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/utils/functional/index.html b/devel/api/pydvl/utils/functional/index.html
index a5c87d36b..96cd31706 100644
--- a/devel/api/pydvl/utils/functional/index.html
+++ b/devel/api/pydvl/utils/functional/index.html
@@ -2888,11 +2888,11 @@ <h2 id="pydvl.utils.functional.maybe_add_argument" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/utils/index.html b/devel/api/pydvl/utils/index.html
index 6f99b593b..fd306cd21 100644
--- a/devel/api/pydvl/utils/index.html
+++ b/devel/api/pydvl/utils/index.html
@@ -2491,11 +2491,11 @@ <h1>Utils</h1>
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/utils/numeric/index.html b/devel/api/pydvl/utils/numeric/index.html
index fc16fe6f3..cd56fe9da 100644
--- a/devel/api/pydvl/utils/numeric/index.html
+++ b/devel/api/pydvl/utils/numeric/index.html
@@ -2973,7 +2973,8 @@ <h2 id="pydvl.utils.numeric.random_subset" class="doc doc-heading">
           <td><code>seed</code></td>
           <td class="doc-param-details">
             <div class="doc-md-description">
-              <p>Either an instance of a numpy random number generator or a seed for it.</p>
+              <p>Either an instance of a numpy random number generator or a seed
+for it.</p>
             </div>
             <p>
                 <span class="doc-param-annotation">
@@ -3034,26 +3035,24 @@ <h2 id="pydvl.utils.numeric.random_subset" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-87">87</a></span>
 <span class="normal"><a href="#__codelineno-0-88">88</a></span>
 <span class="normal"><a href="#__codelineno-0-89">89</a></span>
-<span class="normal"><a href="#__codelineno-0-90">90</a></span>
-<span class="normal"><a href="#__codelineno-0-91">91</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-73"><a id="__codelineno-0-73" name="__codelineno-0-73"></a><span class="k">def</span> <span class="nf">random_subset</span><span class="p">(</span>
-</span><span id="__span-0-74"><a id="__codelineno-0-74" name="__codelineno-0-74"></a>    <span class="n">s</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">],</span>
-</span><span id="__span-0-75"><a id="__codelineno-0-75" name="__codelineno-0-75"></a>    <span class="n">q</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.5</span><span class="p">,</span>
-</span><span id="__span-0-76"><a id="__codelineno-0-76" name="__codelineno-0-76"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-77"><a id="__codelineno-0-77" name="__codelineno-0-77"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">]:</span>
-</span><span id="__span-0-78"><a id="__codelineno-0-78" name="__codelineno-0-78"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns one subset at random from ``s``.</span>
-</span><span id="__span-0-79"><a id="__codelineno-0-79" name="__codelineno-0-79"></a>
-</span><span id="__span-0-80"><a id="__codelineno-0-80" name="__codelineno-0-80"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-81"><a id="__codelineno-0-81" name="__codelineno-0-81"></a><span class="sd">        s: set to sample from</span>
-</span><span id="__span-0-82"><a id="__codelineno-0-82" name="__codelineno-0-82"></a><span class="sd">        q: Sampling probability for elements. The default 0.5 yields a</span>
-</span><span id="__span-0-83"><a id="__codelineno-0-83" name="__codelineno-0-83"></a><span class="sd">            uniform distribution over the power set of s.</span>
-</span><span id="__span-0-84"><a id="__codelineno-0-84" name="__codelineno-0-84"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
-</span><span id="__span-0-85"><a id="__codelineno-0-85" name="__codelineno-0-85"></a>
-</span><span id="__span-0-86"><a id="__codelineno-0-86" name="__codelineno-0-86"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-87"><a id="__codelineno-0-87" name="__codelineno-0-87"></a><span class="sd">        The subset</span>
-</span><span id="__span-0-88"><a id="__codelineno-0-88" name="__codelineno-0-88"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-89"><a id="__codelineno-0-89" name="__codelineno-0-89"></a>    <span class="n">rng</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">default_rng</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
-</span><span id="__span-0-90"><a id="__codelineno-0-90" name="__codelineno-0-90"></a>    <span class="n">selection</span> <span class="o">=</span> <span class="n">rng</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="n">s</span><span class="p">))</span> <span class="o">&gt;</span> <span class="n">q</span>
-</span><span id="__span-0-91"><a id="__codelineno-0-91" name="__codelineno-0-91"></a>    <span class="k">return</span> <span class="n">s</span><span class="p">[</span><span class="n">selection</span><span class="p">]</span>
+<span class="normal"><a href="#__codelineno-0-90">90</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-73"><a id="__codelineno-0-73" name="__codelineno-0-73"></a><span class="k">def</span> <span class="nf">random_subset</span><span class="p">(</span>
+</span><span id="__span-0-74"><a id="__codelineno-0-74" name="__codelineno-0-74"></a>    <span class="n">s</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">],</span> <span class="n">q</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.5</span><span class="p">,</span> <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+</span><span id="__span-0-75"><a id="__codelineno-0-75" name="__codelineno-0-75"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">]:</span>
+</span><span id="__span-0-76"><a id="__codelineno-0-76" name="__codelineno-0-76"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns one subset at random from ``s``.</span>
+</span><span id="__span-0-77"><a id="__codelineno-0-77" name="__codelineno-0-77"></a>
+</span><span id="__span-0-78"><a id="__codelineno-0-78" name="__codelineno-0-78"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-79"><a id="__codelineno-0-79" name="__codelineno-0-79"></a><span class="sd">        s: set to sample from</span>
+</span><span id="__span-0-80"><a id="__codelineno-0-80" name="__codelineno-0-80"></a><span class="sd">        q: Sampling probability for elements. The default 0.5 yields a</span>
+</span><span id="__span-0-81"><a id="__codelineno-0-81" name="__codelineno-0-81"></a><span class="sd">            uniform distribution over the power set of s.</span>
+</span><span id="__span-0-82"><a id="__codelineno-0-82" name="__codelineno-0-82"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed</span>
+</span><span id="__span-0-83"><a id="__codelineno-0-83" name="__codelineno-0-83"></a><span class="sd">            for it.</span>
+</span><span id="__span-0-84"><a id="__codelineno-0-84" name="__codelineno-0-84"></a>
+</span><span id="__span-0-85"><a id="__codelineno-0-85" name="__codelineno-0-85"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-86"><a id="__codelineno-0-86" name="__codelineno-0-86"></a><span class="sd">        The subset</span>
+</span><span id="__span-0-87"><a id="__codelineno-0-87" name="__codelineno-0-87"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-88"><a id="__codelineno-0-88" name="__codelineno-0-88"></a>    <span class="n">rng</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">default_rng</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
+</span><span id="__span-0-89"><a id="__codelineno-0-89" name="__codelineno-0-89"></a>    <span class="n">selection</span> <span class="o">=</span> <span class="n">rng</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="n">s</span><span class="p">))</span> <span class="o">&gt;</span> <span class="n">q</span>
+</span><span id="__span-0-90"><a id="__codelineno-0-90" name="__codelineno-0-90"></a>    <span class="k">return</span> <span class="n">s</span><span class="p">[</span><span class="n">selection</span><span class="p">]</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3217,7 +3216,8 @@ <h2 id="pydvl.utils.numeric.random_powerset" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/numeric.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-94"> 94</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-93"> 93</a></span>
+<span class="normal"><a href="#__codelineno-0-94"> 94</a></span>
 <span class="normal"><a href="#__codelineno-0-95"> 95</a></span>
 <span class="normal"><a href="#__codelineno-0-96"> 96</a></span>
 <span class="normal"><a href="#__codelineno-0-97"> 97</a></span>
@@ -3256,48 +3256,47 @@ <h2 id="pydvl.utils.numeric.random_powerset" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-130">130</a></span>
 <span class="normal"><a href="#__codelineno-0-131">131</a></span>
 <span class="normal"><a href="#__codelineno-0-132">132</a></span>
-<span class="normal"><a href="#__codelineno-0-133">133</a></span>
-<span class="normal"><a href="#__codelineno-0-134">134</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-94"><a id="__codelineno-0-94" name="__codelineno-0-94"></a><span class="k">def</span> <span class="nf">random_powerset</span><span class="p">(</span>
-</span><span id="__span-0-95"><a id="__codelineno-0-95" name="__codelineno-0-95"></a>    <span class="n">s</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">],</span>
-</span><span id="__span-0-96"><a id="__codelineno-0-96" name="__codelineno-0-96"></a>    <span class="n">n_samples</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-97"><a id="__codelineno-0-97" name="__codelineno-0-97"></a>    <span class="n">q</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.5</span><span class="p">,</span>
-</span><span id="__span-0-98"><a id="__codelineno-0-98" name="__codelineno-0-98"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-99"><a id="__codelineno-0-99" name="__codelineno-0-99"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Generator</span><span class="p">[</span><span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">],</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">]:</span>
-</span><span id="__span-0-100"><a id="__codelineno-0-100" name="__codelineno-0-100"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Samples subsets from the power set of the argument, without</span>
-</span><span id="__span-0-101"><a id="__codelineno-0-101" name="__codelineno-0-101"></a><span class="sd">    pre-generating all subsets and in no order.</span>
-</span><span id="__span-0-102"><a id="__codelineno-0-102" name="__codelineno-0-102"></a>
-</span><span id="__span-0-103"><a id="__codelineno-0-103" name="__codelineno-0-103"></a><span class="sd">    See [powerset][pydvl.utils.numeric.powerset] if you wish to deterministically generate all subsets.</span>
-</span><span id="__span-0-104"><a id="__codelineno-0-104" name="__codelineno-0-104"></a>
-</span><span id="__span-0-105"><a id="__codelineno-0-105" name="__codelineno-0-105"></a><span class="sd">    To generate subsets, `len(s)` Bernoulli draws with probability `q` are</span>
-</span><span id="__span-0-106"><a id="__codelineno-0-106" name="__codelineno-0-106"></a><span class="sd">    drawn. The default value of `q = 0.5` provides a uniform distribution over</span>
-</span><span id="__span-0-107"><a id="__codelineno-0-107" name="__codelineno-0-107"></a><span class="sd">    the power set of `s`. Other choices can be used e.g. to implement</span>
-</span><span id="__span-0-108"><a id="__codelineno-0-108" name="__codelineno-0-108"></a><span class="sd">    [owen_sampling_shapley][pydvl.value.shapley.owen.owen_sampling_shapley].</span>
-</span><span id="__span-0-109"><a id="__codelineno-0-109" name="__codelineno-0-109"></a>
-</span><span id="__span-0-110"><a id="__codelineno-0-110" name="__codelineno-0-110"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-111"><a id="__codelineno-0-111" name="__codelineno-0-111"></a><span class="sd">        s: set to sample from</span>
-</span><span id="__span-0-112"><a id="__codelineno-0-112" name="__codelineno-0-112"></a><span class="sd">        n_samples: if set, stop the generator after this many steps.</span>
-</span><span id="__span-0-113"><a id="__codelineno-0-113" name="__codelineno-0-113"></a><span class="sd">            Defaults to `np.iinfo(np.int32).max`</span>
-</span><span id="__span-0-114"><a id="__codelineno-0-114" name="__codelineno-0-114"></a><span class="sd">        q: Sampling probability for elements. The default 0.5 yields a</span>
-</span><span id="__span-0-115"><a id="__codelineno-0-115" name="__codelineno-0-115"></a><span class="sd">            uniform distribution over the power set of s.</span>
-</span><span id="__span-0-116"><a id="__codelineno-0-116" name="__codelineno-0-116"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
-</span><span id="__span-0-117"><a id="__codelineno-0-117" name="__codelineno-0-117"></a>
-</span><span id="__span-0-118"><a id="__codelineno-0-118" name="__codelineno-0-118"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-119"><a id="__codelineno-0-119" name="__codelineno-0-119"></a><span class="sd">        Samples from the power set of `s`.</span>
-</span><span id="__span-0-120"><a id="__codelineno-0-120" name="__codelineno-0-120"></a>
-</span><span id="__span-0-121"><a id="__codelineno-0-121" name="__codelineno-0-121"></a><span class="sd">    Raises:</span>
-</span><span id="__span-0-122"><a id="__codelineno-0-122" name="__codelineno-0-122"></a><span class="sd">        ValueError: if the element sampling probability is not in [0,1]</span>
-</span><span id="__span-0-123"><a id="__codelineno-0-123" name="__codelineno-0-123"></a>
-</span><span id="__span-0-124"><a id="__codelineno-0-124" name="__codelineno-0-124"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-125"><a id="__codelineno-0-125" name="__codelineno-0-125"></a>    <span class="k">if</span> <span class="n">q</span> <span class="o">&lt;</span> <span class="mi">0</span> <span class="ow">or</span> <span class="n">q</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
-</span><span id="__span-0-126"><a id="__codelineno-0-126" name="__codelineno-0-126"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Element sampling probability must be in [0,1]&quot;</span><span class="p">)</span>
-</span><span id="__span-0-127"><a id="__codelineno-0-127" name="__codelineno-0-127"></a>
-</span><span id="__span-0-128"><a id="__codelineno-0-128" name="__codelineno-0-128"></a>    <span class="n">rng</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">default_rng</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
-</span><span id="__span-0-129"><a id="__codelineno-0-129" name="__codelineno-0-129"></a>    <span class="n">total</span> <span class="o">=</span> <span class="mi">1</span>
-</span><span id="__span-0-130"><a id="__codelineno-0-130" name="__codelineno-0-130"></a>    <span class="k">if</span> <span class="n">n_samples</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-</span><span id="__span-0-131"><a id="__codelineno-0-131" name="__codelineno-0-131"></a>        <span class="n">n_samples</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">iinfo</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span><span class="o">.</span><span class="n">max</span>
-</span><span id="__span-0-132"><a id="__codelineno-0-132" name="__codelineno-0-132"></a>    <span class="k">while</span> <span class="n">total</span> <span class="o">&lt;=</span> <span class="n">n_samples</span><span class="p">:</span>
-</span><span id="__span-0-133"><a id="__codelineno-0-133" name="__codelineno-0-133"></a>        <span class="k">yield</span> <span class="n">random_subset</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">q</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="n">rng</span><span class="p">)</span>
-</span><span id="__span-0-134"><a id="__codelineno-0-134" name="__codelineno-0-134"></a>        <span class="n">total</span> <span class="o">+=</span> <span class="mi">1</span>
+<span class="normal"><a href="#__codelineno-0-133">133</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-93"><a id="__codelineno-0-93" name="__codelineno-0-93"></a><span class="k">def</span> <span class="nf">random_powerset</span><span class="p">(</span>
+</span><span id="__span-0-94"><a id="__codelineno-0-94" name="__codelineno-0-94"></a>    <span class="n">s</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">],</span>
+</span><span id="__span-0-95"><a id="__codelineno-0-95" name="__codelineno-0-95"></a>    <span class="n">n_samples</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-96"><a id="__codelineno-0-96" name="__codelineno-0-96"></a>    <span class="n">q</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.5</span><span class="p">,</span>
+</span><span id="__span-0-97"><a id="__codelineno-0-97" name="__codelineno-0-97"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-98"><a id="__codelineno-0-98" name="__codelineno-0-98"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Generator</span><span class="p">[</span><span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">],</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">]:</span>
+</span><span id="__span-0-99"><a id="__codelineno-0-99" name="__codelineno-0-99"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Samples subsets from the power set of the argument, without</span>
+</span><span id="__span-0-100"><a id="__codelineno-0-100" name="__codelineno-0-100"></a><span class="sd">    pre-generating all subsets and in no order.</span>
+</span><span id="__span-0-101"><a id="__codelineno-0-101" name="__codelineno-0-101"></a>
+</span><span id="__span-0-102"><a id="__codelineno-0-102" name="__codelineno-0-102"></a><span class="sd">    See [powerset][pydvl.utils.numeric.powerset] if you wish to deterministically generate all subsets.</span>
+</span><span id="__span-0-103"><a id="__codelineno-0-103" name="__codelineno-0-103"></a>
+</span><span id="__span-0-104"><a id="__codelineno-0-104" name="__codelineno-0-104"></a><span class="sd">    To generate subsets, `len(s)` Bernoulli draws with probability `q` are</span>
+</span><span id="__span-0-105"><a id="__codelineno-0-105" name="__codelineno-0-105"></a><span class="sd">    drawn. The default value of `q = 0.5` provides a uniform distribution over</span>
+</span><span id="__span-0-106"><a id="__codelineno-0-106" name="__codelineno-0-106"></a><span class="sd">    the power set of `s`. Other choices can be used e.g. to implement</span>
+</span><span id="__span-0-107"><a id="__codelineno-0-107" name="__codelineno-0-107"></a><span class="sd">    [owen_sampling_shapley][pydvl.value.shapley.owen.owen_sampling_shapley].</span>
+</span><span id="__span-0-108"><a id="__codelineno-0-108" name="__codelineno-0-108"></a>
+</span><span id="__span-0-109"><a id="__codelineno-0-109" name="__codelineno-0-109"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-110"><a id="__codelineno-0-110" name="__codelineno-0-110"></a><span class="sd">        s: set to sample from</span>
+</span><span id="__span-0-111"><a id="__codelineno-0-111" name="__codelineno-0-111"></a><span class="sd">        n_samples: if set, stop the generator after this many steps.</span>
+</span><span id="__span-0-112"><a id="__codelineno-0-112" name="__codelineno-0-112"></a><span class="sd">            Defaults to `np.iinfo(np.int32).max`</span>
+</span><span id="__span-0-113"><a id="__codelineno-0-113" name="__codelineno-0-113"></a><span class="sd">        q: Sampling probability for elements. The default 0.5 yields a</span>
+</span><span id="__span-0-114"><a id="__codelineno-0-114" name="__codelineno-0-114"></a><span class="sd">            uniform distribution over the power set of s.</span>
+</span><span id="__span-0-115"><a id="__codelineno-0-115" name="__codelineno-0-115"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
+</span><span id="__span-0-116"><a id="__codelineno-0-116" name="__codelineno-0-116"></a>
+</span><span id="__span-0-117"><a id="__codelineno-0-117" name="__codelineno-0-117"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-118"><a id="__codelineno-0-118" name="__codelineno-0-118"></a><span class="sd">        Samples from the power set of `s`.</span>
+</span><span id="__span-0-119"><a id="__codelineno-0-119" name="__codelineno-0-119"></a>
+</span><span id="__span-0-120"><a id="__codelineno-0-120" name="__codelineno-0-120"></a><span class="sd">    Raises:</span>
+</span><span id="__span-0-121"><a id="__codelineno-0-121" name="__codelineno-0-121"></a><span class="sd">        ValueError: if the element sampling probability is not in [0,1]</span>
+</span><span id="__span-0-122"><a id="__codelineno-0-122" name="__codelineno-0-122"></a>
+</span><span id="__span-0-123"><a id="__codelineno-0-123" name="__codelineno-0-123"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-124"><a id="__codelineno-0-124" name="__codelineno-0-124"></a>    <span class="k">if</span> <span class="n">q</span> <span class="o">&lt;</span> <span class="mi">0</span> <span class="ow">or</span> <span class="n">q</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
+</span><span id="__span-0-125"><a id="__codelineno-0-125" name="__codelineno-0-125"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Element sampling probability must be in [0,1]&quot;</span><span class="p">)</span>
+</span><span id="__span-0-126"><a id="__codelineno-0-126" name="__codelineno-0-126"></a>
+</span><span id="__span-0-127"><a id="__codelineno-0-127" name="__codelineno-0-127"></a>    <span class="n">rng</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">default_rng</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
+</span><span id="__span-0-128"><a id="__codelineno-0-128" name="__codelineno-0-128"></a>    <span class="n">total</span> <span class="o">=</span> <span class="mi">1</span>
+</span><span id="__span-0-129"><a id="__codelineno-0-129" name="__codelineno-0-129"></a>    <span class="k">if</span> <span class="n">n_samples</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+</span><span id="__span-0-130"><a id="__codelineno-0-130" name="__codelineno-0-130"></a>        <span class="n">n_samples</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">iinfo</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span><span class="o">.</span><span class="n">max</span>
+</span><span id="__span-0-131"><a id="__codelineno-0-131" name="__codelineno-0-131"></a>    <span class="k">while</span> <span class="n">total</span> <span class="o">&lt;=</span> <span class="n">n_samples</span><span class="p">:</span>
+</span><span id="__span-0-132"><a id="__codelineno-0-132" name="__codelineno-0-132"></a>        <span class="k">yield</span> <span class="n">random_subset</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">q</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="n">rng</span><span class="p">)</span>
+</span><span id="__span-0-133"><a id="__codelineno-0-133" name="__codelineno-0-133"></a>        <span class="n">total</span> <span class="o">+=</span> <span class="mi">1</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3409,7 +3408,8 @@ <h2 id="pydvl.utils.numeric.random_subset_of_size" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/numeric.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-137">137</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-136">136</a></span>
+<span class="normal"><a href="#__codelineno-0-137">137</a></span>
 <span class="normal"><a href="#__codelineno-0-138">138</a></span>
 <span class="normal"><a href="#__codelineno-0-139">139</a></span>
 <span class="normal"><a href="#__codelineno-0-140">140</a></span>
@@ -3428,32 +3428,27 @@ <h2 id="pydvl.utils.numeric.random_subset_of_size" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-153">153</a></span>
 <span class="normal"><a href="#__codelineno-0-154">154</a></span>
 <span class="normal"><a href="#__codelineno-0-155">155</a></span>
-<span class="normal"><a href="#__codelineno-0-156">156</a></span>
-<span class="normal"><a href="#__codelineno-0-157">157</a></span>
-<span class="normal"><a href="#__codelineno-0-158">158</a></span>
-<span class="normal"><a href="#__codelineno-0-159">159</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-137"><a id="__codelineno-0-137" name="__codelineno-0-137"></a><span class="k">def</span> <span class="nf">random_subset_of_size</span><span class="p">(</span>
-</span><span id="__span-0-138"><a id="__codelineno-0-138" name="__codelineno-0-138"></a>    <span class="n">s</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">],</span>
-</span><span id="__span-0-139"><a id="__codelineno-0-139" name="__codelineno-0-139"></a>    <span class="n">size</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-</span><span id="__span-0-140"><a id="__codelineno-0-140" name="__codelineno-0-140"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-141"><a id="__codelineno-0-141" name="__codelineno-0-141"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">]:</span>
-</span><span id="__span-0-142"><a id="__codelineno-0-142" name="__codelineno-0-142"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Samples a random subset of given size uniformly from the powerset</span>
-</span><span id="__span-0-143"><a id="__codelineno-0-143" name="__codelineno-0-143"></a><span class="sd">    of `s`.</span>
-</span><span id="__span-0-144"><a id="__codelineno-0-144" name="__codelineno-0-144"></a>
-</span><span id="__span-0-145"><a id="__codelineno-0-145" name="__codelineno-0-145"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-146"><a id="__codelineno-0-146" name="__codelineno-0-146"></a><span class="sd">        s: Set to sample from</span>
-</span><span id="__span-0-147"><a id="__codelineno-0-147" name="__codelineno-0-147"></a><span class="sd">        size: Size of the subset to generate</span>
-</span><span id="__span-0-148"><a id="__codelineno-0-148" name="__codelineno-0-148"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
+<span class="normal"><a href="#__codelineno-0-156">156</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-136"><a id="__codelineno-0-136" name="__codelineno-0-136"></a><span class="k">def</span> <span class="nf">random_subset_of_size</span><span class="p">(</span>
+</span><span id="__span-0-137"><a id="__codelineno-0-137" name="__codelineno-0-137"></a>    <span class="n">s</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">],</span> <span class="n">size</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+</span><span id="__span-0-138"><a id="__codelineno-0-138" name="__codelineno-0-138"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">]:</span>
+</span><span id="__span-0-139"><a id="__codelineno-0-139" name="__codelineno-0-139"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Samples a random subset of given size uniformly from the powerset</span>
+</span><span id="__span-0-140"><a id="__codelineno-0-140" name="__codelineno-0-140"></a><span class="sd">    of `s`.</span>
+</span><span id="__span-0-141"><a id="__codelineno-0-141" name="__codelineno-0-141"></a>
+</span><span id="__span-0-142"><a id="__codelineno-0-142" name="__codelineno-0-142"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-143"><a id="__codelineno-0-143" name="__codelineno-0-143"></a><span class="sd">        s: Set to sample from</span>
+</span><span id="__span-0-144"><a id="__codelineno-0-144" name="__codelineno-0-144"></a><span class="sd">        size: Size of the subset to generate</span>
+</span><span id="__span-0-145"><a id="__codelineno-0-145" name="__codelineno-0-145"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
+</span><span id="__span-0-146"><a id="__codelineno-0-146" name="__codelineno-0-146"></a>
+</span><span id="__span-0-147"><a id="__codelineno-0-147" name="__codelineno-0-147"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-148"><a id="__codelineno-0-148" name="__codelineno-0-148"></a><span class="sd">        The subset</span>
 </span><span id="__span-0-149"><a id="__codelineno-0-149" name="__codelineno-0-149"></a>
-</span><span id="__span-0-150"><a id="__codelineno-0-150" name="__codelineno-0-150"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-151"><a id="__codelineno-0-151" name="__codelineno-0-151"></a><span class="sd">        The subset</span>
-</span><span id="__span-0-152"><a id="__codelineno-0-152" name="__codelineno-0-152"></a>
-</span><span id="__span-0-153"><a id="__codelineno-0-153" name="__codelineno-0-153"></a><span class="sd">    Raises</span>
-</span><span id="__span-0-154"><a id="__codelineno-0-154" name="__codelineno-0-154"></a><span class="sd">        ValueError: If size &gt; len(s)</span>
-</span><span id="__span-0-155"><a id="__codelineno-0-155" name="__codelineno-0-155"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-156"><a id="__codelineno-0-156" name="__codelineno-0-156"></a>    <span class="k">if</span> <span class="n">size</span> <span class="o">&gt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
-</span><span id="__span-0-157"><a id="__codelineno-0-157" name="__codelineno-0-157"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Cannot sample subset larger than set&quot;</span><span class="p">)</span>
-</span><span id="__span-0-158"><a id="__codelineno-0-158" name="__codelineno-0-158"></a>    <span class="n">rng</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">default_rng</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
-</span><span id="__span-0-159"><a id="__codelineno-0-159" name="__codelineno-0-159"></a>    <span class="k">return</span> <span class="n">rng</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="n">size</span><span class="p">,</span> <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+</span><span id="__span-0-150"><a id="__codelineno-0-150" name="__codelineno-0-150"></a><span class="sd">    Raises</span>
+</span><span id="__span-0-151"><a id="__codelineno-0-151" name="__codelineno-0-151"></a><span class="sd">        ValueError: If size &gt; len(s)</span>
+</span><span id="__span-0-152"><a id="__codelineno-0-152" name="__codelineno-0-152"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-153"><a id="__codelineno-0-153" name="__codelineno-0-153"></a>    <span class="k">if</span> <span class="n">size</span> <span class="o">&gt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
+</span><span id="__span-0-154"><a id="__codelineno-0-154" name="__codelineno-0-154"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Cannot sample subset larger than set&quot;</span><span class="p">)</span>
+</span><span id="__span-0-155"><a id="__codelineno-0-155" name="__codelineno-0-155"></a>    <span class="n">rng</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">default_rng</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
+</span><span id="__span-0-156"><a id="__codelineno-0-156" name="__codelineno-0-156"></a>    <span class="k">return</span> <span class="n">rng</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="n">size</span><span class="p">,</span> <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3566,7 +3561,10 @@ <h2 id="pydvl.utils.numeric.random_matrix_with_condition_number" class="doc doc-
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/numeric.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-162">162</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-159">159</a></span>
+<span class="normal"><a href="#__codelineno-0-160">160</a></span>
+<span class="normal"><a href="#__codelineno-0-161">161</a></span>
+<span class="normal"><a href="#__codelineno-0-162">162</a></span>
 <span class="normal"><a href="#__codelineno-0-163">163</a></span>
 <span class="normal"><a href="#__codelineno-0-164">164</a></span>
 <span class="normal"><a href="#__codelineno-0-165">165</a></span>
@@ -3604,51 +3602,48 @@ <h2 id="pydvl.utils.numeric.random_matrix_with_condition_number" class="doc doc-
 <span class="normal"><a href="#__codelineno-0-197">197</a></span>
 <span class="normal"><a href="#__codelineno-0-198">198</a></span>
 <span class="normal"><a href="#__codelineno-0-199">199</a></span>
-<span class="normal"><a href="#__codelineno-0-200">200</a></span>
-<span class="normal"><a href="#__codelineno-0-201">201</a></span>
-<span class="normal"><a href="#__codelineno-0-202">202</a></span>
-<span class="normal"><a href="#__codelineno-0-203">203</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-162"><a id="__codelineno-0-162" name="__codelineno-0-162"></a><span class="k">def</span> <span class="nf">random_matrix_with_condition_number</span><span class="p">(</span>
-</span><span id="__span-0-163"><a id="__codelineno-0-163" name="__codelineno-0-163"></a>    <span class="n">n</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">condition_number</span><span class="p">:</span> <span class="nb">float</span><span class="p">,</span> <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
-</span><span id="__span-0-164"><a id="__codelineno-0-164" name="__codelineno-0-164"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">NDArray</span><span class="p">:</span>
-</span><span id="__span-0-165"><a id="__codelineno-0-165" name="__codelineno-0-165"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Constructs a square matrix with a given condition number.</span>
-</span><span id="__span-0-166"><a id="__codelineno-0-166" name="__codelineno-0-166"></a>
-</span><span id="__span-0-167"><a id="__codelineno-0-167" name="__codelineno-0-167"></a><span class="sd">    Taken from:</span>
-</span><span id="__span-0-168"><a id="__codelineno-0-168" name="__codelineno-0-168"></a><span class="sd">    [https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py](</span>
-</span><span id="__span-0-169"><a id="__codelineno-0-169" name="__codelineno-0-169"></a><span class="sd">    https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py)</span>
-</span><span id="__span-0-170"><a id="__codelineno-0-170" name="__codelineno-0-170"></a>
-</span><span id="__span-0-171"><a id="__codelineno-0-171" name="__codelineno-0-171"></a><span class="sd">    Also see:</span>
-</span><span id="__span-0-172"><a id="__codelineno-0-172" name="__codelineno-0-172"></a><span class="sd">    [https://math.stackexchange.com/questions/1351616/condition-number-of-ata](</span>
-</span><span id="__span-0-173"><a id="__codelineno-0-173" name="__codelineno-0-173"></a><span class="sd">    https://math.stackexchange.com/questions/1351616/condition-number-of-ata).</span>
-</span><span id="__span-0-174"><a id="__codelineno-0-174" name="__codelineno-0-174"></a>
-</span><span id="__span-0-175"><a id="__codelineno-0-175" name="__codelineno-0-175"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-176"><a id="__codelineno-0-176" name="__codelineno-0-176"></a><span class="sd">        n: size of the matrix</span>
-</span><span id="__span-0-177"><a id="__codelineno-0-177" name="__codelineno-0-177"></a><span class="sd">        condition_number: duh</span>
-</span><span id="__span-0-178"><a id="__codelineno-0-178" name="__codelineno-0-178"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
-</span><span id="__span-0-179"><a id="__codelineno-0-179" name="__codelineno-0-179"></a>
-</span><span id="__span-0-180"><a id="__codelineno-0-180" name="__codelineno-0-180"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-181"><a id="__codelineno-0-181" name="__codelineno-0-181"></a><span class="sd">        An (n,n) matrix with the requested condition number.</span>
-</span><span id="__span-0-182"><a id="__codelineno-0-182" name="__codelineno-0-182"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-183"><a id="__codelineno-0-183" name="__codelineno-0-183"></a>    <span class="k">if</span> <span class="n">n</span> <span class="o">&lt;</span> <span class="mi">2</span><span class="p">:</span>
-</span><span id="__span-0-184"><a id="__codelineno-0-184" name="__codelineno-0-184"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Matrix size must be at least 2&quot;</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-200">200</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-159"><a id="__codelineno-0-159" name="__codelineno-0-159"></a><span class="k">def</span> <span class="nf">random_matrix_with_condition_number</span><span class="p">(</span>
+</span><span id="__span-0-160"><a id="__codelineno-0-160" name="__codelineno-0-160"></a>    <span class="n">n</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">condition_number</span><span class="p">:</span> <span class="nb">float</span><span class="p">,</span> <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+</span><span id="__span-0-161"><a id="__codelineno-0-161" name="__codelineno-0-161"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">NDArray</span><span class="p">:</span>
+</span><span id="__span-0-162"><a id="__codelineno-0-162" name="__codelineno-0-162"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Constructs a square matrix with a given condition number.</span>
+</span><span id="__span-0-163"><a id="__codelineno-0-163" name="__codelineno-0-163"></a>
+</span><span id="__span-0-164"><a id="__codelineno-0-164" name="__codelineno-0-164"></a><span class="sd">    Taken from:</span>
+</span><span id="__span-0-165"><a id="__codelineno-0-165" name="__codelineno-0-165"></a><span class="sd">    [https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py](</span>
+</span><span id="__span-0-166"><a id="__codelineno-0-166" name="__codelineno-0-166"></a><span class="sd">    https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py)</span>
+</span><span id="__span-0-167"><a id="__codelineno-0-167" name="__codelineno-0-167"></a>
+</span><span id="__span-0-168"><a id="__codelineno-0-168" name="__codelineno-0-168"></a><span class="sd">    Also see:</span>
+</span><span id="__span-0-169"><a id="__codelineno-0-169" name="__codelineno-0-169"></a><span class="sd">    [https://math.stackexchange.com/questions/1351616/condition-number-of-ata](</span>
+</span><span id="__span-0-170"><a id="__codelineno-0-170" name="__codelineno-0-170"></a><span class="sd">    https://math.stackexchange.com/questions/1351616/condition-number-of-ata).</span>
+</span><span id="__span-0-171"><a id="__codelineno-0-171" name="__codelineno-0-171"></a>
+</span><span id="__span-0-172"><a id="__codelineno-0-172" name="__codelineno-0-172"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-173"><a id="__codelineno-0-173" name="__codelineno-0-173"></a><span class="sd">        n: size of the matrix</span>
+</span><span id="__span-0-174"><a id="__codelineno-0-174" name="__codelineno-0-174"></a><span class="sd">        condition_number: duh</span>
+</span><span id="__span-0-175"><a id="__codelineno-0-175" name="__codelineno-0-175"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
+</span><span id="__span-0-176"><a id="__codelineno-0-176" name="__codelineno-0-176"></a>
+</span><span id="__span-0-177"><a id="__codelineno-0-177" name="__codelineno-0-177"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-178"><a id="__codelineno-0-178" name="__codelineno-0-178"></a><span class="sd">        An (n,n) matrix with the requested condition number.</span>
+</span><span id="__span-0-179"><a id="__codelineno-0-179" name="__codelineno-0-179"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-180"><a id="__codelineno-0-180" name="__codelineno-0-180"></a>    <span class="k">if</span> <span class="n">n</span> <span class="o">&lt;</span> <span class="mi">2</span><span class="p">:</span>
+</span><span id="__span-0-181"><a id="__codelineno-0-181" name="__codelineno-0-181"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Matrix size must be at least 2&quot;</span><span class="p">)</span>
+</span><span id="__span-0-182"><a id="__codelineno-0-182" name="__codelineno-0-182"></a>
+</span><span id="__span-0-183"><a id="__codelineno-0-183" name="__codelineno-0-183"></a>    <span class="k">if</span> <span class="n">condition_number</span> <span class="o">&lt;=</span> <span class="mi">1</span><span class="p">:</span>
+</span><span id="__span-0-184"><a id="__codelineno-0-184" name="__codelineno-0-184"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Condition number must be greater than 1&quot;</span><span class="p">)</span>
 </span><span id="__span-0-185"><a id="__codelineno-0-185" name="__codelineno-0-185"></a>
-</span><span id="__span-0-186"><a id="__codelineno-0-186" name="__codelineno-0-186"></a>    <span class="k">if</span> <span class="n">condition_number</span> <span class="o">&lt;=</span> <span class="mi">1</span><span class="p">:</span>
-</span><span id="__span-0-187"><a id="__codelineno-0-187" name="__codelineno-0-187"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Condition number must be greater than 1&quot;</span><span class="p">)</span>
-</span><span id="__span-0-188"><a id="__codelineno-0-188" name="__codelineno-0-188"></a>
-</span><span id="__span-0-189"><a id="__codelineno-0-189" name="__codelineno-0-189"></a>    <span class="n">rng</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">default_rng</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
-</span><span id="__span-0-190"><a id="__codelineno-0-190" name="__codelineno-0-190"></a>    <span class="n">log_condition_number</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="n">condition_number</span><span class="p">)</span>
-</span><span id="__span-0-191"><a id="__codelineno-0-191" name="__codelineno-0-191"></a>    <span class="n">exp_vec</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span>
-</span><span id="__span-0-192"><a id="__codelineno-0-192" name="__codelineno-0-192"></a>        <span class="o">-</span><span class="n">log_condition_number</span> <span class="o">/</span> <span class="mf">4.0</span><span class="p">,</span>
-</span><span id="__span-0-193"><a id="__codelineno-0-193" name="__codelineno-0-193"></a>        <span class="n">log_condition_number</span> <span class="o">*</span> <span class="p">(</span><span class="n">n</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="mi">4</span> <span class="o">*</span> <span class="p">(</span><span class="n">n</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)),</span>
-</span><span id="__span-0-194"><a id="__codelineno-0-194" name="__codelineno-0-194"></a>        <span class="n">log_condition_number</span> <span class="o">/</span> <span class="p">(</span><span class="mf">2.0</span> <span class="o">*</span> <span class="p">(</span><span class="n">n</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)),</span>
-</span><span id="__span-0-195"><a id="__codelineno-0-195" name="__codelineno-0-195"></a>    <span class="p">)</span>
-</span><span id="__span-0-196"><a id="__codelineno-0-196" name="__codelineno-0-196"></a>    <span class="n">exp_vec</span> <span class="o">=</span> <span class="n">exp_vec</span><span class="p">[:</span><span class="n">n</span><span class="p">]</span>
-</span><span id="__span-0-197"><a id="__codelineno-0-197" name="__codelineno-0-197"></a>    <span class="n">s</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="n">exp_vec</span><span class="p">)</span>
-</span><span id="__span-0-198"><a id="__codelineno-0-198" name="__codelineno-0-198"></a>    <span class="n">S</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">diag</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
-</span><span id="__span-0-199"><a id="__codelineno-0-199" name="__codelineno-0-199"></a>    <span class="n">U</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">qr</span><span class="p">((</span><span class="n">rng</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">n</span><span class="p">))</span> <span class="o">-</span> <span class="mf">5.0</span><span class="p">)</span> <span class="o">*</span> <span class="mi">200</span><span class="p">)</span>
-</span><span id="__span-0-200"><a id="__codelineno-0-200" name="__codelineno-0-200"></a>    <span class="n">V</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">qr</span><span class="p">((</span><span class="n">rng</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">n</span><span class="p">))</span> <span class="o">-</span> <span class="mf">5.0</span><span class="p">)</span> <span class="o">*</span> <span class="mi">200</span><span class="p">)</span>
-</span><span id="__span-0-201"><a id="__codelineno-0-201" name="__codelineno-0-201"></a>    <span class="n">P</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">=</span> <span class="n">U</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">S</span><span class="p">)</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">V</span><span class="o">.</span><span class="n">T</span><span class="p">)</span>
-</span><span id="__span-0-202"><a id="__codelineno-0-202" name="__codelineno-0-202"></a>    <span class="n">P</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">P</span><span class="o">.</span><span class="n">T</span><span class="p">)</span>
-</span><span id="__span-0-203"><a id="__codelineno-0-203" name="__codelineno-0-203"></a>    <span class="k">return</span> <span class="n">P</span>
+</span><span id="__span-0-186"><a id="__codelineno-0-186" name="__codelineno-0-186"></a>    <span class="n">rng</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">default_rng</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
+</span><span id="__span-0-187"><a id="__codelineno-0-187" name="__codelineno-0-187"></a>    <span class="n">log_condition_number</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="n">condition_number</span><span class="p">)</span>
+</span><span id="__span-0-188"><a id="__codelineno-0-188" name="__codelineno-0-188"></a>    <span class="n">exp_vec</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span>
+</span><span id="__span-0-189"><a id="__codelineno-0-189" name="__codelineno-0-189"></a>        <span class="o">-</span><span class="n">log_condition_number</span> <span class="o">/</span> <span class="mf">4.0</span><span class="p">,</span>
+</span><span id="__span-0-190"><a id="__codelineno-0-190" name="__codelineno-0-190"></a>        <span class="n">log_condition_number</span> <span class="o">*</span> <span class="p">(</span><span class="n">n</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="mi">4</span> <span class="o">*</span> <span class="p">(</span><span class="n">n</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)),</span>
+</span><span id="__span-0-191"><a id="__codelineno-0-191" name="__codelineno-0-191"></a>        <span class="n">log_condition_number</span> <span class="o">/</span> <span class="p">(</span><span class="mf">2.0</span> <span class="o">*</span> <span class="p">(</span><span class="n">n</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)),</span>
+</span><span id="__span-0-192"><a id="__codelineno-0-192" name="__codelineno-0-192"></a>    <span class="p">)</span>
+</span><span id="__span-0-193"><a id="__codelineno-0-193" name="__codelineno-0-193"></a>    <span class="n">exp_vec</span> <span class="o">=</span> <span class="n">exp_vec</span><span class="p">[:</span><span class="n">n</span><span class="p">]</span>
+</span><span id="__span-0-194"><a id="__codelineno-0-194" name="__codelineno-0-194"></a>    <span class="n">s</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="n">exp_vec</span><span class="p">)</span>
+</span><span id="__span-0-195"><a id="__codelineno-0-195" name="__codelineno-0-195"></a>    <span class="n">S</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">diag</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
+</span><span id="__span-0-196"><a id="__codelineno-0-196" name="__codelineno-0-196"></a>    <span class="n">U</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">qr</span><span class="p">((</span><span class="n">rng</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">n</span><span class="p">))</span> <span class="o">-</span> <span class="mf">5.0</span><span class="p">)</span> <span class="o">*</span> <span class="mi">200</span><span class="p">)</span>
+</span><span id="__span-0-197"><a id="__codelineno-0-197" name="__codelineno-0-197"></a>    <span class="n">V</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">qr</span><span class="p">((</span><span class="n">rng</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">n</span><span class="p">))</span> <span class="o">-</span> <span class="mf">5.0</span><span class="p">)</span> <span class="o">*</span> <span class="mi">200</span><span class="p">)</span>
+</span><span id="__span-0-198"><a id="__codelineno-0-198" name="__codelineno-0-198"></a>    <span class="n">P</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span> <span class="o">=</span> <span class="n">U</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">S</span><span class="p">)</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">V</span><span class="o">.</span><span class="n">T</span><span class="p">)</span>
+</span><span id="__span-0-199"><a id="__codelineno-0-199" name="__codelineno-0-199"></a>    <span class="n">P</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">P</span><span class="o">.</span><span class="n">T</span><span class="p">)</span>
+</span><span id="__span-0-200"><a id="__codelineno-0-200" name="__codelineno-0-200"></a>    <span class="k">return</span> <span class="n">P</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3778,7 +3773,10 @@ <h2 id="pydvl.utils.numeric.running_moments" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/numeric.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-223">223</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-220">220</a></span>
+<span class="normal"><a href="#__codelineno-0-221">221</a></span>
+<span class="normal"><a href="#__codelineno-0-222">222</a></span>
+<span class="normal"><a href="#__codelineno-0-223">223</a></span>
 <span class="normal"><a href="#__codelineno-0-224">224</a></span>
 <span class="normal"><a href="#__codelineno-0-225">225</a></span>
 <span class="normal"><a href="#__codelineno-0-226">226</a></span>
@@ -3807,42 +3805,39 @@ <h2 id="pydvl.utils.numeric.running_moments" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-249">249</a></span>
 <span class="normal"><a href="#__codelineno-0-250">250</a></span>
 <span class="normal"><a href="#__codelineno-0-251">251</a></span>
-<span class="normal"><a href="#__codelineno-0-252">252</a></span>
-<span class="normal"><a href="#__codelineno-0-253">253</a></span>
-<span class="normal"><a href="#__codelineno-0-254">254</a></span>
-<span class="normal"><a href="#__codelineno-0-255">255</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-223"><a id="__codelineno-0-223" name="__codelineno-0-223"></a><span class="k">def</span> <span class="nf">running_moments</span><span class="p">(</span>
-</span><span id="__span-0-224"><a id="__codelineno-0-224" name="__codelineno-0-224"></a>    <span class="n">previous_avg</span><span class="p">:</span> <span class="nb">float</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">],</span>
-</span><span id="__span-0-225"><a id="__codelineno-0-225" name="__codelineno-0-225"></a>    <span class="n">previous_variance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">],</span>
-</span><span id="__span-0-226"><a id="__codelineno-0-226" name="__codelineno-0-226"></a>    <span class="n">count</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-</span><span id="__span-0-227"><a id="__codelineno-0-227" name="__codelineno-0-227"></a>    <span class="n">new_value</span><span class="p">:</span> <span class="nb">float</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">],</span>
-</span><span id="__span-0-228"><a id="__codelineno-0-228" name="__codelineno-0-228"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">float</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">],</span> <span class="nb">float</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">]]:</span>
-</span><span id="__span-0-229"><a id="__codelineno-0-229" name="__codelineno-0-229"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Uses Welford&#39;s algorithm to calculate the running average and variance of</span>
-</span><span id="__span-0-230"><a id="__codelineno-0-230" name="__codelineno-0-230"></a><span class="sd">     a set of numbers.</span>
-</span><span id="__span-0-231"><a id="__codelineno-0-231" name="__codelineno-0-231"></a>
-</span><span id="__span-0-232"><a id="__codelineno-0-232" name="__codelineno-0-232"></a><span class="sd">    See [Welford&#39;s algorithm in wikipedia](https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford&#39;s_online_algorithm)</span>
-</span><span id="__span-0-233"><a id="__codelineno-0-233" name="__codelineno-0-233"></a>
-</span><span id="__span-0-234"><a id="__codelineno-0-234" name="__codelineno-0-234"></a><span class="sd">    !!! Warning</span>
-</span><span id="__span-0-235"><a id="__codelineno-0-235" name="__codelineno-0-235"></a><span class="sd">        This is not really using Welford&#39;s correction for numerical stability</span>
-</span><span id="__span-0-236"><a id="__codelineno-0-236" name="__codelineno-0-236"></a><span class="sd">        for the variance. (FIXME)</span>
+<span class="normal"><a href="#__codelineno-0-252">252</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-220"><a id="__codelineno-0-220" name="__codelineno-0-220"></a><span class="k">def</span> <span class="nf">running_moments</span><span class="p">(</span>
+</span><span id="__span-0-221"><a id="__codelineno-0-221" name="__codelineno-0-221"></a>    <span class="n">previous_avg</span><span class="p">:</span> <span class="nb">float</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">],</span>
+</span><span id="__span-0-222"><a id="__codelineno-0-222" name="__codelineno-0-222"></a>    <span class="n">previous_variance</span><span class="p">:</span> <span class="nb">float</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">],</span>
+</span><span id="__span-0-223"><a id="__codelineno-0-223" name="__codelineno-0-223"></a>    <span class="n">count</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+</span><span id="__span-0-224"><a id="__codelineno-0-224" name="__codelineno-0-224"></a>    <span class="n">new_value</span><span class="p">:</span> <span class="nb">float</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">],</span>
+</span><span id="__span-0-225"><a id="__codelineno-0-225" name="__codelineno-0-225"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">float</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">],</span> <span class="nb">float</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">]]:</span>
+</span><span id="__span-0-226"><a id="__codelineno-0-226" name="__codelineno-0-226"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Uses Welford&#39;s algorithm to calculate the running average and variance of</span>
+</span><span id="__span-0-227"><a id="__codelineno-0-227" name="__codelineno-0-227"></a><span class="sd">     a set of numbers.</span>
+</span><span id="__span-0-228"><a id="__codelineno-0-228" name="__codelineno-0-228"></a>
+</span><span id="__span-0-229"><a id="__codelineno-0-229" name="__codelineno-0-229"></a><span class="sd">    See [Welford&#39;s algorithm in wikipedia](https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford&#39;s_online_algorithm)</span>
+</span><span id="__span-0-230"><a id="__codelineno-0-230" name="__codelineno-0-230"></a>
+</span><span id="__span-0-231"><a id="__codelineno-0-231" name="__codelineno-0-231"></a><span class="sd">    !!! Warning</span>
+</span><span id="__span-0-232"><a id="__codelineno-0-232" name="__codelineno-0-232"></a><span class="sd">        This is not really using Welford&#39;s correction for numerical stability</span>
+</span><span id="__span-0-233"><a id="__codelineno-0-233" name="__codelineno-0-233"></a><span class="sd">        for the variance. (FIXME)</span>
+</span><span id="__span-0-234"><a id="__codelineno-0-234" name="__codelineno-0-234"></a>
+</span><span id="__span-0-235"><a id="__codelineno-0-235" name="__codelineno-0-235"></a><span class="sd">    !!! Todo</span>
+</span><span id="__span-0-236"><a id="__codelineno-0-236" name="__codelineno-0-236"></a><span class="sd">        This could be generalised to arbitrary moments. See [this paper](https://www.osti.gov/biblio/1028931)</span>
 </span><span id="__span-0-237"><a id="__codelineno-0-237" name="__codelineno-0-237"></a>
-</span><span id="__span-0-238"><a id="__codelineno-0-238" name="__codelineno-0-238"></a><span class="sd">    !!! Todo</span>
-</span><span id="__span-0-239"><a id="__codelineno-0-239" name="__codelineno-0-239"></a><span class="sd">        This could be generalised to arbitrary moments. See [this paper](https://www.osti.gov/biblio/1028931)</span>
-</span><span id="__span-0-240"><a id="__codelineno-0-240" name="__codelineno-0-240"></a>
-</span><span id="__span-0-241"><a id="__codelineno-0-241" name="__codelineno-0-241"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-242"><a id="__codelineno-0-242" name="__codelineno-0-242"></a><span class="sd">        previous_avg: average value at previous step</span>
-</span><span id="__span-0-243"><a id="__codelineno-0-243" name="__codelineno-0-243"></a><span class="sd">        previous_variance: variance at previous step</span>
-</span><span id="__span-0-244"><a id="__codelineno-0-244" name="__codelineno-0-244"></a><span class="sd">        count: number of points seen so far</span>
-</span><span id="__span-0-245"><a id="__codelineno-0-245" name="__codelineno-0-245"></a><span class="sd">        new_value: new value in the series of numbers</span>
-</span><span id="__span-0-246"><a id="__codelineno-0-246" name="__codelineno-0-246"></a>
-</span><span id="__span-0-247"><a id="__codelineno-0-247" name="__codelineno-0-247"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-248"><a id="__codelineno-0-248" name="__codelineno-0-248"></a><span class="sd">        new_average, new_variance, calculated with the new count</span>
-</span><span id="__span-0-249"><a id="__codelineno-0-249" name="__codelineno-0-249"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-250"><a id="__codelineno-0-250" name="__codelineno-0-250"></a>    <span class="c1"># broadcasted operations seem not to be supported by mypy, so we ignore the type</span>
-</span><span id="__span-0-251"><a id="__codelineno-0-251" name="__codelineno-0-251"></a>    <span class="n">new_average</span> <span class="o">=</span> <span class="p">(</span><span class="n">new_value</span> <span class="o">+</span> <span class="n">count</span> <span class="o">*</span> <span class="n">previous_avg</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="n">count</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
-</span><span id="__span-0-252"><a id="__codelineno-0-252" name="__codelineno-0-252"></a>    <span class="n">new_variance</span> <span class="o">=</span> <span class="n">previous_variance</span> <span class="o">+</span> <span class="p">(</span>
-</span><span id="__span-0-253"><a id="__codelineno-0-253" name="__codelineno-0-253"></a>        <span class="p">(</span><span class="n">new_value</span> <span class="o">-</span> <span class="n">previous_avg</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="n">new_value</span> <span class="o">-</span> <span class="n">new_average</span><span class="p">)</span> <span class="o">-</span> <span class="n">previous_variance</span>
-</span><span id="__span-0-254"><a id="__codelineno-0-254" name="__codelineno-0-254"></a>    <span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="n">count</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
-</span><span id="__span-0-255"><a id="__codelineno-0-255" name="__codelineno-0-255"></a>    <span class="k">return</span> <span class="n">new_average</span><span class="p">,</span> <span class="n">new_variance</span>
+</span><span id="__span-0-238"><a id="__codelineno-0-238" name="__codelineno-0-238"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-239"><a id="__codelineno-0-239" name="__codelineno-0-239"></a><span class="sd">        previous_avg: average value at previous step</span>
+</span><span id="__span-0-240"><a id="__codelineno-0-240" name="__codelineno-0-240"></a><span class="sd">        previous_variance: variance at previous step</span>
+</span><span id="__span-0-241"><a id="__codelineno-0-241" name="__codelineno-0-241"></a><span class="sd">        count: number of points seen so far</span>
+</span><span id="__span-0-242"><a id="__codelineno-0-242" name="__codelineno-0-242"></a><span class="sd">        new_value: new value in the series of numbers</span>
+</span><span id="__span-0-243"><a id="__codelineno-0-243" name="__codelineno-0-243"></a>
+</span><span id="__span-0-244"><a id="__codelineno-0-244" name="__codelineno-0-244"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-245"><a id="__codelineno-0-245" name="__codelineno-0-245"></a><span class="sd">        new_average, new_variance, calculated with the new count</span>
+</span><span id="__span-0-246"><a id="__codelineno-0-246" name="__codelineno-0-246"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-247"><a id="__codelineno-0-247" name="__codelineno-0-247"></a>    <span class="c1"># broadcasted operations seem not to be supported by mypy, so we ignore the type</span>
+</span><span id="__span-0-248"><a id="__codelineno-0-248" name="__codelineno-0-248"></a>    <span class="n">new_average</span> <span class="o">=</span> <span class="p">(</span><span class="n">new_value</span> <span class="o">+</span> <span class="n">count</span> <span class="o">*</span> <span class="n">previous_avg</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="n">count</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
+</span><span id="__span-0-249"><a id="__codelineno-0-249" name="__codelineno-0-249"></a>    <span class="n">new_variance</span> <span class="o">=</span> <span class="n">previous_variance</span> <span class="o">+</span> <span class="p">(</span>
+</span><span id="__span-0-250"><a id="__codelineno-0-250" name="__codelineno-0-250"></a>        <span class="p">(</span><span class="n">new_value</span> <span class="o">-</span> <span class="n">previous_avg</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="n">new_value</span> <span class="o">-</span> <span class="n">new_average</span><span class="p">)</span> <span class="o">-</span> <span class="n">previous_variance</span>
+</span><span id="__span-0-251"><a id="__codelineno-0-251" name="__codelineno-0-251"></a>    <span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="n">count</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
+</span><span id="__span-0-252"><a id="__codelineno-0-252" name="__codelineno-0-252"></a>    <span class="k">return</span> <span class="n">new_average</span><span class="p">,</span> <span class="n">new_variance</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3952,7 +3947,10 @@ <h2 id="pydvl.utils.numeric.top_k_value_accuracy" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/numeric.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-258">258</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-255">255</a></span>
+<span class="normal"><a href="#__codelineno-0-256">256</a></span>
+<span class="normal"><a href="#__codelineno-0-257">257</a></span>
+<span class="normal"><a href="#__codelineno-0-258">258</a></span>
 <span class="normal"><a href="#__codelineno-0-259">259</a></span>
 <span class="normal"><a href="#__codelineno-0-260">260</a></span>
 <span class="normal"><a href="#__codelineno-0-261">261</a></span>
@@ -3966,27 +3964,24 @@ <h2 id="pydvl.utils.numeric.top_k_value_accuracy" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-269">269</a></span>
 <span class="normal"><a href="#__codelineno-0-270">270</a></span>
 <span class="normal"><a href="#__codelineno-0-271">271</a></span>
-<span class="normal"><a href="#__codelineno-0-272">272</a></span>
-<span class="normal"><a href="#__codelineno-0-273">273</a></span>
-<span class="normal"><a href="#__codelineno-0-274">274</a></span>
-<span class="normal"><a href="#__codelineno-0-275">275</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-258"><a id="__codelineno-0-258" name="__codelineno-0-258"></a><span class="k">def</span> <span class="nf">top_k_value_accuracy</span><span class="p">(</span>
-</span><span id="__span-0-259"><a id="__codelineno-0-259" name="__codelineno-0-259"></a>    <span class="n">y_true</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">],</span> <span class="n">y_pred</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">],</span> <span class="n">k</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span>
-</span><span id="__span-0-260"><a id="__codelineno-0-260" name="__codelineno-0-260"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
-</span><span id="__span-0-261"><a id="__codelineno-0-261" name="__codelineno-0-261"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Computes the top-k accuracy for the estimated values by comparing indices</span>
-</span><span id="__span-0-262"><a id="__codelineno-0-262" name="__codelineno-0-262"></a><span class="sd">    of the highest k values.</span>
-</span><span id="__span-0-263"><a id="__codelineno-0-263" name="__codelineno-0-263"></a>
-</span><span id="__span-0-264"><a id="__codelineno-0-264" name="__codelineno-0-264"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-265"><a id="__codelineno-0-265" name="__codelineno-0-265"></a><span class="sd">        y_true: Exact/true value</span>
-</span><span id="__span-0-266"><a id="__codelineno-0-266" name="__codelineno-0-266"></a><span class="sd">        y_pred: Predicted/estimated value</span>
-</span><span id="__span-0-267"><a id="__codelineno-0-267" name="__codelineno-0-267"></a><span class="sd">        k: Number of the highest values taken into account</span>
-</span><span id="__span-0-268"><a id="__codelineno-0-268" name="__codelineno-0-268"></a>
-</span><span id="__span-0-269"><a id="__codelineno-0-269" name="__codelineno-0-269"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-270"><a id="__codelineno-0-270" name="__codelineno-0-270"></a><span class="sd">        Accuracy</span>
-</span><span id="__span-0-271"><a id="__codelineno-0-271" name="__codelineno-0-271"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-272"><a id="__codelineno-0-272" name="__codelineno-0-272"></a>    <span class="n">top_k_exact_values</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">argsort</span><span class="p">(</span><span class="n">y_true</span><span class="p">)[</span><span class="o">-</span><span class="n">k</span><span class="p">:]</span>
-</span><span id="__span-0-273"><a id="__codelineno-0-273" name="__codelineno-0-273"></a>    <span class="n">top_k_pred_values</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">argsort</span><span class="p">(</span><span class="n">y_pred</span><span class="p">)[</span><span class="o">-</span><span class="n">k</span><span class="p">:]</span>
-</span><span id="__span-0-274"><a id="__codelineno-0-274" name="__codelineno-0-274"></a>    <span class="n">top_k_accuracy</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">intersect1d</span><span class="p">(</span><span class="n">top_k_exact_values</span><span class="p">,</span> <span class="n">top_k_pred_values</span><span class="p">))</span> <span class="o">/</span> <span class="n">k</span>
-</span><span id="__span-0-275"><a id="__codelineno-0-275" name="__codelineno-0-275"></a>    <span class="k">return</span> <span class="n">top_k_accuracy</span>
+<span class="normal"><a href="#__codelineno-0-272">272</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-255"><a id="__codelineno-0-255" name="__codelineno-0-255"></a><span class="k">def</span> <span class="nf">top_k_value_accuracy</span><span class="p">(</span>
+</span><span id="__span-0-256"><a id="__codelineno-0-256" name="__codelineno-0-256"></a>    <span class="n">y_true</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">],</span> <span class="n">y_pred</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">],</span> <span class="n">k</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span>
+</span><span id="__span-0-257"><a id="__codelineno-0-257" name="__codelineno-0-257"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
+</span><span id="__span-0-258"><a id="__codelineno-0-258" name="__codelineno-0-258"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Computes the top-k accuracy for the estimated values by comparing indices</span>
+</span><span id="__span-0-259"><a id="__codelineno-0-259" name="__codelineno-0-259"></a><span class="sd">    of the highest k values.</span>
+</span><span id="__span-0-260"><a id="__codelineno-0-260" name="__codelineno-0-260"></a>
+</span><span id="__span-0-261"><a id="__codelineno-0-261" name="__codelineno-0-261"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-262"><a id="__codelineno-0-262" name="__codelineno-0-262"></a><span class="sd">        y_true: Exact/true value</span>
+</span><span id="__span-0-263"><a id="__codelineno-0-263" name="__codelineno-0-263"></a><span class="sd">        y_pred: Predicted/estimated value</span>
+</span><span id="__span-0-264"><a id="__codelineno-0-264" name="__codelineno-0-264"></a><span class="sd">        k: Number of the highest values taken into account</span>
+</span><span id="__span-0-265"><a id="__codelineno-0-265" name="__codelineno-0-265"></a>
+</span><span id="__span-0-266"><a id="__codelineno-0-266" name="__codelineno-0-266"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-267"><a id="__codelineno-0-267" name="__codelineno-0-267"></a><span class="sd">        Accuracy</span>
+</span><span id="__span-0-268"><a id="__codelineno-0-268" name="__codelineno-0-268"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-269"><a id="__codelineno-0-269" name="__codelineno-0-269"></a>    <span class="n">top_k_exact_values</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">argsort</span><span class="p">(</span><span class="n">y_true</span><span class="p">)[</span><span class="o">-</span><span class="n">k</span><span class="p">:]</span>
+</span><span id="__span-0-270"><a id="__codelineno-0-270" name="__codelineno-0-270"></a>    <span class="n">top_k_pred_values</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">argsort</span><span class="p">(</span><span class="n">y_pred</span><span class="p">)[</span><span class="o">-</span><span class="n">k</span><span class="p">:]</span>
+</span><span id="__span-0-271"><a id="__codelineno-0-271" name="__codelineno-0-271"></a>    <span class="n">top_k_accuracy</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">intersect1d</span><span class="p">(</span><span class="n">top_k_exact_values</span><span class="p">,</span> <span class="n">top_k_pred_values</span><span class="p">))</span> <span class="o">/</span> <span class="n">k</span>
+</span><span id="__span-0-272"><a id="__codelineno-0-272" name="__codelineno-0-272"></a>    <span class="k">return</span> <span class="n">top_k_accuracy</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -4006,11 +4001,11 @@ <h2 id="pydvl.utils.numeric.top_k_value_accuracy" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/utils/parallel/index.html b/devel/api/pydvl/utils/parallel/index.html
index 8898c9039..770f81a2e 100644
--- a/devel/api/pydvl/utils/parallel/index.html
+++ b/devel/api/pydvl/utils/parallel/index.html
@@ -2556,11 +2556,11 @@ <h2 id="pydvl.utils.parallel--this-module-is-deprecated">This module is deprecat
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/utils/progress/index.html b/devel/api/pydvl/utils/progress/index.html
index c7ec3566f..686dcfb4f 100644
--- a/devel/api/pydvl/utils/progress/index.html
+++ b/devel/api/pydvl/utils/progress/index.html
@@ -2737,11 +2737,11 @@ <h2 id="pydvl.utils.progress.maybe_progress" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/utils/score/index.html b/devel/api/pydvl/utils/score/index.html
index 5019e15ba..9053e993f 100644
--- a/devel/api/pydvl/utils/score/index.html
+++ b/devel/api/pydvl/utils/score/index.html
@@ -2581,6 +2581,9 @@ <h1>Score</h1>
   
       <p>This module provides a <a class="autorefs autorefs-internal" href="#pydvl.utils.score.Scorer">Scorer</a> class that wraps
 scoring functions with additional information.</p>
+<p>Scorers are the fundamental building block of many data valuation methods. They
+are typically used by the <a class="autorefs autorefs-internal" href="../utility/#pydvl.utils.utility.Utility">Utility</a> class to
+evaluate the quality of a model when trained on subsets of the training data.</p>
 <p>Scorers can be constructed in the same way as in scikit-learn: either from 
 known strings or from a callable. Greater values must be better. If they are not,
 a negated version can be used, see scikit-learn's
@@ -2797,11 +2800,7 @@ <h2 id="pydvl.utils.score.Scorer" class="doc doc-heading">
 
               <details class="quote">
                 <summary>Source code in <code>src/pydvl/utils/score.py</code></summary>
-                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-59">59</a></span>
-<span class="normal"><a href="#__codelineno-0-60">60</a></span>
-<span class="normal"><a href="#__codelineno-0-61">61</a></span>
-<span class="normal"><a href="#__codelineno-0-62">62</a></span>
-<span class="normal"><a href="#__codelineno-0-63">63</a></span>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-63">63</a></span>
 <span class="normal"><a href="#__codelineno-0-64">64</a></span>
 <span class="normal"><a href="#__codelineno-0-65">65</a></span>
 <span class="normal"><a href="#__codelineno-0-66">66</a></span>
@@ -2810,20 +2809,24 @@ <h2 id="pydvl.utils.score.Scorer" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-69">69</a></span>
 <span class="normal"><a href="#__codelineno-0-70">70</a></span>
 <span class="normal"><a href="#__codelineno-0-71">71</a></span>
-<span class="normal"><a href="#__codelineno-0-72">72</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-59"><a id="__codelineno-0-59" name="__codelineno-0-59"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-</span><span id="__span-0-60"><a id="__codelineno-0-60" name="__codelineno-0-60"></a>    <span class="bp">self</span><span class="p">,</span>
-</span><span id="__span-0-61"><a id="__codelineno-0-61" name="__codelineno-0-61"></a>    <span class="n">scoring</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">ScorerCallable</span><span class="p">],</span>
-</span><span id="__span-0-62"><a id="__codelineno-0-62" name="__codelineno-0-62"></a>    <span class="n">default</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">nan</span><span class="p">,</span>
-</span><span id="__span-0-63"><a id="__codelineno-0-63" name="__codelineno-0-63"></a>    <span class="nb">range</span><span class="p">:</span> <span class="n">Tuple</span> <span class="o">=</span> <span class="p">(</span><span class="o">-</span><span class="n">np</span><span class="o">.</span><span class="n">inf</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">inf</span><span class="p">),</span>
-</span><span id="__span-0-64"><a id="__codelineno-0-64" name="__codelineno-0-64"></a>    <span class="n">name</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-65"><a id="__codelineno-0-65" name="__codelineno-0-65"></a><span class="p">):</span>
-</span><span id="__span-0-66"><a id="__codelineno-0-66" name="__codelineno-0-66"></a>    <span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">scoring</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
-</span><span id="__span-0-67"><a id="__codelineno-0-67" name="__codelineno-0-67"></a>        <span class="n">name</span> <span class="o">=</span> <span class="n">scoring</span>
-</span><span id="__span-0-68"><a id="__codelineno-0-68" name="__codelineno-0-68"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_scorer</span> <span class="o">=</span> <span class="n">get_scorer</span><span class="p">(</span><span class="n">scoring</span><span class="p">)</span>
-</span><span id="__span-0-69"><a id="__codelineno-0-69" name="__codelineno-0-69"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">default</span> <span class="o">=</span> <span class="n">default</span>
-</span><span id="__span-0-70"><a id="__codelineno-0-70" name="__codelineno-0-70"></a>    <span class="c1"># TODO: auto-fill from known scorers ?</span>
-</span><span id="__span-0-71"><a id="__codelineno-0-71" name="__codelineno-0-71"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">range</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="nb">range</span><span class="p">)</span>
-</span><span id="__span-0-72"><a id="__codelineno-0-72" name="__codelineno-0-72"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_scorer</span><span class="p">,</span> <span class="s2">&quot;__name__&quot;</span><span class="p">,</span> <span class="n">name</span> <span class="ow">or</span> <span class="s2">&quot;scorer&quot;</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-72">72</a></span>
+<span class="normal"><a href="#__codelineno-0-73">73</a></span>
+<span class="normal"><a href="#__codelineno-0-74">74</a></span>
+<span class="normal"><a href="#__codelineno-0-75">75</a></span>
+<span class="normal"><a href="#__codelineno-0-76">76</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-63"><a id="__codelineno-0-63" name="__codelineno-0-63"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+</span><span id="__span-0-64"><a id="__codelineno-0-64" name="__codelineno-0-64"></a>    <span class="bp">self</span><span class="p">,</span>
+</span><span id="__span-0-65"><a id="__codelineno-0-65" name="__codelineno-0-65"></a>    <span class="n">scoring</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">ScorerCallable</span><span class="p">],</span>
+</span><span id="__span-0-66"><a id="__codelineno-0-66" name="__codelineno-0-66"></a>    <span class="n">default</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">nan</span><span class="p">,</span>
+</span><span id="__span-0-67"><a id="__codelineno-0-67" name="__codelineno-0-67"></a>    <span class="nb">range</span><span class="p">:</span> <span class="n">Tuple</span> <span class="o">=</span> <span class="p">(</span><span class="o">-</span><span class="n">np</span><span class="o">.</span><span class="n">inf</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">inf</span><span class="p">),</span>
+</span><span id="__span-0-68"><a id="__codelineno-0-68" name="__codelineno-0-68"></a>    <span class="n">name</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-69"><a id="__codelineno-0-69" name="__codelineno-0-69"></a><span class="p">):</span>
+</span><span id="__span-0-70"><a id="__codelineno-0-70" name="__codelineno-0-70"></a>    <span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">scoring</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
+</span><span id="__span-0-71"><a id="__codelineno-0-71" name="__codelineno-0-71"></a>        <span class="n">name</span> <span class="o">=</span> <span class="n">scoring</span>
+</span><span id="__span-0-72"><a id="__codelineno-0-72" name="__codelineno-0-72"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_scorer</span> <span class="o">=</span> <span class="n">get_scorer</span><span class="p">(</span><span class="n">scoring</span><span class="p">)</span>
+</span><span id="__span-0-73"><a id="__codelineno-0-73" name="__codelineno-0-73"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">default</span> <span class="o">=</span> <span class="n">default</span>
+</span><span id="__span-0-74"><a id="__codelineno-0-74" name="__codelineno-0-74"></a>    <span class="c1"># TODO: auto-fill from known scorers ?</span>
+</span><span id="__span-0-75"><a id="__codelineno-0-75" name="__codelineno-0-75"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">range</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="nb">range</span><span class="p">)</span>
+</span><span id="__span-0-76"><a id="__codelineno-0-76" name="__codelineno-0-76"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_scorer</span><span class="p">,</span> <span class="s2">&quot;__name__&quot;</span><span class="p">,</span> <span class="n">name</span> <span class="ow">or</span> <span class="s2">&quot;scorer&quot;</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
               </details>
 
@@ -2967,11 +2970,7 @@ <h2 id="pydvl.utils.score.compose_score" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/score.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-85"> 85</a></span>
-<span class="normal"><a href="#__codelineno-0-86"> 86</a></span>
-<span class="normal"><a href="#__codelineno-0-87"> 87</a></span>
-<span class="normal"><a href="#__codelineno-0-88"> 88</a></span>
-<span class="normal"><a href="#__codelineno-0-89"> 89</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-89"> 89</a></span>
 <span class="normal"><a href="#__codelineno-0-90"> 90</a></span>
 <span class="normal"><a href="#__codelineno-0-91"> 91</a></span>
 <span class="normal"><a href="#__codelineno-0-92"> 92</a></span>
@@ -3001,41 +3000,45 @@ <h2 id="pydvl.utils.score.compose_score" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-116">116</a></span>
 <span class="normal"><a href="#__codelineno-0-117">117</a></span>
 <span class="normal"><a href="#__codelineno-0-118">118</a></span>
-<span class="normal"><a href="#__codelineno-0-119">119</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-85"><a id="__codelineno-0-85" name="__codelineno-0-85"></a><span class="k">def</span> <span class="nf">compose_score</span><span class="p">(</span>
-</span><span id="__span-0-86"><a id="__codelineno-0-86" name="__codelineno-0-86"></a>    <span class="n">scorer</span><span class="p">:</span> <span class="n">Scorer</span><span class="p">,</span>
-</span><span id="__span-0-87"><a id="__codelineno-0-87" name="__codelineno-0-87"></a>    <span class="n">transformation</span><span class="p">:</span> <span class="n">Callable</span><span class="p">[[</span><span class="nb">float</span><span class="p">],</span> <span class="nb">float</span><span class="p">],</span>
-</span><span id="__span-0-88"><a id="__codelineno-0-88" name="__codelineno-0-88"></a>    <span class="nb">range</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">float</span><span class="p">],</span>
-</span><span id="__span-0-89"><a id="__codelineno-0-89" name="__codelineno-0-89"></a>    <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-</span><span id="__span-0-90"><a id="__codelineno-0-90" name="__codelineno-0-90"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Scorer</span><span class="p">:</span>
-</span><span id="__span-0-91"><a id="__codelineno-0-91" name="__codelineno-0-91"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Composes a scoring function with an arbitrary scalar transformation.</span>
-</span><span id="__span-0-92"><a id="__codelineno-0-92" name="__codelineno-0-92"></a>
-</span><span id="__span-0-93"><a id="__codelineno-0-93" name="__codelineno-0-93"></a><span class="sd">    Useful to squash unbounded scores into ranges manageable by data valuation</span>
-</span><span id="__span-0-94"><a id="__codelineno-0-94" name="__codelineno-0-94"></a><span class="sd">    methods.</span>
-</span><span id="__span-0-95"><a id="__codelineno-0-95" name="__codelineno-0-95"></a>
-</span><span id="__span-0-96"><a id="__codelineno-0-96" name="__codelineno-0-96"></a><span class="sd">    Example:</span>
-</span><span id="__span-0-97"><a id="__codelineno-0-97" name="__codelineno-0-97"></a>
-</span><span id="__span-0-98"><a id="__codelineno-0-98" name="__codelineno-0-98"></a><span class="sd">    ```python</span>
-</span><span id="__span-0-99"><a id="__codelineno-0-99" name="__codelineno-0-99"></a><span class="sd">    sigmoid = lambda x: 1/(1+np.exp(-x))</span>
-</span><span id="__span-0-100"><a id="__codelineno-0-100" name="__codelineno-0-100"></a><span class="sd">    compose_score(Scorer(&quot;r2&quot;), sigmoid, range=(0,1), name=&quot;squashed r2&quot;)</span>
-</span><span id="__span-0-101"><a id="__codelineno-0-101" name="__codelineno-0-101"></a><span class="sd">    ```</span>
-</span><span id="__span-0-102"><a id="__codelineno-0-102" name="__codelineno-0-102"></a>
-</span><span id="__span-0-103"><a id="__codelineno-0-103" name="__codelineno-0-103"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-104"><a id="__codelineno-0-104" name="__codelineno-0-104"></a><span class="sd">        scorer: The object to be composed.</span>
-</span><span id="__span-0-105"><a id="__codelineno-0-105" name="__codelineno-0-105"></a><span class="sd">        transformation: A scalar transformation</span>
-</span><span id="__span-0-106"><a id="__codelineno-0-106" name="__codelineno-0-106"></a><span class="sd">        range: The range of the transformation. This will be used e.g. by</span>
-</span><span id="__span-0-107"><a id="__codelineno-0-107" name="__codelineno-0-107"></a><span class="sd">            [Utility][pydvl.utils.utility.Utility] for the range of the composed.</span>
-</span><span id="__span-0-108"><a id="__codelineno-0-108" name="__codelineno-0-108"></a><span class="sd">        name: A string representation for the composition, for `str()`.</span>
-</span><span id="__span-0-109"><a id="__codelineno-0-109" name="__codelineno-0-109"></a>
-</span><span id="__span-0-110"><a id="__codelineno-0-110" name="__codelineno-0-110"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-111"><a id="__codelineno-0-111" name="__codelineno-0-111"></a><span class="sd">        The composite [Scorer][pydvl.utils.score.Scorer].</span>
-</span><span id="__span-0-112"><a id="__codelineno-0-112" name="__codelineno-0-112"></a><span class="sd">    &quot;&quot;&quot;</span>
+<span class="normal"><a href="#__codelineno-0-119">119</a></span>
+<span class="normal"><a href="#__codelineno-0-120">120</a></span>
+<span class="normal"><a href="#__codelineno-0-121">121</a></span>
+<span class="normal"><a href="#__codelineno-0-122">122</a></span>
+<span class="normal"><a href="#__codelineno-0-123">123</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-89"><a id="__codelineno-0-89" name="__codelineno-0-89"></a><span class="k">def</span> <span class="nf">compose_score</span><span class="p">(</span>
+</span><span id="__span-0-90"><a id="__codelineno-0-90" name="__codelineno-0-90"></a>    <span class="n">scorer</span><span class="p">:</span> <span class="n">Scorer</span><span class="p">,</span>
+</span><span id="__span-0-91"><a id="__codelineno-0-91" name="__codelineno-0-91"></a>    <span class="n">transformation</span><span class="p">:</span> <span class="n">Callable</span><span class="p">[[</span><span class="nb">float</span><span class="p">],</span> <span class="nb">float</span><span class="p">],</span>
+</span><span id="__span-0-92"><a id="__codelineno-0-92" name="__codelineno-0-92"></a>    <span class="nb">range</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">float</span><span class="p">],</span>
+</span><span id="__span-0-93"><a id="__codelineno-0-93" name="__codelineno-0-93"></a>    <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+</span><span id="__span-0-94"><a id="__codelineno-0-94" name="__codelineno-0-94"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Scorer</span><span class="p">:</span>
+</span><span id="__span-0-95"><a id="__codelineno-0-95" name="__codelineno-0-95"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Composes a scoring function with an arbitrary scalar transformation.</span>
+</span><span id="__span-0-96"><a id="__codelineno-0-96" name="__codelineno-0-96"></a>
+</span><span id="__span-0-97"><a id="__codelineno-0-97" name="__codelineno-0-97"></a><span class="sd">    Useful to squash unbounded scores into ranges manageable by data valuation</span>
+</span><span id="__span-0-98"><a id="__codelineno-0-98" name="__codelineno-0-98"></a><span class="sd">    methods.</span>
+</span><span id="__span-0-99"><a id="__codelineno-0-99" name="__codelineno-0-99"></a>
+</span><span id="__span-0-100"><a id="__codelineno-0-100" name="__codelineno-0-100"></a><span class="sd">    Example:</span>
+</span><span id="__span-0-101"><a id="__codelineno-0-101" name="__codelineno-0-101"></a>
+</span><span id="__span-0-102"><a id="__codelineno-0-102" name="__codelineno-0-102"></a><span class="sd">    ```python</span>
+</span><span id="__span-0-103"><a id="__codelineno-0-103" name="__codelineno-0-103"></a><span class="sd">    sigmoid = lambda x: 1/(1+np.exp(-x))</span>
+</span><span id="__span-0-104"><a id="__codelineno-0-104" name="__codelineno-0-104"></a><span class="sd">    compose_score(Scorer(&quot;r2&quot;), sigmoid, range=(0,1), name=&quot;squashed r2&quot;)</span>
+</span><span id="__span-0-105"><a id="__codelineno-0-105" name="__codelineno-0-105"></a><span class="sd">    ```</span>
+</span><span id="__span-0-106"><a id="__codelineno-0-106" name="__codelineno-0-106"></a>
+</span><span id="__span-0-107"><a id="__codelineno-0-107" name="__codelineno-0-107"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-108"><a id="__codelineno-0-108" name="__codelineno-0-108"></a><span class="sd">        scorer: The object to be composed.</span>
+</span><span id="__span-0-109"><a id="__codelineno-0-109" name="__codelineno-0-109"></a><span class="sd">        transformation: A scalar transformation</span>
+</span><span id="__span-0-110"><a id="__codelineno-0-110" name="__codelineno-0-110"></a><span class="sd">        range: The range of the transformation. This will be used e.g. by</span>
+</span><span id="__span-0-111"><a id="__codelineno-0-111" name="__codelineno-0-111"></a><span class="sd">            [Utility][pydvl.utils.utility.Utility] for the range of the composed.</span>
+</span><span id="__span-0-112"><a id="__codelineno-0-112" name="__codelineno-0-112"></a><span class="sd">        name: A string representation for the composition, for `str()`.</span>
 </span><span id="__span-0-113"><a id="__codelineno-0-113" name="__codelineno-0-113"></a>
-</span><span id="__span-0-114"><a id="__codelineno-0-114" name="__codelineno-0-114"></a>    <span class="k">class</span> <span class="nc">CompositeScorer</span><span class="p">(</span><span class="n">Scorer</span><span class="p">):</span>
-</span><span id="__span-0-115"><a id="__codelineno-0-115" name="__codelineno-0-115"></a>        <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">model</span><span class="p">:</span> <span class="n">SupervisedModel</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
-</span><span id="__span-0-116"><a id="__codelineno-0-116" name="__codelineno-0-116"></a>            <span class="n">score</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_scorer</span><span class="p">(</span><span class="n">model</span><span class="o">=</span><span class="n">model</span><span class="p">,</span> <span class="n">X</span><span class="o">=</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="o">=</span><span class="n">y</span><span class="p">)</span>
-</span><span id="__span-0-117"><a id="__codelineno-0-117" name="__codelineno-0-117"></a>            <span class="k">return</span> <span class="n">transformation</span><span class="p">(</span><span class="n">score</span><span class="p">)</span>
-</span><span id="__span-0-118"><a id="__codelineno-0-118" name="__codelineno-0-118"></a>
-</span><span id="__span-0-119"><a id="__codelineno-0-119" name="__codelineno-0-119"></a>    <span class="k">return</span> <span class="n">CompositeScorer</span><span class="p">(</span><span class="n">scorer</span><span class="p">,</span> <span class="nb">range</span><span class="o">=</span><span class="nb">range</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">)</span>
+</span><span id="__span-0-114"><a id="__codelineno-0-114" name="__codelineno-0-114"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-115"><a id="__codelineno-0-115" name="__codelineno-0-115"></a><span class="sd">        The composite [Scorer][pydvl.utils.score.Scorer].</span>
+</span><span id="__span-0-116"><a id="__codelineno-0-116" name="__codelineno-0-116"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-117"><a id="__codelineno-0-117" name="__codelineno-0-117"></a>
+</span><span id="__span-0-118"><a id="__codelineno-0-118" name="__codelineno-0-118"></a>    <span class="k">class</span> <span class="nc">CompositeScorer</span><span class="p">(</span><span class="n">Scorer</span><span class="p">):</span>
+</span><span id="__span-0-119"><a id="__codelineno-0-119" name="__codelineno-0-119"></a>        <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">model</span><span class="p">:</span> <span class="n">SupervisedModel</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
+</span><span id="__span-0-120"><a id="__codelineno-0-120" name="__codelineno-0-120"></a>            <span class="n">score</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_scorer</span><span class="p">(</span><span class="n">model</span><span class="o">=</span><span class="n">model</span><span class="p">,</span> <span class="n">X</span><span class="o">=</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="o">=</span><span class="n">y</span><span class="p">)</span>
+</span><span id="__span-0-121"><a id="__codelineno-0-121" name="__codelineno-0-121"></a>            <span class="k">return</span> <span class="n">transformation</span><span class="p">(</span><span class="n">score</span><span class="p">)</span>
+</span><span id="__span-0-122"><a id="__codelineno-0-122" name="__codelineno-0-122"></a>
+</span><span id="__span-0-123"><a id="__codelineno-0-123" name="__codelineno-0-123"></a>    <span class="k">return</span> <span class="n">CompositeScorer</span><span class="p">(</span><span class="n">scorer</span><span class="p">,</span> <span class="nb">range</span><span class="o">=</span><span class="nb">range</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3055,11 +3058,11 @@ <h2 id="pydvl.utils.score.compose_score" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/utils/status/index.html b/devel/api/pydvl/utils/status/index.html
index bf6832054..503f47d85 100644
--- a/devel/api/pydvl/utils/status/index.html
+++ b/devel/api/pydvl/utils/status/index.html
@@ -2678,11 +2678,11 @@ <h4 id="pydvl.utils.status.Status--boolean-casting">Boolean casting<a class="hea
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/utils/types/index.html b/devel/api/pydvl/utils/types/index.html
index 5c9770f95..f958365ee 100644
--- a/devel/api/pydvl/utils/types/index.html
+++ b/devel/api/pydvl/utils/types/index.html
@@ -1794,6 +1794,33 @@
     SupervisedModel
   </a>
   
+    <nav class="md-nav" aria-label="SupervisedModel">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#pydvl.utils.types.SupervisedModel.fit" class="md-nav__link">
+    fit()
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#pydvl.utils.types.SupervisedModel.predict" class="md-nav__link">
+    predict()
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#pydvl.utils.types.SupervisedModel.score" class="md-nav__link">
+    score()
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
 </li>
       
         <li class="md-nav__item">
@@ -1801,6 +1828,19 @@
     NoPublicConstructor
   </a>
   
+    <nav class="md-nav" aria-label="NoPublicConstructor">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#pydvl.utils.types.NoPublicConstructor.create" class="md-nav__link">
+    create()
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
 </li>
       
         <li class="md-nav__item">
@@ -2508,6 +2548,33 @@
     SupervisedModel
   </a>
   
+    <nav class="md-nav" aria-label="SupervisedModel">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#pydvl.utils.types.SupervisedModel.fit" class="md-nav__link">
+    fit()
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#pydvl.utils.types.SupervisedModel.predict" class="md-nav__link">
+    predict()
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#pydvl.utils.types.SupervisedModel.score" class="md-nav__link">
+    score()
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
 </li>
       
         <li class="md-nav__item">
@@ -2515,6 +2582,19 @@
     NoPublicConstructor
   </a>
   
+    <nav class="md-nav" aria-label="NoPublicConstructor">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#pydvl.utils.types.NoPublicConstructor.create" class="md-nav__link">
+    create()
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
 </li>
       
         <li class="md-nav__item">
@@ -2601,6 +2681,292 @@ <h2 id="pydvl.utils.types.SupervisedModel" class="doc doc-heading">
 
 
 
+<div class="doc doc-object doc-function">
+
+
+
+
+<h3 id="pydvl.utils.types.SupervisedModel.fit" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">fit</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span></code>
+
+<a href="#pydvl.utils.types.SupervisedModel.fit" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>Fit the model to the data</p>
+
+
+
+  <table>
+    <thead>
+      <tr>
+        <th><b>PARAMETER</b></th>
+        <th><b> DESCRIPTION</b></th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr>
+          <td><code>x</code></td>
+          <td class="doc-param-details">
+            <div class="doc-md-description">
+              <p>Independent variables</p>
+            </div>
+            <p>
+                <span class="doc-param-annotation">
+                  <b>TYPE:</b>
+                    <code><a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a></code>
+                </span>
+            </p>
+          </td>
+        </tr>
+        <tr>
+          <td><code>y</code></td>
+          <td class="doc-param-details">
+            <div class="doc-md-description">
+              <p>Dependent variable</p>
+            </div>
+            <p>
+                <span class="doc-param-annotation">
+                  <b>TYPE:</b>
+                    <code><a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a></code>
+                </span>
+            </p>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+          <details class="quote">
+            <summary>Source code in <code>src/pydvl/utils/types.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-48">48</a></span>
+<span class="normal"><a href="#__codelineno-0-49">49</a></span>
+<span class="normal"><a href="#__codelineno-0-50">50</a></span>
+<span class="normal"><a href="#__codelineno-0-51">51</a></span>
+<span class="normal"><a href="#__codelineno-0-52">52</a></span>
+<span class="normal"><a href="#__codelineno-0-53">53</a></span>
+<span class="normal"><a href="#__codelineno-0-54">54</a></span>
+<span class="normal"><a href="#__codelineno-0-55">55</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-48"><a id="__codelineno-0-48" name="__codelineno-0-48"></a><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">):</span>
+</span><span id="__span-0-49"><a id="__codelineno-0-49" name="__codelineno-0-49"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Fit the model to the data</span>
+</span><span id="__span-0-50"><a id="__codelineno-0-50" name="__codelineno-0-50"></a>
+</span><span id="__span-0-51"><a id="__codelineno-0-51" name="__codelineno-0-51"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-52"><a id="__codelineno-0-52" name="__codelineno-0-52"></a><span class="sd">        x: Independent variables</span>
+</span><span id="__span-0-53"><a id="__codelineno-0-53" name="__codelineno-0-53"></a><span class="sd">        y: Dependent variable</span>
+</span><span id="__span-0-54"><a id="__codelineno-0-54" name="__codelineno-0-54"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-55"><a id="__codelineno-0-55" name="__codelineno-0-55"></a>    <span class="k">pass</span>
+</span></code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
+<div class="doc doc-object doc-function">
+
+
+
+
+<h3 id="pydvl.utils.types.SupervisedModel.predict" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">predict</span><span class="p">(</span><span class="n">x</span><span class="p">)</span></code>
+
+<a href="#pydvl.utils.types.SupervisedModel.predict" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>Compute predictions for the input</p>
+
+
+
+  <table>
+    <thead>
+      <tr>
+        <th><b>PARAMETER</b></th>
+        <th><b> DESCRIPTION</b></th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr>
+          <td><code>x</code></td>
+          <td class="doc-param-details">
+            <div class="doc-md-description">
+              <p>Independent variables for which to compute predictions</p>
+            </div>
+            <p>
+                <span class="doc-param-annotation">
+                  <b>TYPE:</b>
+                    <code><a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a></code>
+                </span>
+            </p>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+  <table>
+    <thead>
+      <tr>
+        <th><b>RETURNS</b></th>
+        <th><b>DESCRIPTION</b></th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr>
+          <td>
+              <span class="doc-returns-annotation">
+                  <code><a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a></code>
+              </span>
+          </td>
+          <td class="doc-returns-details">
+            <div class="doc-md-description">
+              <p>Predictions for the input</p>
+            </div>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+          <details class="quote">
+            <summary>Source code in <code>src/pydvl/utils/types.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-57">57</a></span>
+<span class="normal"><a href="#__codelineno-0-58">58</a></span>
+<span class="normal"><a href="#__codelineno-0-59">59</a></span>
+<span class="normal"><a href="#__codelineno-0-60">60</a></span>
+<span class="normal"><a href="#__codelineno-0-61">61</a></span>
+<span class="normal"><a href="#__codelineno-0-62">62</a></span>
+<span class="normal"><a href="#__codelineno-0-63">63</a></span>
+<span class="normal"><a href="#__codelineno-0-64">64</a></span>
+<span class="normal"><a href="#__codelineno-0-65">65</a></span>
+<span class="normal"><a href="#__codelineno-0-66">66</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-57"><a id="__codelineno-0-57" name="__codelineno-0-57"></a><span class="k">def</span> <span class="nf">predict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">NDArray</span><span class="p">:</span>
+</span><span id="__span-0-58"><a id="__codelineno-0-58" name="__codelineno-0-58"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Compute predictions for the input</span>
+</span><span id="__span-0-59"><a id="__codelineno-0-59" name="__codelineno-0-59"></a>
+</span><span id="__span-0-60"><a id="__codelineno-0-60" name="__codelineno-0-60"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-61"><a id="__codelineno-0-61" name="__codelineno-0-61"></a><span class="sd">        x: Independent variables for which to compute predictions</span>
+</span><span id="__span-0-62"><a id="__codelineno-0-62" name="__codelineno-0-62"></a>
+</span><span id="__span-0-63"><a id="__codelineno-0-63" name="__codelineno-0-63"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-64"><a id="__codelineno-0-64" name="__codelineno-0-64"></a><span class="sd">        Predictions for the input</span>
+</span><span id="__span-0-65"><a id="__codelineno-0-65" name="__codelineno-0-65"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-66"><a id="__codelineno-0-66" name="__codelineno-0-66"></a>    <span class="k">pass</span>
+</span></code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
+<div class="doc doc-object doc-function">
+
+
+
+
+<h3 id="pydvl.utils.types.SupervisedModel.score" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">score</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span></code>
+
+<a href="#pydvl.utils.types.SupervisedModel.score" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>Compute the score of the model given test data</p>
+
+
+
+  <table>
+    <thead>
+      <tr>
+        <th><b>PARAMETER</b></th>
+        <th><b> DESCRIPTION</b></th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr>
+          <td><code>x</code></td>
+          <td class="doc-param-details">
+            <div class="doc-md-description">
+              <p>Independent variables</p>
+            </div>
+            <p>
+                <span class="doc-param-annotation">
+                  <b>TYPE:</b>
+                    <code><a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a></code>
+                </span>
+            </p>
+          </td>
+        </tr>
+        <tr>
+          <td><code>y</code></td>
+          <td class="doc-param-details">
+            <div class="doc-md-description">
+              <p>Dependent variable</p>
+            </div>
+            <p>
+                <span class="doc-param-annotation">
+                  <b>TYPE:</b>
+                    <code><a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a></code>
+                </span>
+            </p>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+
+
+  <table>
+    <thead>
+      <tr>
+        <th><b>RETURNS</b></th>
+        <th><b>DESCRIPTION</b></th>
+      </tr>
+    </thead>
+    <tbody>
+        <tr>
+          <td>
+              <span class="doc-returns-annotation">
+                  <code><a class="autorefs autorefs-external" href="https://docs.python.org/3/library/functions.html#float">float</a></code>
+              </span>
+          </td>
+          <td class="doc-returns-details">
+            <div class="doc-md-description">
+              <p>The score of the model on <code>(x, y)</code></p>
+            </div>
+          </td>
+        </tr>
+    </tbody>
+  </table>
+
+          <details class="quote">
+            <summary>Source code in <code>src/pydvl/utils/types.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-68">68</a></span>
+<span class="normal"><a href="#__codelineno-0-69">69</a></span>
+<span class="normal"><a href="#__codelineno-0-70">70</a></span>
+<span class="normal"><a href="#__codelineno-0-71">71</a></span>
+<span class="normal"><a href="#__codelineno-0-72">72</a></span>
+<span class="normal"><a href="#__codelineno-0-73">73</a></span>
+<span class="normal"><a href="#__codelineno-0-74">74</a></span>
+<span class="normal"><a href="#__codelineno-0-75">75</a></span>
+<span class="normal"><a href="#__codelineno-0-76">76</a></span>
+<span class="normal"><a href="#__codelineno-0-77">77</a></span>
+<span class="normal"><a href="#__codelineno-0-78">78</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-68"><a id="__codelineno-0-68" name="__codelineno-0-68"></a><span class="k">def</span> <span class="nf">score</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
+</span><span id="__span-0-69"><a id="__codelineno-0-69" name="__codelineno-0-69"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Compute the score of the model given test data</span>
+</span><span id="__span-0-70"><a id="__codelineno-0-70" name="__codelineno-0-70"></a>
+</span><span id="__span-0-71"><a id="__codelineno-0-71" name="__codelineno-0-71"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-72"><a id="__codelineno-0-72" name="__codelineno-0-72"></a><span class="sd">        x: Independent variables</span>
+</span><span id="__span-0-73"><a id="__codelineno-0-73" name="__codelineno-0-73"></a><span class="sd">        y: Dependent variable</span>
+</span><span id="__span-0-74"><a id="__codelineno-0-74" name="__codelineno-0-74"></a>
+</span><span id="__span-0-75"><a id="__codelineno-0-75" name="__codelineno-0-75"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-76"><a id="__codelineno-0-76" name="__codelineno-0-76"></a><span class="sd">        The score of the model on `(x, y)`</span>
+</span><span id="__span-0-77"><a id="__codelineno-0-77" name="__codelineno-0-77"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-78"><a id="__codelineno-0-78" name="__codelineno-0-78"></a>    <span class="k">pass</span>
+</span></code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
 
   </div>
 
@@ -2650,6 +3016,35 @@ <h2 id="pydvl.utils.types.NoPublicConstructor" class="doc doc-heading">
 
 
 
+<div class="doc doc-object doc-function">
+
+
+
+
+<h3 id="pydvl.utils.types.NoPublicConstructor.create" class="doc doc-heading">
+          <code class="highlight language-python"><span class="n">create</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></code>
+
+<a href="#pydvl.utils.types.NoPublicConstructor.create" class="headerlink" title="Permanent link">&para;</a></h3>
+
+
+  <div class="doc doc-contents ">
+  
+      <p>Create an instance of the class</p>
+
+          <details class="quote">
+            <summary>Source code in <code>src/pydvl/utils/types.py</code></summary>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-102">102</a></span>
+<span class="normal"><a href="#__codelineno-0-103">103</a></span>
+<span class="normal"><a href="#__codelineno-0-104">104</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-102"><a id="__codelineno-0-102" name="__codelineno-0-102"></a><span class="k">def</span> <span class="nf">create</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">:</span> <span class="n">Any</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Any</span><span class="p">):</span>
+</span><span id="__span-0-103"><a id="__codelineno-0-103" name="__codelineno-0-103"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Create an instance of the class&quot;&quot;&quot;</span>
+</span><span id="__span-0-104"><a id="__codelineno-0-104" name="__codelineno-0-104"></a>    <span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__call__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+</span></code></pre></div></td></tr></table></div>
+          </details>
+  </div>
+
+</div>
+
+
 
   </div>
 
@@ -2737,49 +3132,49 @@ <h2 id="pydvl.utils.types.ensure_seed_sequence" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/types.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-80"> 80</a></span>
-<span class="normal"><a href="#__codelineno-0-81"> 81</a></span>
-<span class="normal"><a href="#__codelineno-0-82"> 82</a></span>
-<span class="normal"><a href="#__codelineno-0-83"> 83</a></span>
-<span class="normal"><a href="#__codelineno-0-84"> 84</a></span>
-<span class="normal"><a href="#__codelineno-0-85"> 85</a></span>
-<span class="normal"><a href="#__codelineno-0-86"> 86</a></span>
-<span class="normal"><a href="#__codelineno-0-87"> 87</a></span>
-<span class="normal"><a href="#__codelineno-0-88"> 88</a></span>
-<span class="normal"><a href="#__codelineno-0-89"> 89</a></span>
-<span class="normal"><a href="#__codelineno-0-90"> 90</a></span>
-<span class="normal"><a href="#__codelineno-0-91"> 91</a></span>
-<span class="normal"><a href="#__codelineno-0-92"> 92</a></span>
-<span class="normal"><a href="#__codelineno-0-93"> 93</a></span>
-<span class="normal"><a href="#__codelineno-0-94"> 94</a></span>
-<span class="normal"><a href="#__codelineno-0-95"> 95</a></span>
-<span class="normal"><a href="#__codelineno-0-96"> 96</a></span>
-<span class="normal"><a href="#__codelineno-0-97"> 97</a></span>
-<span class="normal"><a href="#__codelineno-0-98"> 98</a></span>
-<span class="normal"><a href="#__codelineno-0-99"> 99</a></span>
-<span class="normal"><a href="#__codelineno-0-100">100</a></span>
-<span class="normal"><a href="#__codelineno-0-101">101</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-80"><a id="__codelineno-0-80" name="__codelineno-0-80"></a><span class="k">def</span> <span class="nf">ensure_seed_sequence</span><span class="p">(</span>
-</span><span id="__span-0-81"><a id="__codelineno-0-81" name="__codelineno-0-81"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">Seed</span><span class="p">,</span> <span class="n">SeedSequence</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
-</span><span id="__span-0-82"><a id="__codelineno-0-82" name="__codelineno-0-82"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">SeedSequence</span><span class="p">:</span>
-</span><span id="__span-0-83"><a id="__codelineno-0-83" name="__codelineno-0-83"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-</span><span id="__span-0-84"><a id="__codelineno-0-84" name="__codelineno-0-84"></a><span class="sd">    If the passed seed is a SeedSequence object then it is returned as is. If it is</span>
-</span><span id="__span-0-85"><a id="__codelineno-0-85" name="__codelineno-0-85"></a><span class="sd">    a Generator the internal protected seed sequence from the generator gets extracted.</span>
-</span><span id="__span-0-86"><a id="__codelineno-0-86" name="__codelineno-0-86"></a><span class="sd">    Otherwise, a new SeedSequence object is created from the passed (optional) seed.</span>
-</span><span id="__span-0-87"><a id="__codelineno-0-87" name="__codelineno-0-87"></a>
-</span><span id="__span-0-88"><a id="__codelineno-0-88" name="__codelineno-0-88"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-89"><a id="__codelineno-0-89" name="__codelineno-0-89"></a><span class="sd">        seed: Either an int, a Generator object a SeedSequence object or None.</span>
-</span><span id="__span-0-90"><a id="__codelineno-0-90" name="__codelineno-0-90"></a>
-</span><span id="__span-0-91"><a id="__codelineno-0-91" name="__codelineno-0-91"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-92"><a id="__codelineno-0-92" name="__codelineno-0-92"></a><span class="sd">        A SeedSequence object.</span>
-</span><span id="__span-0-93"><a id="__codelineno-0-93" name="__codelineno-0-93"></a>
-</span><span id="__span-0-94"><a id="__codelineno-0-94" name="__codelineno-0-94"></a><span class="sd">    !!! tip &quot;New in version 0.7.0&quot;</span>
-</span><span id="__span-0-95"><a id="__codelineno-0-95" name="__codelineno-0-95"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-96"><a id="__codelineno-0-96" name="__codelineno-0-96"></a>    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">seed</span><span class="p">,</span> <span class="n">SeedSequence</span><span class="p">):</span>
-</span><span id="__span-0-97"><a id="__codelineno-0-97" name="__codelineno-0-97"></a>        <span class="k">return</span> <span class="n">seed</span>
-</span><span id="__span-0-98"><a id="__codelineno-0-98" name="__codelineno-0-98"></a>    <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">seed</span><span class="p">,</span> <span class="n">Generator</span><span class="p">):</span>
-</span><span id="__span-0-99"><a id="__codelineno-0-99" name="__codelineno-0-99"></a>        <span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">SeedSequence</span><span class="p">,</span> <span class="n">seed</span><span class="o">.</span><span class="n">bit_generator</span><span class="o">.</span><span class="n">seed_seq</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
-</span><span id="__span-0-100"><a id="__codelineno-0-100" name="__codelineno-0-100"></a>    <span class="k">else</span><span class="p">:</span>
-</span><span id="__span-0-101"><a id="__codelineno-0-101" name="__codelineno-0-101"></a>        <span class="k">return</span> <span class="n">SeedSequence</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-107">107</a></span>
+<span class="normal"><a href="#__codelineno-0-108">108</a></span>
+<span class="normal"><a href="#__codelineno-0-109">109</a></span>
+<span class="normal"><a href="#__codelineno-0-110">110</a></span>
+<span class="normal"><a href="#__codelineno-0-111">111</a></span>
+<span class="normal"><a href="#__codelineno-0-112">112</a></span>
+<span class="normal"><a href="#__codelineno-0-113">113</a></span>
+<span class="normal"><a href="#__codelineno-0-114">114</a></span>
+<span class="normal"><a href="#__codelineno-0-115">115</a></span>
+<span class="normal"><a href="#__codelineno-0-116">116</a></span>
+<span class="normal"><a href="#__codelineno-0-117">117</a></span>
+<span class="normal"><a href="#__codelineno-0-118">118</a></span>
+<span class="normal"><a href="#__codelineno-0-119">119</a></span>
+<span class="normal"><a href="#__codelineno-0-120">120</a></span>
+<span class="normal"><a href="#__codelineno-0-121">121</a></span>
+<span class="normal"><a href="#__codelineno-0-122">122</a></span>
+<span class="normal"><a href="#__codelineno-0-123">123</a></span>
+<span class="normal"><a href="#__codelineno-0-124">124</a></span>
+<span class="normal"><a href="#__codelineno-0-125">125</a></span>
+<span class="normal"><a href="#__codelineno-0-126">126</a></span>
+<span class="normal"><a href="#__codelineno-0-127">127</a></span>
+<span class="normal"><a href="#__codelineno-0-128">128</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-107"><a id="__codelineno-0-107" name="__codelineno-0-107"></a><span class="k">def</span> <span class="nf">ensure_seed_sequence</span><span class="p">(</span>
+</span><span id="__span-0-108"><a id="__codelineno-0-108" name="__codelineno-0-108"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">Seed</span><span class="p">,</span> <span class="n">SeedSequence</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
+</span><span id="__span-0-109"><a id="__codelineno-0-109" name="__codelineno-0-109"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">SeedSequence</span><span class="p">:</span>
+</span><span id="__span-0-110"><a id="__codelineno-0-110" name="__codelineno-0-110"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+</span><span id="__span-0-111"><a id="__codelineno-0-111" name="__codelineno-0-111"></a><span class="sd">    If the passed seed is a SeedSequence object then it is returned as is. If it is</span>
+</span><span id="__span-0-112"><a id="__codelineno-0-112" name="__codelineno-0-112"></a><span class="sd">    a Generator the internal protected seed sequence from the generator gets extracted.</span>
+</span><span id="__span-0-113"><a id="__codelineno-0-113" name="__codelineno-0-113"></a><span class="sd">    Otherwise, a new SeedSequence object is created from the passed (optional) seed.</span>
+</span><span id="__span-0-114"><a id="__codelineno-0-114" name="__codelineno-0-114"></a>
+</span><span id="__span-0-115"><a id="__codelineno-0-115" name="__codelineno-0-115"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-116"><a id="__codelineno-0-116" name="__codelineno-0-116"></a><span class="sd">        seed: Either an int, a Generator object a SeedSequence object or None.</span>
+</span><span id="__span-0-117"><a id="__codelineno-0-117" name="__codelineno-0-117"></a>
+</span><span id="__span-0-118"><a id="__codelineno-0-118" name="__codelineno-0-118"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-119"><a id="__codelineno-0-119" name="__codelineno-0-119"></a><span class="sd">        A SeedSequence object.</span>
+</span><span id="__span-0-120"><a id="__codelineno-0-120" name="__codelineno-0-120"></a>
+</span><span id="__span-0-121"><a id="__codelineno-0-121" name="__codelineno-0-121"></a><span class="sd">    !!! tip &quot;New in version 0.7.0&quot;</span>
+</span><span id="__span-0-122"><a id="__codelineno-0-122" name="__codelineno-0-122"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-123"><a id="__codelineno-0-123" name="__codelineno-0-123"></a>    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">seed</span><span class="p">,</span> <span class="n">SeedSequence</span><span class="p">):</span>
+</span><span id="__span-0-124"><a id="__codelineno-0-124" name="__codelineno-0-124"></a>        <span class="k">return</span> <span class="n">seed</span>
+</span><span id="__span-0-125"><a id="__codelineno-0-125" name="__codelineno-0-125"></a>    <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">seed</span><span class="p">,</span> <span class="n">Generator</span><span class="p">):</span>
+</span><span id="__span-0-126"><a id="__codelineno-0-126" name="__codelineno-0-126"></a>        <span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">SeedSequence</span><span class="p">,</span> <span class="n">seed</span><span class="o">.</span><span class="n">bit_generator</span><span class="o">.</span><span class="n">seed_seq</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
+</span><span id="__span-0-127"><a id="__codelineno-0-127" name="__codelineno-0-127"></a>    <span class="k">else</span><span class="p">:</span>
+</span><span id="__span-0-128"><a id="__codelineno-0-128" name="__codelineno-0-128"></a>        <span class="k">return</span> <span class="n">SeedSequence</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -2799,11 +3194,11 @@ <h2 id="pydvl.utils.types.ensure_seed_sequence" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/utils/utility/index.html b/devel/api/pydvl/utils/utility/index.html
index d7dbdb1bb..7e9d76080 100644
--- a/devel/api/pydvl/utils/utility/index.html
+++ b/devel/api/pydvl/utils/utility/index.html
@@ -2677,11 +2677,12 @@ <h1>Utility</h1>
 <p><a class="autorefs autorefs-internal" href="#pydvl.utils.utility.Utility">Utility</a> holds information about model,
 data and scoring function (the latter being what one usually understands
 under <em>utility</em> in the general definition of Shapley value).
-It is automatically cached across machines.</p>
+It is automatically cached across machines when the
+<a class="autorefs autorefs-internal" href="../../../../getting-started/installation/#setting-up-the-cache">cache is configured</a> and it is enabled upon construction.</p>
 <p><a class="autorefs autorefs-internal" href="#pydvl.utils.utility.DataUtilityLearning">DataUtilityLearning</a> adds support
 for learning the scoring function to avoid repeated re-training
 of the model to compute the score.</p>
-<p>This module also contains Utility classes for toy games that are used
+<p>This module also contains derived <code>Utility</code> classes for toy games that are used
 for testing and for demonstration purposes.</p>
 <h3 id="pydvl.utils.utility--references">References<a class="headerlink" href="#pydvl.utils.utility--references" title="Permanent link">&para;</a></h3>
 <div class="footnote">
@@ -2689,8 +2690,8 @@ <h3 id="pydvl.utils.utility--references">References<a class="headerlink" href="#
 <ol>
 <li id="pydvl.utils.utility--fn:1">
 <p><a name="wang_improving_2022"></a>Wang, T., Yang, Y. and Jia, R., 2021.
-<a href="https://arxiv.org/abs/2107.06336">Improving cooperative game theory-based data valuation via data utility learning</a>.
-arXiv preprint arXiv:2107.06336.&#160;<a class="footnote-backref" href="#pydvl.utils.utility--fnref:1" title="Jump back to footnote 1 in the text">&#8617;</a></p>
+<a href="https://arxiv.org/abs/2107.06336">Improving cooperative game theory-based data valuation via data utility
+learning</a>. arXiv preprint arXiv:2107.06336.&#160;<a class="footnote-backref" href="#pydvl.utils.utility--fnref:1" title="Jump back to footnote 1 in the text">&#8617;</a></p>
 </li>
 </ol>
 </div>
@@ -2724,24 +2725,21 @@ <h2 id="pydvl.utils.utility.Utility" class="doc doc-heading">
 function.</p>
 <p>An instance of <code>Utility</code> holds the triple of model, dataset and scoring
 function which determines the value of data points. This is used for the
-computation of
-<a class="autorefs autorefs-internal" href="../../../../value/#game-theoretical-methods">all game-theoretic values</a>
-like <a class="autorefs autorefs-internal" href="../../value/shapley/#pydvl.value.shapley">Shapley values</a> and
-<a class="autorefs autorefs-internal" href="../../value/least_core/#pydvl.value.least_core">the Least Core</a>.</p>
-<p>The Utility expect the model to fulfill
-the <a class="autorefs autorefs-internal" href="../types/#pydvl.utils.types.SupervisedModel">SupervisedModel</a> interface i.e.
+computation of <a class="autorefs autorefs-internal" href="../../../../value/#game-theoretical-methods">all game-theoretic values</a> like
+<a class="autorefs autorefs-internal" href="../../value/shapley/#pydvl.value.shapley">Shapley values</a> and <span data-autorefs-identifier="pydvl.value.least_core">the Least
+Core</span>.</p>
+<p>The Utility expect the model to fulfill the
+<a class="autorefs autorefs-internal" href="../types/#pydvl.utils.types.SupervisedModel">SupervisedModel</a> interface i.e.
 to have <code>fit()</code>, <code>predict()</code>, and <code>score()</code> methods.</p>
 <p>When calling the utility, the model will be
-<a href="https://scikit-learn.org/stable/modules/generated/sklearn.base
-.clone.html">cloned</a>
+<a href="https://scikit-learn.org/stable/modules/generated/sklearn.base.clone.html">cloned</a>
 if it is a Sci-Kit Learn model, otherwise a copy is created using
-<code>deepcopy()</code> from the builtin <a href="https://docs.python.org/3/
-library/copy.html">copy</a> module.</p>
-<p>Since evaluating the scoring function requires retraining the model
-and that can be time-consuming, this class wraps it and caches
-the results of each execution. Caching is available both locally
-and across nodes, but must always be enabled for your
-project first, see <a class="autorefs autorefs-internal" href="../../../../getting-started/installation/#setting-up-the-cache">Setting up the cache</a>.</p>
+<a class="autorefs autorefs-external" href="https://docs.python.org/3/library/copy.html#copy.deepcopy">copy.deepcopy</a></p>
+<p>Since evaluating the scoring function requires retraining the model and that
+can be time-consuming, this class wraps it and caches the results of each
+execution. Caching is available both locally and across nodes, but must
+always be enabled for your project first, see <span data-autorefs-identifier="setting-up-the-cache">Setting up the
+cache</span>.</p>
 
 
 
@@ -2847,9 +2845,9 @@ <h2 id="pydvl.utils.utility.Utility" class="doc doc-heading">
             <div class="doc-md-description">
               <p>A scoring object. If None, the <code>score()</code> method of the model
 will be used. See <a class="autorefs autorefs-internal" href="../score/#pydvl.utils.score">score</a> for ways to create
-and compose scorers, in particular how to set default values and ranges.
-For convenience, a string can be passed, which will be used to construct
-a <a class="autorefs autorefs-internal" href="../score/#pydvl.utils.score.Scorer">Scorer</a>.</p>
+and compose scorers, in particular how to set default values and
+ranges. For convenience, a string can be passed, which will be used
+to construct a <a class="autorefs autorefs-internal" href="../score/#pydvl.utils.score.Scorer">Scorer</a>.</p>
             </div>
             <p>
                 <span class="doc-param-annotation">
@@ -2868,9 +2866,9 @@ <h2 id="pydvl.utils.utility.Utility" class="doc doc-heading">
           <td class="doc-param-details">
             <div class="doc-md-description">
               <p>As a convenience when no <code>scorer</code> object is passed
-(where a default value can be provided), this argument also allows to set
-the default score for models that have not been fit, e.g. when too little
-data is passed, or errors arise.</p>
+(where a default value can be provided), this argument also allows
+to set the default score for models that have not been fit, e.g.
+when too little data is passed, or errors arise.</p>
             </div>
             <p>
                 <span class="doc-param-annotation">
@@ -3017,7 +3015,9 @@ <h2 id="pydvl.utils.utility.Utility" class="doc doc-heading">
 
               <details class="quote">
                 <summary>Source code in <code>src/pydvl/utils/utility.py</code></summary>
-                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-129">129</a></span>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-127">127</a></span>
+<span class="normal"><a href="#__codelineno-0-128">128</a></span>
+<span class="normal"><a href="#__codelineno-0-129">129</a></span>
 <span class="normal"><a href="#__codelineno-0-130">130</a></span>
 <span class="normal"><a href="#__codelineno-0-131">131</a></span>
 <span class="normal"><a href="#__codelineno-0-132">132</a></span>
@@ -3043,37 +3043,35 @@ <h2 id="pydvl.utils.utility.Utility" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-152">152</a></span>
 <span class="normal"><a href="#__codelineno-0-153">153</a></span>
 <span class="normal"><a href="#__codelineno-0-154">154</a></span>
-<span class="normal"><a href="#__codelineno-0-155">155</a></span>
-<span class="normal"><a href="#__codelineno-0-156">156</a></span>
-<span class="normal"><a href="#__codelineno-0-157">157</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-129"><a id="__codelineno-0-129" name="__codelineno-0-129"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-</span><span id="__span-0-130"><a id="__codelineno-0-130" name="__codelineno-0-130"></a>    <span class="bp">self</span><span class="p">,</span>
-</span><span id="__span-0-131"><a id="__codelineno-0-131" name="__codelineno-0-131"></a>    <span class="n">model</span><span class="p">:</span> <span class="n">SupervisedModel</span><span class="p">,</span>
-</span><span id="__span-0-132"><a id="__codelineno-0-132" name="__codelineno-0-132"></a>    <span class="n">data</span><span class="p">:</span> <span class="n">Dataset</span><span class="p">,</span>
-</span><span id="__span-0-133"><a id="__codelineno-0-133" name="__codelineno-0-133"></a>    <span class="n">scorer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Scorer</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-134"><a id="__codelineno-0-134" name="__codelineno-0-134"></a>    <span class="o">*</span><span class="p">,</span>
-</span><span id="__span-0-135"><a id="__codelineno-0-135" name="__codelineno-0-135"></a>    <span class="n">default_score</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
-</span><span id="__span-0-136"><a id="__codelineno-0-136" name="__codelineno-0-136"></a>    <span class="n">score_range</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="o">-</span><span class="n">np</span><span class="o">.</span><span class="n">inf</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">inf</span><span class="p">),</span>
-</span><span id="__span-0-137"><a id="__codelineno-0-137" name="__codelineno-0-137"></a>    <span class="n">catch_errors</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-</span><span id="__span-0-138"><a id="__codelineno-0-138" name="__codelineno-0-138"></a>    <span class="n">show_warnings</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-</span><span id="__span-0-139"><a id="__codelineno-0-139" name="__codelineno-0-139"></a>    <span class="n">enable_cache</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-</span><span id="__span-0-140"><a id="__codelineno-0-140" name="__codelineno-0-140"></a>    <span class="n">cache_options</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">MemcachedConfig</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-141"><a id="__codelineno-0-141" name="__codelineno-0-141"></a>    <span class="n">clone_before_fit</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-</span><span id="__span-0-142"><a id="__codelineno-0-142" name="__codelineno-0-142"></a><span class="p">):</span>
-</span><span id="__span-0-143"><a id="__codelineno-0-143" name="__codelineno-0-143"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">model</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_clone_model</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
-</span><span id="__span-0-144"><a id="__codelineno-0-144" name="__codelineno-0-144"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">data</span>
-</span><span id="__span-0-145"><a id="__codelineno-0-145" name="__codelineno-0-145"></a>    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">scorer</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
-</span><span id="__span-0-146"><a id="__codelineno-0-146" name="__codelineno-0-146"></a>        <span class="n">scorer</span> <span class="o">=</span> <span class="n">Scorer</span><span class="p">(</span><span class="n">scorer</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">default_score</span><span class="p">,</span> <span class="nb">range</span><span class="o">=</span><span class="n">score_range</span><span class="p">)</span>
-</span><span id="__span-0-147"><a id="__codelineno-0-147" name="__codelineno-0-147"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">scorer</span> <span class="o">=</span> <span class="n">check_scoring</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">,</span> <span class="n">scorer</span><span class="p">)</span>
-</span><span id="__span-0-148"><a id="__codelineno-0-148" name="__codelineno-0-148"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">default_score</span> <span class="o">=</span> <span class="n">scorer</span><span class="o">.</span><span class="n">default</span> <span class="k">if</span> <span class="n">scorer</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">default_score</span>
-</span><span id="__span-0-149"><a id="__codelineno-0-149" name="__codelineno-0-149"></a>    <span class="c1"># TODO: auto-fill from known scorers ?</span>
-</span><span id="__span-0-150"><a id="__codelineno-0-150" name="__codelineno-0-150"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">score_range</span> <span class="o">=</span> <span class="n">scorer</span><span class="o">.</span><span class="n">range</span> <span class="k">if</span> <span class="n">scorer</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">score_range</span><span class="p">)</span>
-</span><span id="__span-0-151"><a id="__codelineno-0-151" name="__codelineno-0-151"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">catch_errors</span> <span class="o">=</span> <span class="n">catch_errors</span>
-</span><span id="__span-0-152"><a id="__codelineno-0-152" name="__codelineno-0-152"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">show_warnings</span> <span class="o">=</span> <span class="n">show_warnings</span>
-</span><span id="__span-0-153"><a id="__codelineno-0-153" name="__codelineno-0-153"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">enable_cache</span> <span class="o">=</span> <span class="n">enable_cache</span>
-</span><span id="__span-0-154"><a id="__codelineno-0-154" name="__codelineno-0-154"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">cache_options</span><span class="p">:</span> <span class="n">MemcachedConfig</span> <span class="o">=</span> <span class="n">cache_options</span> <span class="ow">or</span> <span class="n">MemcachedConfig</span><span class="p">()</span>
-</span><span id="__span-0-155"><a id="__codelineno-0-155" name="__codelineno-0-155"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">clone_before_fit</span> <span class="o">=</span> <span class="n">clone_before_fit</span>
-</span><span id="__span-0-156"><a id="__codelineno-0-156" name="__codelineno-0-156"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_signature</span> <span class="o">=</span> <span class="n">serialize</span><span class="p">((</span><span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">),</span> <span class="nb">hash</span><span class="p">(</span><span class="n">data</span><span class="p">),</span> <span class="nb">hash</span><span class="p">(</span><span class="n">scorer</span><span class="p">)))</span>
-</span><span id="__span-0-157"><a id="__codelineno-0-157" name="__codelineno-0-157"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_initialize_utility_wrapper</span><span class="p">()</span>
+<span class="normal"><a href="#__codelineno-0-155">155</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-127"><a id="__codelineno-0-127" name="__codelineno-0-127"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+</span><span id="__span-0-128"><a id="__codelineno-0-128" name="__codelineno-0-128"></a>    <span class="bp">self</span><span class="p">,</span>
+</span><span id="__span-0-129"><a id="__codelineno-0-129" name="__codelineno-0-129"></a>    <span class="n">model</span><span class="p">:</span> <span class="n">SupervisedModel</span><span class="p">,</span>
+</span><span id="__span-0-130"><a id="__codelineno-0-130" name="__codelineno-0-130"></a>    <span class="n">data</span><span class="p">:</span> <span class="n">Dataset</span><span class="p">,</span>
+</span><span id="__span-0-131"><a id="__codelineno-0-131" name="__codelineno-0-131"></a>    <span class="n">scorer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Scorer</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-132"><a id="__codelineno-0-132" name="__codelineno-0-132"></a>    <span class="o">*</span><span class="p">,</span>
+</span><span id="__span-0-133"><a id="__codelineno-0-133" name="__codelineno-0-133"></a>    <span class="n">default_score</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
+</span><span id="__span-0-134"><a id="__codelineno-0-134" name="__codelineno-0-134"></a>    <span class="n">score_range</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="o">-</span><span class="n">np</span><span class="o">.</span><span class="n">inf</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">inf</span><span class="p">),</span>
+</span><span id="__span-0-135"><a id="__codelineno-0-135" name="__codelineno-0-135"></a>    <span class="n">catch_errors</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+</span><span id="__span-0-136"><a id="__codelineno-0-136" name="__codelineno-0-136"></a>    <span class="n">show_warnings</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+</span><span id="__span-0-137"><a id="__codelineno-0-137" name="__codelineno-0-137"></a>    <span class="n">enable_cache</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+</span><span id="__span-0-138"><a id="__codelineno-0-138" name="__codelineno-0-138"></a>    <span class="n">cache_options</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">MemcachedConfig</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-139"><a id="__codelineno-0-139" name="__codelineno-0-139"></a>    <span class="n">clone_before_fit</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+</span><span id="__span-0-140"><a id="__codelineno-0-140" name="__codelineno-0-140"></a><span class="p">):</span>
+</span><span id="__span-0-141"><a id="__codelineno-0-141" name="__codelineno-0-141"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">model</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_clone_model</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
+</span><span id="__span-0-142"><a id="__codelineno-0-142" name="__codelineno-0-142"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">data</span>
+</span><span id="__span-0-143"><a id="__codelineno-0-143" name="__codelineno-0-143"></a>    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">scorer</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
+</span><span id="__span-0-144"><a id="__codelineno-0-144" name="__codelineno-0-144"></a>        <span class="n">scorer</span> <span class="o">=</span> <span class="n">Scorer</span><span class="p">(</span><span class="n">scorer</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">default_score</span><span class="p">,</span> <span class="nb">range</span><span class="o">=</span><span class="n">score_range</span><span class="p">)</span>
+</span><span id="__span-0-145"><a id="__codelineno-0-145" name="__codelineno-0-145"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">scorer</span> <span class="o">=</span> <span class="n">check_scoring</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">,</span> <span class="n">scorer</span><span class="p">)</span>
+</span><span id="__span-0-146"><a id="__codelineno-0-146" name="__codelineno-0-146"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">default_score</span> <span class="o">=</span> <span class="n">scorer</span><span class="o">.</span><span class="n">default</span> <span class="k">if</span> <span class="n">scorer</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">default_score</span>
+</span><span id="__span-0-147"><a id="__codelineno-0-147" name="__codelineno-0-147"></a>    <span class="c1"># TODO: auto-fill from known scorers ?</span>
+</span><span id="__span-0-148"><a id="__codelineno-0-148" name="__codelineno-0-148"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">score_range</span> <span class="o">=</span> <span class="n">scorer</span><span class="o">.</span><span class="n">range</span> <span class="k">if</span> <span class="n">scorer</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">score_range</span><span class="p">)</span>
+</span><span id="__span-0-149"><a id="__codelineno-0-149" name="__codelineno-0-149"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">catch_errors</span> <span class="o">=</span> <span class="n">catch_errors</span>
+</span><span id="__span-0-150"><a id="__codelineno-0-150" name="__codelineno-0-150"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">show_warnings</span> <span class="o">=</span> <span class="n">show_warnings</span>
+</span><span id="__span-0-151"><a id="__codelineno-0-151" name="__codelineno-0-151"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">enable_cache</span> <span class="o">=</span> <span class="n">enable_cache</span>
+</span><span id="__span-0-152"><a id="__codelineno-0-152" name="__codelineno-0-152"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">cache_options</span><span class="p">:</span> <span class="n">MemcachedConfig</span> <span class="o">=</span> <span class="n">cache_options</span> <span class="ow">or</span> <span class="n">MemcachedConfig</span><span class="p">()</span>
+</span><span id="__span-0-153"><a id="__codelineno-0-153" name="__codelineno-0-153"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">clone_before_fit</span> <span class="o">=</span> <span class="n">clone_before_fit</span>
+</span><span id="__span-0-154"><a id="__codelineno-0-154" name="__codelineno-0-154"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_signature</span> <span class="o">=</span> <span class="n">serialize</span><span class="p">((</span><span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">),</span> <span class="nb">hash</span><span class="p">(</span><span class="n">data</span><span class="p">),</span> <span class="nb">hash</span><span class="p">(</span><span class="n">scorer</span><span class="p">)))</span>
+</span><span id="__span-0-155"><a id="__codelineno-0-155" name="__codelineno-0-155"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_initialize_utility_wrapper</span><span class="p">()</span>
 </span></code></pre></div></td></tr></table></div>
               </details>
 
@@ -3179,21 +3177,21 @@ <h3 id="pydvl.utils.utility.Utility.__call__" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/utils/utility.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-174">174</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-172">172</a></span>
+<span class="normal"><a href="#__codelineno-0-173">173</a></span>
+<span class="normal"><a href="#__codelineno-0-174">174</a></span>
 <span class="normal"><a href="#__codelineno-0-175">175</a></span>
 <span class="normal"><a href="#__codelineno-0-176">176</a></span>
 <span class="normal"><a href="#__codelineno-0-177">177</a></span>
 <span class="normal"><a href="#__codelineno-0-178">178</a></span>
-<span class="normal"><a href="#__codelineno-0-179">179</a></span>
-<span class="normal"><a href="#__codelineno-0-180">180</a></span>
-<span class="normal"><a href="#__codelineno-0-181">181</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-174"><a id="__codelineno-0-174" name="__codelineno-0-174"></a><span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">indices</span><span class="p">:</span> <span class="n">Iterable</span><span class="p">[</span><span class="nb">int</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
-</span><span id="__span-0-175"><a id="__codelineno-0-175" name="__codelineno-0-175"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-</span><span id="__span-0-176"><a id="__codelineno-0-176" name="__codelineno-0-176"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-177"><a id="__codelineno-0-177" name="__codelineno-0-177"></a><span class="sd">        indices: a subset of valid indices for the</span>
-</span><span id="__span-0-178"><a id="__codelineno-0-178" name="__codelineno-0-178"></a><span class="sd">            `x_train` attribute of [Dataset][pydvl.utils.dataset.Dataset].</span>
-</span><span id="__span-0-179"><a id="__codelineno-0-179" name="__codelineno-0-179"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-180"><a id="__codelineno-0-180" name="__codelineno-0-180"></a>    <span class="n">utility</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_utility_wrapper</span><span class="p">(</span><span class="nb">frozenset</span><span class="p">(</span><span class="n">indices</span><span class="p">))</span>
-</span><span id="__span-0-181"><a id="__codelineno-0-181" name="__codelineno-0-181"></a>    <span class="k">return</span> <span class="n">utility</span>
+<span class="normal"><a href="#__codelineno-0-179">179</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-172"><a id="__codelineno-0-172" name="__codelineno-0-172"></a><span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">indices</span><span class="p">:</span> <span class="n">Iterable</span><span class="p">[</span><span class="nb">int</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
+</span><span id="__span-0-173"><a id="__codelineno-0-173" name="__codelineno-0-173"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+</span><span id="__span-0-174"><a id="__codelineno-0-174" name="__codelineno-0-174"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-175"><a id="__codelineno-0-175" name="__codelineno-0-175"></a><span class="sd">        indices: a subset of valid indices for the</span>
+</span><span id="__span-0-176"><a id="__codelineno-0-176" name="__codelineno-0-176"></a><span class="sd">            `x_train` attribute of [Dataset][pydvl.utils.dataset.Dataset].</span>
+</span><span id="__span-0-177"><a id="__codelineno-0-177" name="__codelineno-0-177"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-178"><a id="__codelineno-0-178" name="__codelineno-0-178"></a>    <span class="n">utility</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_utility_wrapper</span><span class="p">(</span><span class="nb">frozenset</span><span class="p">(</span><span class="n">indices</span><span class="p">))</span>
+</span><span id="__span-0-179"><a id="__codelineno-0-179" name="__codelineno-0-179"></a>    <span class="k">return</span> <span class="n">utility</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3304,23 +3302,23 @@ <h2 id="pydvl.utils.utility.DataUtilityLearning" class="doc doc-heading">
 
               <details class="quote">
                 <summary>Source code in <code>src/pydvl/utils/utility.py</code></summary>
-                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-309">309</a></span>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-307">307</a></span>
+<span class="normal"><a href="#__codelineno-0-308">308</a></span>
+<span class="normal"><a href="#__codelineno-0-309">309</a></span>
 <span class="normal"><a href="#__codelineno-0-310">310</a></span>
 <span class="normal"><a href="#__codelineno-0-311">311</a></span>
 <span class="normal"><a href="#__codelineno-0-312">312</a></span>
 <span class="normal"><a href="#__codelineno-0-313">313</a></span>
 <span class="normal"><a href="#__codelineno-0-314">314</a></span>
-<span class="normal"><a href="#__codelineno-0-315">315</a></span>
-<span class="normal"><a href="#__codelineno-0-316">316</a></span>
-<span class="normal"><a href="#__codelineno-0-317">317</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-309"><a id="__codelineno-0-309" name="__codelineno-0-309"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-</span><span id="__span-0-310"><a id="__codelineno-0-310" name="__codelineno-0-310"></a>    <span class="bp">self</span><span class="p">,</span> <span class="n">u</span><span class="p">:</span> <span class="n">Utility</span><span class="p">,</span> <span class="n">training_budget</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">model</span><span class="p">:</span> <span class="n">SupervisedModel</span>
-</span><span id="__span-0-311"><a id="__codelineno-0-311" name="__codelineno-0-311"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
-</span><span id="__span-0-312"><a id="__codelineno-0-312" name="__codelineno-0-312"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">utility</span> <span class="o">=</span> <span class="n">u</span>
-</span><span id="__span-0-313"><a id="__codelineno-0-313" name="__codelineno-0-313"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">training_budget</span> <span class="o">=</span> <span class="n">training_budget</span>
-</span><span id="__span-0-314"><a id="__codelineno-0-314" name="__codelineno-0-314"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">model</span> <span class="o">=</span> <span class="n">model</span>
-</span><span id="__span-0-315"><a id="__codelineno-0-315" name="__codelineno-0-315"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_current_iteration</span> <span class="o">=</span> <span class="mi">0</span>
-</span><span id="__span-0-316"><a id="__codelineno-0-316" name="__codelineno-0-316"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_is_model_fit</span> <span class="o">=</span> <span class="kc">False</span>
-</span><span id="__span-0-317"><a id="__codelineno-0-317" name="__codelineno-0-317"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_utility_samples</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="n">FrozenSet</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">bool_</span><span class="p">],</span> <span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
+<span class="normal"><a href="#__codelineno-0-315">315</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-307"><a id="__codelineno-0-307" name="__codelineno-0-307"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+</span><span id="__span-0-308"><a id="__codelineno-0-308" name="__codelineno-0-308"></a>    <span class="bp">self</span><span class="p">,</span> <span class="n">u</span><span class="p">:</span> <span class="n">Utility</span><span class="p">,</span> <span class="n">training_budget</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">model</span><span class="p">:</span> <span class="n">SupervisedModel</span>
+</span><span id="__span-0-309"><a id="__codelineno-0-309" name="__codelineno-0-309"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+</span><span id="__span-0-310"><a id="__codelineno-0-310" name="__codelineno-0-310"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">utility</span> <span class="o">=</span> <span class="n">u</span>
+</span><span id="__span-0-311"><a id="__codelineno-0-311" name="__codelineno-0-311"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">training_budget</span> <span class="o">=</span> <span class="n">training_budget</span>
+</span><span id="__span-0-312"><a id="__codelineno-0-312" name="__codelineno-0-312"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">model</span> <span class="o">=</span> <span class="n">model</span>
+</span><span id="__span-0-313"><a id="__codelineno-0-313" name="__codelineno-0-313"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_current_iteration</span> <span class="o">=</span> <span class="mi">0</span>
+</span><span id="__span-0-314"><a id="__codelineno-0-314" name="__codelineno-0-314"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_is_model_fit</span> <span class="o">=</span> <span class="kc">False</span>
+</span><span id="__span-0-315"><a id="__codelineno-0-315" name="__codelineno-0-315"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_utility_samples</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="n">FrozenSet</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">bool_</span><span class="p">],</span> <span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
 </span></code></pre></div></td></tr></table></div>
               </details>
 
@@ -3426,25 +3424,25 @@ <h2 id="pydvl.utils.utility.MinerGameUtility" class="doc doc-heading">
 
               <details class="quote">
                 <summary>Source code in <code>src/pydvl/utils/utility.py</code></summary>
-                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-378">378</a></span>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-376">376</a></span>
+<span class="normal"><a href="#__codelineno-0-377">377</a></span>
+<span class="normal"><a href="#__codelineno-0-378">378</a></span>
 <span class="normal"><a href="#__codelineno-0-379">379</a></span>
 <span class="normal"><a href="#__codelineno-0-380">380</a></span>
 <span class="normal"><a href="#__codelineno-0-381">381</a></span>
 <span class="normal"><a href="#__codelineno-0-382">382</a></span>
 <span class="normal"><a href="#__codelineno-0-383">383</a></span>
 <span class="normal"><a href="#__codelineno-0-384">384</a></span>
-<span class="normal"><a href="#__codelineno-0-385">385</a></span>
-<span class="normal"><a href="#__codelineno-0-386">386</a></span>
-<span class="normal"><a href="#__codelineno-0-387">387</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-378"><a id="__codelineno-0-378" name="__codelineno-0-378"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">n_miners</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-</span><span id="__span-0-379"><a id="__codelineno-0-379" name="__codelineno-0-379"></a>    <span class="k">if</span> <span class="n">n_miners</span> <span class="o">&lt;=</span> <span class="mi">2</span><span class="p">:</span>
-</span><span id="__span-0-380"><a id="__codelineno-0-380" name="__codelineno-0-380"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;n_miners, </span><span class="si">{</span><span class="n">n_miners</span><span class="si">}</span><span class="s2"> should be &gt; 2&quot;</span><span class="p">)</span>
-</span><span id="__span-0-381"><a id="__codelineno-0-381" name="__codelineno-0-381"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">n_miners</span> <span class="o">=</span> <span class="n">n_miners</span>
-</span><span id="__span-0-382"><a id="__codelineno-0-382" name="__codelineno-0-382"></a>
-</span><span id="__span-0-383"><a id="__codelineno-0-383" name="__codelineno-0-383"></a>    <span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="n">n_miners</span><span class="p">)[</span><span class="o">...</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">newaxis</span><span class="p">]</span>
-</span><span id="__span-0-384"><a id="__codelineno-0-384" name="__codelineno-0-384"></a>    <span class="c1"># The y values don&#39;t matter here</span>
-</span><span id="__span-0-385"><a id="__codelineno-0-385" name="__codelineno-0-385"></a>    <span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
-</span><span id="__span-0-386"><a id="__codelineno-0-386" name="__codelineno-0-386"></a>
-</span><span id="__span-0-387"><a id="__codelineno-0-387" name="__codelineno-0-387"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">Dataset</span><span class="p">(</span><span class="n">x_train</span><span class="o">=</span><span class="n">x</span><span class="p">,</span> <span class="n">y_train</span><span class="o">=</span><span class="n">y</span><span class="p">,</span> <span class="n">x_test</span><span class="o">=</span><span class="n">x</span><span class="p">,</span> <span class="n">y_test</span><span class="o">=</span><span class="n">y</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-385">385</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-376"><a id="__codelineno-0-376" name="__codelineno-0-376"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">n_miners</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+</span><span id="__span-0-377"><a id="__codelineno-0-377" name="__codelineno-0-377"></a>    <span class="k">if</span> <span class="n">n_miners</span> <span class="o">&lt;=</span> <span class="mi">2</span><span class="p">:</span>
+</span><span id="__span-0-378"><a id="__codelineno-0-378" name="__codelineno-0-378"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;n_miners, </span><span class="si">{</span><span class="n">n_miners</span><span class="si">}</span><span class="s2"> should be &gt; 2&quot;</span><span class="p">)</span>
+</span><span id="__span-0-379"><a id="__codelineno-0-379" name="__codelineno-0-379"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">n_miners</span> <span class="o">=</span> <span class="n">n_miners</span>
+</span><span id="__span-0-380"><a id="__codelineno-0-380" name="__codelineno-0-380"></a>
+</span><span id="__span-0-381"><a id="__codelineno-0-381" name="__codelineno-0-381"></a>    <span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="n">n_miners</span><span class="p">)[</span><span class="o">...</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">newaxis</span><span class="p">]</span>
+</span><span id="__span-0-382"><a id="__codelineno-0-382" name="__codelineno-0-382"></a>    <span class="c1"># The y values don&#39;t matter here</span>
+</span><span id="__span-0-383"><a id="__codelineno-0-383" name="__codelineno-0-383"></a>    <span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+</span><span id="__span-0-384"><a id="__codelineno-0-384" name="__codelineno-0-384"></a>
+</span><span id="__span-0-385"><a id="__codelineno-0-385" name="__codelineno-0-385"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">Dataset</span><span class="p">(</span><span class="n">x_train</span><span class="o">=</span><span class="n">x</span><span class="p">,</span> <span class="n">y_train</span><span class="o">=</span><span class="n">y</span><span class="p">,</span> <span class="n">x_test</span><span class="o">=</span><span class="n">x</span><span class="p">,</span> <span class="n">y_test</span><span class="o">=</span><span class="n">y</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
               </details>
 
@@ -3537,23 +3535,23 @@ <h2 id="pydvl.utils.utility.GlovesGameUtility" class="doc doc-heading">
 
               <details class="quote">
                 <summary>Source code in <code>src/pydvl/utils/utility.py</code></summary>
-                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-435">435</a></span>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-433">433</a></span>
+<span class="normal"><a href="#__codelineno-0-434">434</a></span>
+<span class="normal"><a href="#__codelineno-0-435">435</a></span>
 <span class="normal"><a href="#__codelineno-0-436">436</a></span>
 <span class="normal"><a href="#__codelineno-0-437">437</a></span>
 <span class="normal"><a href="#__codelineno-0-438">438</a></span>
 <span class="normal"><a href="#__codelineno-0-439">439</a></span>
 <span class="normal"><a href="#__codelineno-0-440">440</a></span>
-<span class="normal"><a href="#__codelineno-0-441">441</a></span>
-<span class="normal"><a href="#__codelineno-0-442">442</a></span>
-<span class="normal"><a href="#__codelineno-0-443">443</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-435"><a id="__codelineno-0-435" name="__codelineno-0-435"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">left</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">right</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-</span><span id="__span-0-436"><a id="__codelineno-0-436" name="__codelineno-0-436"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">left</span> <span class="o">=</span> <span class="n">left</span>
-</span><span id="__span-0-437"><a id="__codelineno-0-437" name="__codelineno-0-437"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">right</span> <span class="o">=</span> <span class="n">right</span>
-</span><span id="__span-0-438"><a id="__codelineno-0-438" name="__codelineno-0-438"></a>
-</span><span id="__span-0-439"><a id="__codelineno-0-439" name="__codelineno-0-439"></a>    <span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="n">left</span> <span class="o">+</span> <span class="n">right</span><span class="p">)[</span><span class="o">...</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">newaxis</span><span class="p">]</span>
-</span><span id="__span-0-440"><a id="__codelineno-0-440" name="__codelineno-0-440"></a>    <span class="c1"># The y values don&#39;t matter here</span>
-</span><span id="__span-0-441"><a id="__codelineno-0-441" name="__codelineno-0-441"></a>    <span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
-</span><span id="__span-0-442"><a id="__codelineno-0-442" name="__codelineno-0-442"></a>
-</span><span id="__span-0-443"><a id="__codelineno-0-443" name="__codelineno-0-443"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">Dataset</span><span class="p">(</span><span class="n">x_train</span><span class="o">=</span><span class="n">x</span><span class="p">,</span> <span class="n">y_train</span><span class="o">=</span><span class="n">y</span><span class="p">,</span> <span class="n">x_test</span><span class="o">=</span><span class="n">x</span><span class="p">,</span> <span class="n">y_test</span><span class="o">=</span><span class="n">y</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-441">441</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-433"><a id="__codelineno-0-433" name="__codelineno-0-433"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">left</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">right</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+</span><span id="__span-0-434"><a id="__codelineno-0-434" name="__codelineno-0-434"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">left</span> <span class="o">=</span> <span class="n">left</span>
+</span><span id="__span-0-435"><a id="__codelineno-0-435" name="__codelineno-0-435"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">right</span> <span class="o">=</span> <span class="n">right</span>
+</span><span id="__span-0-436"><a id="__codelineno-0-436" name="__codelineno-0-436"></a>
+</span><span id="__span-0-437"><a id="__codelineno-0-437" name="__codelineno-0-437"></a>    <span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="n">left</span> <span class="o">+</span> <span class="n">right</span><span class="p">)[</span><span class="o">...</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">newaxis</span><span class="p">]</span>
+</span><span id="__span-0-438"><a id="__codelineno-0-438" name="__codelineno-0-438"></a>    <span class="c1"># The y values don&#39;t matter here</span>
+</span><span id="__span-0-439"><a id="__codelineno-0-439" name="__codelineno-0-439"></a>    <span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+</span><span id="__span-0-440"><a id="__codelineno-0-440" name="__codelineno-0-440"></a>
+</span><span id="__span-0-441"><a id="__codelineno-0-441" name="__codelineno-0-441"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">Dataset</span><span class="p">(</span><span class="n">x_train</span><span class="o">=</span><span class="n">x</span><span class="p">,</span> <span class="n">y_train</span><span class="o">=</span><span class="n">y</span><span class="p">,</span> <span class="n">x_test</span><span class="o">=</span><span class="n">x</span><span class="p">,</span> <span class="n">y_test</span><span class="o">=</span><span class="n">y</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
               </details>
 
@@ -3591,11 +3589,11 @@ <h2 id="pydvl.utils.utility.GlovesGameUtility" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/index.html b/devel/api/pydvl/value/index.html
index 10a24f12a..778983427 100644
--- a/devel/api/pydvl/value/index.html
+++ b/devel/api/pydvl/value/index.html
@@ -2496,11 +2496,11 @@ <h1>Value</h1>
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/least_core/common/index.html b/devel/api/pydvl/value/least_core/common/index.html
index 6ec6e1b40..f44a5ce8b 100644
--- a/devel/api/pydvl/value/least_core/common/index.html
+++ b/devel/api/pydvl/value/least_core/common/index.html
@@ -3135,11 +3135,11 @@ <h2 id="pydvl.value.least_core.common.lc_solve_problems" class="doc doc-heading"
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/least_core/index.html b/devel/api/pydvl/value/least_core/index.html
index f15700f17..0ed80bc2e 100644
--- a/devel/api/pydvl/value/least_core/index.html
+++ b/devel/api/pydvl/value/least_core/index.html
@@ -2906,11 +2906,11 @@ <h2 id="pydvl.value.least_core.compute_least_core_values" class="doc doc-heading
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/least_core/montecarlo/index.html b/devel/api/pydvl/value/least_core/montecarlo/index.html
index 55d052518..10a60def0 100644
--- a/devel/api/pydvl/value/least_core/montecarlo/index.html
+++ b/devel/api/pydvl/value/least_core/montecarlo/index.html
@@ -3044,11 +3044,11 @@ <h2 id="pydvl.value.least_core.montecarlo.mclc_prepare_problem" class="doc doc-h
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/least_core/naive/index.html b/devel/api/pydvl/value/least_core/naive/index.html
index 5a7ec170d..8463f715d 100644
--- a/devel/api/pydvl/value/least_core/naive/index.html
+++ b/devel/api/pydvl/value/least_core/naive/index.html
@@ -2940,11 +2940,11 @@ <h2 id="pydvl.value.least_core.naive.lc_prepare_problem" class="doc doc-heading"
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/loo/index.html b/devel/api/pydvl/value/loo/index.html
index 57141df6d..440cc9cd4 100644
--- a/devel/api/pydvl/value/loo/index.html
+++ b/devel/api/pydvl/value/loo/index.html
@@ -2493,11 +2493,11 @@ <h1>Loo</h1>
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/loo/loo/index.html b/devel/api/pydvl/value/loo/loo/index.html
index 6e05baa3f..941a93571 100644
--- a/devel/api/pydvl/value/loo/loo/index.html
+++ b/devel/api/pydvl/value/loo/loo/index.html
@@ -2841,11 +2841,11 @@ <h2 id="pydvl.value.loo.loo.compute_loo" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/loo/naive/index.html b/devel/api/pydvl/value/loo/naive/index.html
index cb9a40ace..c030e6eef 100644
--- a/devel/api/pydvl/value/loo/naive/index.html
+++ b/devel/api/pydvl/value/loo/naive/index.html
@@ -2591,11 +2591,11 @@ <h2 id="pydvl.value.loo.naive.naive_loo" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/oob/index.html b/devel/api/pydvl/value/oob/index.html
index 4e11f9d90..d15aaaed7 100644
--- a/devel/api/pydvl/value/oob/index.html
+++ b/devel/api/pydvl/value/oob/index.html
@@ -2493,11 +2493,11 @@ <h1>Oob</h1>
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/oob/oob/index.html b/devel/api/pydvl/value/oob/oob/index.html
index 7749809af..19b91cb72 100644
--- a/devel/api/pydvl/value/oob/oob/index.html
+++ b/devel/api/pydvl/value/oob/oob/index.html
@@ -2904,7 +2904,7 @@ <h2 id="pydvl.value.oob.oob.compute_data_oob" class="doc doc-heading">
 </span><span id="__span-0-70"><a id="__codelineno-0-70" name="__codelineno-0-70"></a><span class="sd">        Object with the data values.</span>
 </span><span id="__span-0-71"><a id="__codelineno-0-71" name="__codelineno-0-71"></a><span class="sd">    &quot;&quot;&quot;</span>
 </span><span id="__span-0-72"><a id="__codelineno-0-72" name="__codelineno-0-72"></a>
-</span><span id="__span-0-73"><a id="__codelineno-0-73" name="__codelineno-0-73"></a>    <span class="n">result</span><span class="p">:</span> <span class="n">ValuationResult</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">]</span> <span class="o">=</span> <span class="n">ValuationResult</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span>
+</span><span id="__span-0-73"><a id="__codelineno-0-73" name="__codelineno-0-73"></a>    <span class="n">result</span><span class="p">:</span> <span class="n">ValuationResult</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">object_</span><span class="p">]</span> <span class="o">=</span> <span class="n">ValuationResult</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span>
 </span><span id="__span-0-74"><a id="__codelineno-0-74" name="__codelineno-0-74"></a>        <span class="n">algorithm</span><span class="o">=</span><span class="s2">&quot;data_oob&quot;</span><span class="p">,</span> <span class="n">indices</span><span class="o">=</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="n">data_names</span><span class="o">=</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">data_names</span>
 </span><span id="__span-0-75"><a id="__codelineno-0-75" name="__codelineno-0-75"></a>    <span class="p">)</span>
 </span><span id="__span-0-76"><a id="__codelineno-0-76" name="__codelineno-0-76"></a>
@@ -3192,11 +3192,11 @@ <h2 id="pydvl.value.oob.oob.neg_l2_distance" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/result/index.html b/devel/api/pydvl/value/result/index.html
index 46990d0cb..68f94256a 100644
--- a/devel/api/pydvl/value/result/index.html
+++ b/devel/api/pydvl/value/result/index.html
@@ -2890,7 +2890,7 @@ <h2 id="pydvl.value.result.ValueItem" class="doc doc-heading">
 
   <div class="doc doc-contents ">
           <p class="doc doc-class-bases">
-            Bases: <code><a class="autorefs autorefs-external" title="typing.Generic" href="https://docs.python.org/3/library/typing.html#typing.Generic">Generic</a>[<span title="pydvl.value.result.IndexT">IndexT</span>, <span title="pydvl.value.result.NameT">NameT</span>]</code></p>
+            Bases: <code><a class="autorefs autorefs-external" title="typing.Generic" href="https://docs.python.org/3/library/typing.html#typing.Generic">Generic</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>, <span title="pydvl.utils.types.NameT">NameT</span>]</code></p>
 
   
       <p>The result of a value computation for one datum.</p>
@@ -2922,7 +2922,7 @@ <h2 id="pydvl.value.result.ValueItem" class="doc doc-heading">
             <p>
                 <span class="doc-attribute-annotation">
                   <b>TYPE:</b>
-                    <code><span title="pydvl.value.result.IndexT">IndexT</span></code>
+                    <code><span title="pydvl.utils.types.IndexT">IndexT</span></code>
                 </span>
             </p>
           </td>
@@ -2936,7 +2936,7 @@ <h2 id="pydvl.value.result.ValueItem" class="doc doc-heading">
             <p>
                 <span class="doc-attribute-annotation">
                   <b>TYPE:</b>
-                    <code><span title="pydvl.value.result.NameT">NameT</span></code>
+                    <code><span title="pydvl.utils.types.NameT">NameT</span></code>
                 </span>
             </p>
           </td>
@@ -3043,7 +3043,7 @@ <h2 id="pydvl.value.result.ValuationResult" class="doc doc-heading">
 
   <div class="doc doc-contents ">
           <p class="doc doc-class-bases">
-            Bases: <code><a class="autorefs autorefs-external" title="collections.abc.Sequence" href="https://docs.python.org/3/library/collections.abc.html#collections.abc.Sequence">Sequence</a></code>, <code><a class="autorefs autorefs-external" title="typing.Iterable" href="https://docs.python.org/3/library/typing.html#typing.Iterable">Iterable</a>[<a class="autorefs autorefs-internal" title="pydvl.value.result.ValueItem" href="#pydvl.value.result.ValueItem">ValueItem</a>[<span title="pydvl.value.result.IndexT">IndexT</span>, <span title="pydvl.value.result.NameT">NameT</span>]]</code>, <code><a class="autorefs autorefs-external" title="typing.Generic" href="https://docs.python.org/3/library/typing.html#typing.Generic">Generic</a>[<span title="pydvl.value.result.IndexT">IndexT</span>, <span title="pydvl.value.result.NameT">NameT</span>]</code></p>
+            Bases: <code><a class="autorefs autorefs-external" title="collections.abc.Sequence" href="https://docs.python.org/3/library/collections.abc.html#collections.abc.Sequence">Sequence</a></code>, <code><a class="autorefs autorefs-external" title="typing.Iterable" href="https://docs.python.org/3/library/typing.html#typing.Iterable">Iterable</a>[<a class="autorefs autorefs-internal" title="pydvl.value.result.ValueItem" href="#pydvl.value.result.ValueItem">ValueItem</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>, <span title="pydvl.utils.types.NameT">NameT</span>]]</code>, <code><a class="autorefs autorefs-external" title="typing.Generic" href="https://docs.python.org/3/library/typing.html#typing.Generic">Generic</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>, <span title="pydvl.utils.types.NameT">NameT</span>]</code></p>
 
   
       <p>Objects of this class hold the results of valuation algorithms.</p>
@@ -3125,7 +3125,7 @@ <h4 id="pydvl.value.result.ValuationResult--operating-on-results">Operating on r
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-external" title="typing.Optional" href="https://docs.python.org/3/library/typing.html#typing.Optional">Optional</a>[<a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a>[<span title="pydvl.value.result.IndexT">IndexT</span>]]</code>
+                    <code><a class="autorefs autorefs-external" title="typing.Optional" href="https://docs.python.org/3/library/typing.html#typing.Optional">Optional</a>[<a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>]]</code>
                 </span>
                 <span class="doc-param-default">
                   <b>DEFAULT:</b> 
@@ -3180,7 +3180,7 @@ <h4 id="pydvl.value.result.ValuationResult--operating-on-results">Operating on r
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-external" title="typing.Optional" href="https://docs.python.org/3/library/typing.html#typing.Optional">Optional</a>[<a class="autorefs autorefs-external" title="typing.Sequence" href="https://docs.python.org/3/library/typing.html#typing.Sequence">Sequence</a>[<span title="pydvl.value.result.NameT">NameT</span>] | <a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a>[<span title="pydvl.value.result.NameT">NameT</span>]]</code>
+                    <code><a class="autorefs autorefs-external" title="typing.Optional" href="https://docs.python.org/3/library/typing.html#typing.Optional">Optional</a>[<a class="autorefs autorefs-external" title="typing.Sequence" href="https://docs.python.org/3/library/typing.html#typing.Sequence">Sequence</a>[<span title="pydvl.utils.types.NameT">NameT</span>] | <a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a>[<span title="pydvl.utils.types.NameT">NameT</span>]]</code>
                 </span>
                 <span class="doc-param-default">
                   <b>DEFAULT:</b> 
@@ -3289,7 +3289,13 @@ <h4 id="pydvl.value.result.ValuationResult--operating-on-results">Operating on r
 
               <details class="quote">
                 <summary>Source code in <code>src/pydvl/value/result.py</code></summary>
-                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-226">226</a></span>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-220">220</a></span>
+<span class="normal"><a href="#__codelineno-0-221">221</a></span>
+<span class="normal"><a href="#__codelineno-0-222">222</a></span>
+<span class="normal"><a href="#__codelineno-0-223">223</a></span>
+<span class="normal"><a href="#__codelineno-0-224">224</a></span>
+<span class="normal"><a href="#__codelineno-0-225">225</a></span>
+<span class="normal"><a href="#__codelineno-0-226">226</a></span>
 <span class="normal"><a href="#__codelineno-0-227">227</a></span>
 <span class="normal"><a href="#__codelineno-0-228">228</a></span>
 <span class="normal"><a href="#__codelineno-0-229">229</a></span>
@@ -3333,63 +3339,57 @@ <h4 id="pydvl.value.result.ValuationResult--operating-on-results">Operating on r
 <span class="normal"><a href="#__codelineno-0-267">267</a></span>
 <span class="normal"><a href="#__codelineno-0-268">268</a></span>
 <span class="normal"><a href="#__codelineno-0-269">269</a></span>
-<span class="normal"><a href="#__codelineno-0-270">270</a></span>
-<span class="normal"><a href="#__codelineno-0-271">271</a></span>
-<span class="normal"><a href="#__codelineno-0-272">272</a></span>
-<span class="normal"><a href="#__codelineno-0-273">273</a></span>
-<span class="normal"><a href="#__codelineno-0-274">274</a></span>
-<span class="normal"><a href="#__codelineno-0-275">275</a></span>
-<span class="normal"><a href="#__codelineno-0-276">276</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-226"><a id="__codelineno-0-226" name="__codelineno-0-226"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-</span><span id="__span-0-227"><a id="__codelineno-0-227" name="__codelineno-0-227"></a>    <span class="bp">self</span><span class="p">,</span>
-</span><span id="__span-0-228"><a id="__codelineno-0-228" name="__codelineno-0-228"></a>    <span class="o">*</span><span class="p">,</span>
-</span><span id="__span-0-229"><a id="__codelineno-0-229" name="__codelineno-0-229"></a>    <span class="n">values</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">],</span>
-</span><span id="__span-0-230"><a id="__codelineno-0-230" name="__codelineno-0-230"></a>    <span class="n">variances</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-231"><a id="__codelineno-0-231" name="__codelineno-0-231"></a>    <span class="n">counts</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-232"><a id="__codelineno-0-232" name="__codelineno-0-232"></a>    <span class="n">indices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">NDArray</span><span class="p">[</span><span class="n">IndexT</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-233"><a id="__codelineno-0-233" name="__codelineno-0-233"></a>    <span class="n">data_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="n">NameT</span><span class="p">]</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">NameT</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-234"><a id="__codelineno-0-234" name="__codelineno-0-234"></a>    <span class="n">algorithm</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="p">,</span>
-</span><span id="__span-0-235"><a id="__codelineno-0-235" name="__codelineno-0-235"></a>    <span class="n">status</span><span class="p">:</span> <span class="n">Status</span> <span class="o">=</span> <span class="n">Status</span><span class="o">.</span><span class="n">Pending</span><span class="p">,</span>
-</span><span id="__span-0-236"><a id="__codelineno-0-236" name="__codelineno-0-236"></a>    <span class="n">sort</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-</span><span id="__span-0-237"><a id="__codelineno-0-237" name="__codelineno-0-237"></a>    <span class="o">**</span><span class="n">extra_values</span><span class="p">,</span>
-</span><span id="__span-0-238"><a id="__codelineno-0-238" name="__codelineno-0-238"></a><span class="p">):</span>
-</span><span id="__span-0-239"><a id="__codelineno-0-239" name="__codelineno-0-239"></a>    <span class="k">if</span> <span class="n">variances</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">variances</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">values</span><span class="p">):</span>
-</span><span id="__span-0-240"><a id="__codelineno-0-240" name="__codelineno-0-240"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Lengths of values and variances do not match&quot;</span><span class="p">)</span>
-</span><span id="__span-0-241"><a id="__codelineno-0-241" name="__codelineno-0-241"></a>    <span class="k">if</span> <span class="n">data_names</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">data_names</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">values</span><span class="p">):</span>
-</span><span id="__span-0-242"><a id="__codelineno-0-242" name="__codelineno-0-242"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Lengths of values and data_names do not match&quot;</span><span class="p">)</span>
-</span><span id="__span-0-243"><a id="__codelineno-0-243" name="__codelineno-0-243"></a>    <span class="k">if</span> <span class="n">indices</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">indices</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">values</span><span class="p">):</span>
-</span><span id="__span-0-244"><a id="__codelineno-0-244" name="__codelineno-0-244"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Lengths of values and indices do not match&quot;</span><span class="p">)</span>
-</span><span id="__span-0-245"><a id="__codelineno-0-245" name="__codelineno-0-245"></a>
-</span><span id="__span-0-246"><a id="__codelineno-0-246" name="__codelineno-0-246"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_algorithm</span> <span class="o">=</span> <span class="n">algorithm</span>
-</span><span id="__span-0-247"><a id="__codelineno-0-247" name="__codelineno-0-247"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_status</span> <span class="o">=</span> <span class="n">Status</span><span class="p">(</span><span class="n">status</span><span class="p">)</span>  <span class="c1"># Just in case we are given a string</span>
-</span><span id="__span-0-248"><a id="__codelineno-0-248" name="__codelineno-0-248"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_values</span> <span class="o">=</span> <span class="n">values</span>
-</span><span id="__span-0-249"><a id="__codelineno-0-249" name="__codelineno-0-249"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_variances</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">values</span><span class="p">)</span> <span class="k">if</span> <span class="n">variances</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">variances</span>
-</span><span id="__span-0-250"><a id="__codelineno-0-250" name="__codelineno-0-250"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_counts</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ones_like</span><span class="p">(</span><span class="n">values</span><span class="p">)</span> <span class="k">if</span> <span class="n">counts</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">counts</span>
-</span><span id="__span-0-251"><a id="__codelineno-0-251" name="__codelineno-0-251"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_sort_order</span> <span class="o">=</span> <span class="kc">None</span>
-</span><span id="__span-0-252"><a id="__codelineno-0-252" name="__codelineno-0-252"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_extra_values</span> <span class="o">=</span> <span class="n">extra_values</span> <span class="ow">or</span> <span class="p">{}</span>
-</span><span id="__span-0-253"><a id="__codelineno-0-253" name="__codelineno-0-253"></a>
-</span><span id="__span-0-254"><a id="__codelineno-0-254" name="__codelineno-0-254"></a>    <span class="c1"># Yuk...</span>
-</span><span id="__span-0-255"><a id="__codelineno-0-255" name="__codelineno-0-255"></a>    <span class="k">if</span> <span class="n">data_names</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-</span><span id="__span-0-256"><a id="__codelineno-0-256" name="__codelineno-0-256"></a>        <span class="k">if</span> <span class="n">indices</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-</span><span id="__span-0-257"><a id="__codelineno-0-257" name="__codelineno-0-257"></a>            <span class="bp">self</span><span class="o">.</span><span class="n">_names</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">indices</span><span class="p">)</span>
-</span><span id="__span-0-258"><a id="__codelineno-0-258" name="__codelineno-0-258"></a>        <span class="k">else</span><span class="p">:</span>
-</span><span id="__span-0-259"><a id="__codelineno-0-259" name="__codelineno-0-259"></a>            <span class="bp">self</span><span class="o">.</span><span class="n">_names</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_values</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">)</span>
-</span><span id="__span-0-260"><a id="__codelineno-0-260" name="__codelineno-0-260"></a>    <span class="k">elif</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data_names</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">):</span>
-</span><span id="__span-0-261"><a id="__codelineno-0-261" name="__codelineno-0-261"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_names</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">data_names</span><span class="p">)</span>
-</span><span id="__span-0-262"><a id="__codelineno-0-262" name="__codelineno-0-262"></a>    <span class="k">else</span><span class="p">:</span>
-</span><span id="__span-0-263"><a id="__codelineno-0-263" name="__codelineno-0-263"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_names</span> <span class="o">=</span> <span class="n">data_names</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
-</span><span id="__span-0-264"><a id="__codelineno-0-264" name="__codelineno-0-264"></a>    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="p">))</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="p">):</span>
-</span><span id="__span-0-265"><a id="__codelineno-0-265" name="__codelineno-0-265"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Data names must be unique&quot;</span><span class="p">)</span>
-</span><span id="__span-0-266"><a id="__codelineno-0-266" name="__codelineno-0-266"></a>
-</span><span id="__span-0-267"><a id="__codelineno-0-267" name="__codelineno-0-267"></a>    <span class="k">if</span> <span class="n">indices</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-</span><span id="__span-0-268"><a id="__codelineno-0-268" name="__codelineno-0-268"></a>        <span class="n">indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_values</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">)</span>
-</span><span id="__span-0-269"><a id="__codelineno-0-269" name="__codelineno-0-269"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span> <span class="o">=</span> <span class="n">indices</span>
-</span><span id="__span-0-270"><a id="__codelineno-0-270" name="__codelineno-0-270"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_positions</span> <span class="o">=</span> <span class="p">{</span><span class="n">idx</span><span class="p">:</span> <span class="n">pos</span> <span class="k">for</span> <span class="n">pos</span><span class="p">,</span> <span class="n">idx</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">indices</span><span class="p">)}</span>
-</span><span id="__span-0-271"><a id="__codelineno-0-271" name="__codelineno-0-271"></a>
-</span><span id="__span-0-272"><a id="__codelineno-0-272" name="__codelineno-0-272"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span>
-</span><span id="__span-0-273"><a id="__codelineno-0-273" name="__codelineno-0-273"></a>        <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_values</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span>
-</span><span id="__span-0-274"><a id="__codelineno-0-274" name="__codelineno-0-274"></a>    <span class="p">)</span>
-</span><span id="__span-0-275"><a id="__codelineno-0-275" name="__codelineno-0-275"></a>    <span class="k">if</span> <span class="n">sort</span><span class="p">:</span>
-</span><span id="__span-0-276"><a id="__codelineno-0-276" name="__codelineno-0-276"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span>
+<span class="normal"><a href="#__codelineno-0-270">270</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-220"><a id="__codelineno-0-220" name="__codelineno-0-220"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+</span><span id="__span-0-221"><a id="__codelineno-0-221" name="__codelineno-0-221"></a>    <span class="bp">self</span><span class="p">,</span>
+</span><span id="__span-0-222"><a id="__codelineno-0-222" name="__codelineno-0-222"></a>    <span class="o">*</span><span class="p">,</span>
+</span><span id="__span-0-223"><a id="__codelineno-0-223" name="__codelineno-0-223"></a>    <span class="n">values</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">],</span>
+</span><span id="__span-0-224"><a id="__codelineno-0-224" name="__codelineno-0-224"></a>    <span class="n">variances</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">float_</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-225"><a id="__codelineno-0-225" name="__codelineno-0-225"></a>    <span class="n">counts</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-226"><a id="__codelineno-0-226" name="__codelineno-0-226"></a>    <span class="n">indices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">NDArray</span><span class="p">[</span><span class="n">IndexT</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-227"><a id="__codelineno-0-227" name="__codelineno-0-227"></a>    <span class="n">data_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="n">NameT</span><span class="p">]</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">NameT</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-228"><a id="__codelineno-0-228" name="__codelineno-0-228"></a>    <span class="n">algorithm</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="p">,</span>
+</span><span id="__span-0-229"><a id="__codelineno-0-229" name="__codelineno-0-229"></a>    <span class="n">status</span><span class="p">:</span> <span class="n">Status</span> <span class="o">=</span> <span class="n">Status</span><span class="o">.</span><span class="n">Pending</span><span class="p">,</span>
+</span><span id="__span-0-230"><a id="__codelineno-0-230" name="__codelineno-0-230"></a>    <span class="n">sort</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+</span><span id="__span-0-231"><a id="__codelineno-0-231" name="__codelineno-0-231"></a>    <span class="o">**</span><span class="n">extra_values</span><span class="p">,</span>
+</span><span id="__span-0-232"><a id="__codelineno-0-232" name="__codelineno-0-232"></a><span class="p">):</span>
+</span><span id="__span-0-233"><a id="__codelineno-0-233" name="__codelineno-0-233"></a>    <span class="k">if</span> <span class="n">variances</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">variances</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">values</span><span class="p">):</span>
+</span><span id="__span-0-234"><a id="__codelineno-0-234" name="__codelineno-0-234"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Lengths of values and variances do not match&quot;</span><span class="p">)</span>
+</span><span id="__span-0-235"><a id="__codelineno-0-235" name="__codelineno-0-235"></a>    <span class="k">if</span> <span class="n">data_names</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">data_names</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">values</span><span class="p">):</span>
+</span><span id="__span-0-236"><a id="__codelineno-0-236" name="__codelineno-0-236"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Lengths of values and data_names do not match&quot;</span><span class="p">)</span>
+</span><span id="__span-0-237"><a id="__codelineno-0-237" name="__codelineno-0-237"></a>    <span class="k">if</span> <span class="n">indices</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">indices</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">values</span><span class="p">):</span>
+</span><span id="__span-0-238"><a id="__codelineno-0-238" name="__codelineno-0-238"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Lengths of values and indices do not match&quot;</span><span class="p">)</span>
+</span><span id="__span-0-239"><a id="__codelineno-0-239" name="__codelineno-0-239"></a>
+</span><span id="__span-0-240"><a id="__codelineno-0-240" name="__codelineno-0-240"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_algorithm</span> <span class="o">=</span> <span class="n">algorithm</span>
+</span><span id="__span-0-241"><a id="__codelineno-0-241" name="__codelineno-0-241"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_status</span> <span class="o">=</span> <span class="n">Status</span><span class="p">(</span><span class="n">status</span><span class="p">)</span>  <span class="c1"># Just in case we are given a string</span>
+</span><span id="__span-0-242"><a id="__codelineno-0-242" name="__codelineno-0-242"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_values</span> <span class="o">=</span> <span class="n">values</span>
+</span><span id="__span-0-243"><a id="__codelineno-0-243" name="__codelineno-0-243"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_variances</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">values</span><span class="p">)</span> <span class="k">if</span> <span class="n">variances</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">variances</span>
+</span><span id="__span-0-244"><a id="__codelineno-0-244" name="__codelineno-0-244"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_counts</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ones_like</span><span class="p">(</span><span class="n">values</span><span class="p">)</span> <span class="k">if</span> <span class="n">counts</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">counts</span>
+</span><span id="__span-0-245"><a id="__codelineno-0-245" name="__codelineno-0-245"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_sort_order</span> <span class="o">=</span> <span class="kc">None</span>
+</span><span id="__span-0-246"><a id="__codelineno-0-246" name="__codelineno-0-246"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_extra_values</span> <span class="o">=</span> <span class="n">extra_values</span> <span class="ow">or</span> <span class="p">{}</span>
+</span><span id="__span-0-247"><a id="__codelineno-0-247" name="__codelineno-0-247"></a>
+</span><span id="__span-0-248"><a id="__codelineno-0-248" name="__codelineno-0-248"></a>    <span class="c1"># Yuk...</span>
+</span><span id="__span-0-249"><a id="__codelineno-0-249" name="__codelineno-0-249"></a>    <span class="k">if</span> <span class="n">data_names</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+</span><span id="__span-0-250"><a id="__codelineno-0-250" name="__codelineno-0-250"></a>        <span class="k">if</span> <span class="n">indices</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+</span><span id="__span-0-251"><a id="__codelineno-0-251" name="__codelineno-0-251"></a>            <span class="bp">self</span><span class="o">.</span><span class="n">_names</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">indices</span><span class="p">)</span>
+</span><span id="__span-0-252"><a id="__codelineno-0-252" name="__codelineno-0-252"></a>        <span class="k">else</span><span class="p">:</span>
+</span><span id="__span-0-253"><a id="__codelineno-0-253" name="__codelineno-0-253"></a>            <span class="bp">self</span><span class="o">.</span><span class="n">_names</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_values</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">)</span>
+</span><span id="__span-0-254"><a id="__codelineno-0-254" name="__codelineno-0-254"></a>    <span class="k">elif</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data_names</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">):</span>
+</span><span id="__span-0-255"><a id="__codelineno-0-255" name="__codelineno-0-255"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_names</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">data_names</span><span class="p">)</span>
+</span><span id="__span-0-256"><a id="__codelineno-0-256" name="__codelineno-0-256"></a>    <span class="k">else</span><span class="p">:</span>
+</span><span id="__span-0-257"><a id="__codelineno-0-257" name="__codelineno-0-257"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_names</span> <span class="o">=</span> <span class="n">data_names</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+</span><span id="__span-0-258"><a id="__codelineno-0-258" name="__codelineno-0-258"></a>    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="p">))</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="p">):</span>
+</span><span id="__span-0-259"><a id="__codelineno-0-259" name="__codelineno-0-259"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Data names must be unique&quot;</span><span class="p">)</span>
+</span><span id="__span-0-260"><a id="__codelineno-0-260" name="__codelineno-0-260"></a>
+</span><span id="__span-0-261"><a id="__codelineno-0-261" name="__codelineno-0-261"></a>    <span class="k">if</span> <span class="n">indices</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+</span><span id="__span-0-262"><a id="__codelineno-0-262" name="__codelineno-0-262"></a>        <span class="n">indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_values</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">)</span>
+</span><span id="__span-0-263"><a id="__codelineno-0-263" name="__codelineno-0-263"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span> <span class="o">=</span> <span class="n">indices</span>
+</span><span id="__span-0-264"><a id="__codelineno-0-264" name="__codelineno-0-264"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_positions</span> <span class="o">=</span> <span class="p">{</span><span class="n">idx</span><span class="p">:</span> <span class="n">pos</span> <span class="k">for</span> <span class="n">pos</span><span class="p">,</span> <span class="n">idx</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">indices</span><span class="p">)}</span>
+</span><span id="__span-0-265"><a id="__codelineno-0-265" name="__codelineno-0-265"></a>
+</span><span id="__span-0-266"><a id="__codelineno-0-266" name="__codelineno-0-266"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span>
+</span><span id="__span-0-267"><a id="__codelineno-0-267" name="__codelineno-0-267"></a>        <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_values</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span>
+</span><span id="__span-0-268"><a id="__codelineno-0-268" name="__codelineno-0-268"></a>    <span class="p">)</span>
+</span><span id="__span-0-269"><a id="__codelineno-0-269" name="__codelineno-0-269"></a>    <span class="k">if</span> <span class="n">sort</span><span class="p">:</span>
+</span><span id="__span-0-270"><a id="__codelineno-0-270" name="__codelineno-0-270"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span>
 </span></code></pre></div></td></tr></table></div>
               </details>
 
@@ -3617,7 +3617,13 @@ <h3 id="pydvl.value.result.ValuationResult.sort" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/result.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-278">278</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-272">272</a></span>
+<span class="normal"><a href="#__codelineno-0-273">273</a></span>
+<span class="normal"><a href="#__codelineno-0-274">274</a></span>
+<span class="normal"><a href="#__codelineno-0-275">275</a></span>
+<span class="normal"><a href="#__codelineno-0-276">276</a></span>
+<span class="normal"><a href="#__codelineno-0-277">277</a></span>
+<span class="normal"><a href="#__codelineno-0-278">278</a></span>
 <span class="normal"><a href="#__codelineno-0-279">279</a></span>
 <span class="normal"><a href="#__codelineno-0-280">280</a></span>
 <span class="normal"><a href="#__codelineno-0-281">281</a></span>
@@ -3642,44 +3648,38 @@ <h3 id="pydvl.value.result.ValuationResult.sort" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-300">300</a></span>
 <span class="normal"><a href="#__codelineno-0-301">301</a></span>
 <span class="normal"><a href="#__codelineno-0-302">302</a></span>
-<span class="normal"><a href="#__codelineno-0-303">303</a></span>
-<span class="normal"><a href="#__codelineno-0-304">304</a></span>
-<span class="normal"><a href="#__codelineno-0-305">305</a></span>
-<span class="normal"><a href="#__codelineno-0-306">306</a></span>
-<span class="normal"><a href="#__codelineno-0-307">307</a></span>
-<span class="normal"><a href="#__codelineno-0-308">308</a></span>
-<span class="normal"><a href="#__codelineno-0-309">309</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-278"><a id="__codelineno-0-278" name="__codelineno-0-278"></a><span class="k">def</span> <span class="nf">sort</span><span class="p">(</span>
-</span><span id="__span-0-279"><a id="__codelineno-0-279" name="__codelineno-0-279"></a>    <span class="bp">self</span><span class="p">,</span>
-</span><span id="__span-0-280"><a id="__codelineno-0-280" name="__codelineno-0-280"></a>    <span class="n">reverse</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-</span><span id="__span-0-281"><a id="__codelineno-0-281" name="__codelineno-0-281"></a>    <span class="c1"># Need a &quot;Comparable&quot; type here</span>
-</span><span id="__span-0-282"><a id="__codelineno-0-282" name="__codelineno-0-282"></a>    <span class="n">key</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;value&quot;</span><span class="p">,</span> <span class="s2">&quot;variance&quot;</span><span class="p">,</span> <span class="s2">&quot;index&quot;</span><span class="p">,</span> <span class="s2">&quot;name&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;value&quot;</span><span class="p">,</span>
-</span><span id="__span-0-283"><a id="__codelineno-0-283" name="__codelineno-0-283"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
-</span><span id="__span-0-284"><a id="__codelineno-0-284" name="__codelineno-0-284"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Sorts the indices in place by `key`.</span>
-</span><span id="__span-0-285"><a id="__codelineno-0-285" name="__codelineno-0-285"></a>
-</span><span id="__span-0-286"><a id="__codelineno-0-286" name="__codelineno-0-286"></a><span class="sd">    Once sorted, iteration over the results, and indexing of all the</span>
-</span><span id="__span-0-287"><a id="__codelineno-0-287" name="__codelineno-0-287"></a><span class="sd">    properties</span>
-</span><span id="__span-0-288"><a id="__codelineno-0-288" name="__codelineno-0-288"></a><span class="sd">    [ValuationResult.values][pydvl.value.result.ValuationResult.values],</span>
-</span><span id="__span-0-289"><a id="__codelineno-0-289" name="__codelineno-0-289"></a><span class="sd">    [ValuationResult.variances][pydvl.value.result.ValuationResult.variances],</span>
-</span><span id="__span-0-290"><a id="__codelineno-0-290" name="__codelineno-0-290"></a><span class="sd">    [ValuationResult.counts][pydvl.value.result.ValuationResult.counts],</span>
-</span><span id="__span-0-291"><a id="__codelineno-0-291" name="__codelineno-0-291"></a><span class="sd">    [ValuationResult.indices][pydvl.value.result.ValuationResult.indices]</span>
-</span><span id="__span-0-292"><a id="__codelineno-0-292" name="__codelineno-0-292"></a><span class="sd">    and [ValuationResult.names][pydvl.value.result.ValuationResult.names]</span>
-</span><span id="__span-0-293"><a id="__codelineno-0-293" name="__codelineno-0-293"></a><span class="sd">    will follow the same order.</span>
-</span><span id="__span-0-294"><a id="__codelineno-0-294" name="__codelineno-0-294"></a>
-</span><span id="__span-0-295"><a id="__codelineno-0-295" name="__codelineno-0-295"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-296"><a id="__codelineno-0-296" name="__codelineno-0-296"></a><span class="sd">        reverse: Whether to sort in descending order by value.</span>
-</span><span id="__span-0-297"><a id="__codelineno-0-297" name="__codelineno-0-297"></a><span class="sd">        key: The key to sort by. Defaults to</span>
-</span><span id="__span-0-298"><a id="__codelineno-0-298" name="__codelineno-0-298"></a><span class="sd">            [ValueItem.value][pydvl.value.result.ValueItem].</span>
-</span><span id="__span-0-299"><a id="__codelineno-0-299" name="__codelineno-0-299"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-300"><a id="__codelineno-0-300" name="__codelineno-0-300"></a>    <span class="n">keymap</span> <span class="o">=</span> <span class="p">{</span>
-</span><span id="__span-0-301"><a id="__codelineno-0-301" name="__codelineno-0-301"></a>        <span class="s2">&quot;index&quot;</span><span class="p">:</span> <span class="s2">&quot;_indices&quot;</span><span class="p">,</span>
-</span><span id="__span-0-302"><a id="__codelineno-0-302" name="__codelineno-0-302"></a>        <span class="s2">&quot;value&quot;</span><span class="p">:</span> <span class="s2">&quot;_values&quot;</span><span class="p">,</span>
-</span><span id="__span-0-303"><a id="__codelineno-0-303" name="__codelineno-0-303"></a>        <span class="s2">&quot;variance&quot;</span><span class="p">:</span> <span class="s2">&quot;_variances&quot;</span><span class="p">,</span>
-</span><span id="__span-0-304"><a id="__codelineno-0-304" name="__codelineno-0-304"></a>        <span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;_names&quot;</span><span class="p">,</span>
-</span><span id="__span-0-305"><a id="__codelineno-0-305" name="__codelineno-0-305"></a>    <span class="p">}</span>
-</span><span id="__span-0-306"><a id="__codelineno-0-306" name="__codelineno-0-306"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">argsort</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">keymap</span><span class="p">[</span><span class="n">key</span><span class="p">]))</span>
-</span><span id="__span-0-307"><a id="__codelineno-0-307" name="__codelineno-0-307"></a>    <span class="k">if</span> <span class="n">reverse</span><span class="p">:</span>
-</span><span id="__span-0-308"><a id="__codelineno-0-308" name="__codelineno-0-308"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span><span class="p">[::</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
-</span><span id="__span-0-309"><a id="__codelineno-0-309" name="__codelineno-0-309"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_sort_order</span> <span class="o">=</span> <span class="n">reverse</span>
+<span class="normal"><a href="#__codelineno-0-303">303</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-272"><a id="__codelineno-0-272" name="__codelineno-0-272"></a><span class="k">def</span> <span class="nf">sort</span><span class="p">(</span>
+</span><span id="__span-0-273"><a id="__codelineno-0-273" name="__codelineno-0-273"></a>    <span class="bp">self</span><span class="p">,</span>
+</span><span id="__span-0-274"><a id="__codelineno-0-274" name="__codelineno-0-274"></a>    <span class="n">reverse</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+</span><span id="__span-0-275"><a id="__codelineno-0-275" name="__codelineno-0-275"></a>    <span class="c1"># Need a &quot;Comparable&quot; type here</span>
+</span><span id="__span-0-276"><a id="__codelineno-0-276" name="__codelineno-0-276"></a>    <span class="n">key</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;value&quot;</span><span class="p">,</span> <span class="s2">&quot;variance&quot;</span><span class="p">,</span> <span class="s2">&quot;index&quot;</span><span class="p">,</span> <span class="s2">&quot;name&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;value&quot;</span><span class="p">,</span>
+</span><span id="__span-0-277"><a id="__codelineno-0-277" name="__codelineno-0-277"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+</span><span id="__span-0-278"><a id="__codelineno-0-278" name="__codelineno-0-278"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Sorts the indices in place by `key`.</span>
+</span><span id="__span-0-279"><a id="__codelineno-0-279" name="__codelineno-0-279"></a>
+</span><span id="__span-0-280"><a id="__codelineno-0-280" name="__codelineno-0-280"></a><span class="sd">    Once sorted, iteration over the results, and indexing of all the</span>
+</span><span id="__span-0-281"><a id="__codelineno-0-281" name="__codelineno-0-281"></a><span class="sd">    properties</span>
+</span><span id="__span-0-282"><a id="__codelineno-0-282" name="__codelineno-0-282"></a><span class="sd">    [ValuationResult.values][pydvl.value.result.ValuationResult.values],</span>
+</span><span id="__span-0-283"><a id="__codelineno-0-283" name="__codelineno-0-283"></a><span class="sd">    [ValuationResult.variances][pydvl.value.result.ValuationResult.variances],</span>
+</span><span id="__span-0-284"><a id="__codelineno-0-284" name="__codelineno-0-284"></a><span class="sd">    [ValuationResult.counts][pydvl.value.result.ValuationResult.counts],</span>
+</span><span id="__span-0-285"><a id="__codelineno-0-285" name="__codelineno-0-285"></a><span class="sd">    [ValuationResult.indices][pydvl.value.result.ValuationResult.indices]</span>
+</span><span id="__span-0-286"><a id="__codelineno-0-286" name="__codelineno-0-286"></a><span class="sd">    and [ValuationResult.names][pydvl.value.result.ValuationResult.names]</span>
+</span><span id="__span-0-287"><a id="__codelineno-0-287" name="__codelineno-0-287"></a><span class="sd">    will follow the same order.</span>
+</span><span id="__span-0-288"><a id="__codelineno-0-288" name="__codelineno-0-288"></a>
+</span><span id="__span-0-289"><a id="__codelineno-0-289" name="__codelineno-0-289"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-290"><a id="__codelineno-0-290" name="__codelineno-0-290"></a><span class="sd">        reverse: Whether to sort in descending order by value.</span>
+</span><span id="__span-0-291"><a id="__codelineno-0-291" name="__codelineno-0-291"></a><span class="sd">        key: The key to sort by. Defaults to</span>
+</span><span id="__span-0-292"><a id="__codelineno-0-292" name="__codelineno-0-292"></a><span class="sd">            [ValueItem.value][pydvl.value.result.ValueItem].</span>
+</span><span id="__span-0-293"><a id="__codelineno-0-293" name="__codelineno-0-293"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-294"><a id="__codelineno-0-294" name="__codelineno-0-294"></a>    <span class="n">keymap</span> <span class="o">=</span> <span class="p">{</span>
+</span><span id="__span-0-295"><a id="__codelineno-0-295" name="__codelineno-0-295"></a>        <span class="s2">&quot;index&quot;</span><span class="p">:</span> <span class="s2">&quot;_indices&quot;</span><span class="p">,</span>
+</span><span id="__span-0-296"><a id="__codelineno-0-296" name="__codelineno-0-296"></a>        <span class="s2">&quot;value&quot;</span><span class="p">:</span> <span class="s2">&quot;_values&quot;</span><span class="p">,</span>
+</span><span id="__span-0-297"><a id="__codelineno-0-297" name="__codelineno-0-297"></a>        <span class="s2">&quot;variance&quot;</span><span class="p">:</span> <span class="s2">&quot;_variances&quot;</span><span class="p">,</span>
+</span><span id="__span-0-298"><a id="__codelineno-0-298" name="__codelineno-0-298"></a>        <span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;_names&quot;</span><span class="p">,</span>
+</span><span id="__span-0-299"><a id="__codelineno-0-299" name="__codelineno-0-299"></a>    <span class="p">}</span>
+</span><span id="__span-0-300"><a id="__codelineno-0-300" name="__codelineno-0-300"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">argsort</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">keymap</span><span class="p">[</span><span class="n">key</span><span class="p">]))</span>
+</span><span id="__span-0-301"><a id="__codelineno-0-301" name="__codelineno-0-301"></a>    <span class="k">if</span> <span class="n">reverse</span><span class="p">:</span>
+</span><span id="__span-0-302"><a id="__codelineno-0-302" name="__codelineno-0-302"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span><span class="p">[::</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+</span><span id="__span-0-303"><a id="__codelineno-0-303" name="__codelineno-0-303"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_sort_order</span> <span class="o">=</span> <span class="n">reverse</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3704,27 +3704,27 @@ <h3 id="pydvl.value.result.ValuationResult.__getattr__" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/result.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-358">358</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-352">352</a></span>
+<span class="normal"><a href="#__codelineno-0-353">353</a></span>
+<span class="normal"><a href="#__codelineno-0-354">354</a></span>
+<span class="normal"><a href="#__codelineno-0-355">355</a></span>
+<span class="normal"><a href="#__codelineno-0-356">356</a></span>
+<span class="normal"><a href="#__codelineno-0-357">357</a></span>
+<span class="normal"><a href="#__codelineno-0-358">358</a></span>
 <span class="normal"><a href="#__codelineno-0-359">359</a></span>
 <span class="normal"><a href="#__codelineno-0-360">360</a></span>
 <span class="normal"><a href="#__codelineno-0-361">361</a></span>
-<span class="normal"><a href="#__codelineno-0-362">362</a></span>
-<span class="normal"><a href="#__codelineno-0-363">363</a></span>
-<span class="normal"><a href="#__codelineno-0-364">364</a></span>
-<span class="normal"><a href="#__codelineno-0-365">365</a></span>
-<span class="normal"><a href="#__codelineno-0-366">366</a></span>
-<span class="normal"><a href="#__codelineno-0-367">367</a></span>
-<span class="normal"><a href="#__codelineno-0-368">368</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-358"><a id="__codelineno-0-358" name="__codelineno-0-358"></a><span class="k">def</span> <span class="fm">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attr</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Any</span><span class="p">:</span>
-</span><span id="__span-0-359"><a id="__codelineno-0-359" name="__codelineno-0-359"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Allows access to extra values as if they were properties of the instance.&quot;&quot;&quot;</span>
-</span><span id="__span-0-360"><a id="__codelineno-0-360" name="__codelineno-0-360"></a>    <span class="c1"># This is here to avoid a RecursionError when copying or pickling the object</span>
-</span><span id="__span-0-361"><a id="__codelineno-0-361" name="__codelineno-0-361"></a>    <span class="k">if</span> <span class="n">attr</span> <span class="o">==</span> <span class="s2">&quot;_extra_values&quot;</span><span class="p">:</span>
-</span><span id="__span-0-362"><a id="__codelineno-0-362" name="__codelineno-0-362"></a>        <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">()</span>
-</span><span id="__span-0-363"><a id="__codelineno-0-363" name="__codelineno-0-363"></a>    <span class="k">try</span><span class="p">:</span>
-</span><span id="__span-0-364"><a id="__codelineno-0-364" name="__codelineno-0-364"></a>        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extra_values</span><span class="p">[</span><span class="n">attr</span><span class="p">]</span>
-</span><span id="__span-0-365"><a id="__codelineno-0-365" name="__codelineno-0-365"></a>    <span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
-</span><span id="__span-0-366"><a id="__codelineno-0-366" name="__codelineno-0-366"></a>        <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span>
-</span><span id="__span-0-367"><a id="__codelineno-0-367" name="__codelineno-0-367"></a>            <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2"> object has no attribute </span><span class="si">{</span><span class="n">attr</span><span class="si">}</span><span class="s2">&quot;</span>
-</span><span id="__span-0-368"><a id="__codelineno-0-368" name="__codelineno-0-368"></a>        <span class="p">)</span> <span class="kn">from</span> <span class="nn">e</span>
+<span class="normal"><a href="#__codelineno-0-362">362</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-352"><a id="__codelineno-0-352" name="__codelineno-0-352"></a><span class="k">def</span> <span class="fm">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attr</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Any</span><span class="p">:</span>
+</span><span id="__span-0-353"><a id="__codelineno-0-353" name="__codelineno-0-353"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Allows access to extra values as if they were properties of the instance.&quot;&quot;&quot;</span>
+</span><span id="__span-0-354"><a id="__codelineno-0-354" name="__codelineno-0-354"></a>    <span class="c1"># This is here to avoid a RecursionError when copying or pickling the object</span>
+</span><span id="__span-0-355"><a id="__codelineno-0-355" name="__codelineno-0-355"></a>    <span class="k">if</span> <span class="n">attr</span> <span class="o">==</span> <span class="s2">&quot;_extra_values&quot;</span><span class="p">:</span>
+</span><span id="__span-0-356"><a id="__codelineno-0-356" name="__codelineno-0-356"></a>        <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">()</span>
+</span><span id="__span-0-357"><a id="__codelineno-0-357" name="__codelineno-0-357"></a>    <span class="k">try</span><span class="p">:</span>
+</span><span id="__span-0-358"><a id="__codelineno-0-358" name="__codelineno-0-358"></a>        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extra_values</span><span class="p">[</span><span class="n">attr</span><span class="p">]</span>
+</span><span id="__span-0-359"><a id="__codelineno-0-359" name="__codelineno-0-359"></a>    <span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+</span><span id="__span-0-360"><a id="__codelineno-0-360" name="__codelineno-0-360"></a>        <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span>
+</span><span id="__span-0-361"><a id="__codelineno-0-361" name="__codelineno-0-361"></a>            <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2"> object has no attribute </span><span class="si">{</span><span class="n">attr</span><span class="si">}</span><span class="s2">&quot;</span>
+</span><span id="__span-0-362"><a id="__codelineno-0-362" name="__codelineno-0-362"></a>        <span class="p">)</span> <span class="kn">from</span> <span class="nn">e</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3750,29 +3750,29 @@ <h3 id="pydvl.value.result.ValuationResult.__iter__" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/result.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-440">440</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-434">434</a></span>
+<span class="normal"><a href="#__codelineno-0-435">435</a></span>
+<span class="normal"><a href="#__codelineno-0-436">436</a></span>
+<span class="normal"><a href="#__codelineno-0-437">437</a></span>
+<span class="normal"><a href="#__codelineno-0-438">438</a></span>
+<span class="normal"><a href="#__codelineno-0-439">439</a></span>
+<span class="normal"><a href="#__codelineno-0-440">440</a></span>
 <span class="normal"><a href="#__codelineno-0-441">441</a></span>
 <span class="normal"><a href="#__codelineno-0-442">442</a></span>
 <span class="normal"><a href="#__codelineno-0-443">443</a></span>
 <span class="normal"><a href="#__codelineno-0-444">444</a></span>
-<span class="normal"><a href="#__codelineno-0-445">445</a></span>
-<span class="normal"><a href="#__codelineno-0-446">446</a></span>
-<span class="normal"><a href="#__codelineno-0-447">447</a></span>
-<span class="normal"><a href="#__codelineno-0-448">448</a></span>
-<span class="normal"><a href="#__codelineno-0-449">449</a></span>
-<span class="normal"><a href="#__codelineno-0-450">450</a></span>
-<span class="normal"><a href="#__codelineno-0-451">451</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-440"><a id="__codelineno-0-440" name="__codelineno-0-440"></a><span class="k">def</span> <span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Iterator</span><span class="p">[</span><span class="n">ValueItem</span><span class="p">[</span><span class="n">IndexT</span><span class="p">,</span> <span class="n">NameT</span><span class="p">]]:</span>
-</span><span id="__span-0-441"><a id="__codelineno-0-441" name="__codelineno-0-441"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Iterate over the results returning [ValueItem][pydvl.value.result.ValueItem] objects.</span>
-</span><span id="__span-0-442"><a id="__codelineno-0-442" name="__codelineno-0-442"></a><span class="sd">    To sort in place before iteration, use [sort()][pydvl.value.result.ValuationResult.sort].</span>
-</span><span id="__span-0-443"><a id="__codelineno-0-443" name="__codelineno-0-443"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-444"><a id="__codelineno-0-444" name="__codelineno-0-444"></a>    <span class="k">for</span> <span class="n">pos</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span><span class="p">:</span>
-</span><span id="__span-0-445"><a id="__codelineno-0-445" name="__codelineno-0-445"></a>        <span class="k">yield</span> <span class="n">ValueItem</span><span class="p">(</span>
-</span><span id="__span-0-446"><a id="__codelineno-0-446" name="__codelineno-0-446"></a>            <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
-</span><span id="__span-0-447"><a id="__codelineno-0-447" name="__codelineno-0-447"></a>            <span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
-</span><span id="__span-0-448"><a id="__codelineno-0-448" name="__codelineno-0-448"></a>            <span class="bp">self</span><span class="o">.</span><span class="n">_values</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
-</span><span id="__span-0-449"><a id="__codelineno-0-449" name="__codelineno-0-449"></a>            <span class="bp">self</span><span class="o">.</span><span class="n">_variances</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
-</span><span id="__span-0-450"><a id="__codelineno-0-450" name="__codelineno-0-450"></a>            <span class="bp">self</span><span class="o">.</span><span class="n">_counts</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
-</span><span id="__span-0-451"><a id="__codelineno-0-451" name="__codelineno-0-451"></a>        <span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-445">445</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-434"><a id="__codelineno-0-434" name="__codelineno-0-434"></a><span class="k">def</span> <span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Iterator</span><span class="p">[</span><span class="n">ValueItem</span><span class="p">[</span><span class="n">IndexT</span><span class="p">,</span> <span class="n">NameT</span><span class="p">]]:</span>
+</span><span id="__span-0-435"><a id="__codelineno-0-435" name="__codelineno-0-435"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Iterate over the results returning [ValueItem][pydvl.value.result.ValueItem] objects.</span>
+</span><span id="__span-0-436"><a id="__codelineno-0-436" name="__codelineno-0-436"></a><span class="sd">    To sort in place before iteration, use [sort()][pydvl.value.result.ValuationResult.sort].</span>
+</span><span id="__span-0-437"><a id="__codelineno-0-437" name="__codelineno-0-437"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-438"><a id="__codelineno-0-438" name="__codelineno-0-438"></a>    <span class="k">for</span> <span class="n">pos</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span><span class="p">:</span>
+</span><span id="__span-0-439"><a id="__codelineno-0-439" name="__codelineno-0-439"></a>        <span class="k">yield</span> <span class="n">ValueItem</span><span class="p">(</span>
+</span><span id="__span-0-440"><a id="__codelineno-0-440" name="__codelineno-0-440"></a>            <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
+</span><span id="__span-0-441"><a id="__codelineno-0-441" name="__codelineno-0-441"></a>            <span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
+</span><span id="__span-0-442"><a id="__codelineno-0-442" name="__codelineno-0-442"></a>            <span class="bp">self</span><span class="o">.</span><span class="n">_values</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
+</span><span id="__span-0-443"><a id="__codelineno-0-443" name="__codelineno-0-443"></a>            <span class="bp">self</span><span class="o">.</span><span class="n">_variances</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
+</span><span id="__span-0-444"><a id="__codelineno-0-444" name="__codelineno-0-444"></a>            <span class="bp">self</span><span class="o">.</span><span class="n">_counts</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
+</span><span id="__span-0-445"><a id="__codelineno-0-445" name="__codelineno-0-445"></a>        <span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3816,7 +3816,13 @@ <h3 id="pydvl.value.result.ValuationResult.__add__" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/result.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-495">495</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-489">489</a></span>
+<span class="normal"><a href="#__codelineno-0-490">490</a></span>
+<span class="normal"><a href="#__codelineno-0-491">491</a></span>
+<span class="normal"><a href="#__codelineno-0-492">492</a></span>
+<span class="normal"><a href="#__codelineno-0-493">493</a></span>
+<span class="normal"><a href="#__codelineno-0-494">494</a></span>
+<span class="normal"><a href="#__codelineno-0-495">495</a></span>
 <span class="normal"><a href="#__codelineno-0-496">496</a></span>
 <span class="normal"><a href="#__codelineno-0-497">497</a></span>
 <span class="normal"><a href="#__codelineno-0-498">498</a></span>
@@ -3919,118 +3925,116 @@ <h3 id="pydvl.value.result.ValuationResult.__add__" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-595">595</a></span>
 <span class="normal"><a href="#__codelineno-0-596">596</a></span>
 <span class="normal"><a href="#__codelineno-0-597">597</a></span>
-<span class="normal"><a href="#__codelineno-0-598">598</a></span>
-<span class="normal"><a href="#__codelineno-0-599">599</a></span>
-<span class="normal"><a href="#__codelineno-0-600">600</a></span>
-<span class="normal"><a href="#__codelineno-0-601">601</a></span>
-<span class="normal"><a href="#__codelineno-0-602">602</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-495"><a id="__codelineno-0-495" name="__codelineno-0-495"></a><span class="k">def</span> <span class="fm">__add__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="s2">&quot;ValuationResult&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ValuationResult&quot;</span><span class="p">:</span>
-</span><span id="__span-0-496"><a id="__codelineno-0-496" name="__codelineno-0-496"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Adds two ValuationResults.</span>
+<span class="normal"><a href="#__codelineno-0-598">598</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-489"><a id="__codelineno-0-489" name="__codelineno-0-489"></a><span class="k">def</span> <span class="fm">__add__</span><span class="p">(</span>
+</span><span id="__span-0-490"><a id="__codelineno-0-490" name="__codelineno-0-490"></a>    <span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="n">ValuationResult</span><span class="p">[</span><span class="n">IndexT</span><span class="p">,</span> <span class="n">NameT</span><span class="p">]</span>
+</span><span id="__span-0-491"><a id="__codelineno-0-491" name="__codelineno-0-491"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">[</span><span class="n">IndexT</span><span class="p">,</span> <span class="n">NameT</span><span class="p">]:</span>
+</span><span id="__span-0-492"><a id="__codelineno-0-492" name="__codelineno-0-492"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Adds two ValuationResults.</span>
+</span><span id="__span-0-493"><a id="__codelineno-0-493" name="__codelineno-0-493"></a>
+</span><span id="__span-0-494"><a id="__codelineno-0-494" name="__codelineno-0-494"></a><span class="sd">    The values must have been computed with the same algorithm. An exception</span>
+</span><span id="__span-0-495"><a id="__codelineno-0-495" name="__codelineno-0-495"></a><span class="sd">    to this is if one argument has empty values, in which case the other</span>
+</span><span id="__span-0-496"><a id="__codelineno-0-496" name="__codelineno-0-496"></a><span class="sd">    argument is returned.</span>
 </span><span id="__span-0-497"><a id="__codelineno-0-497" name="__codelineno-0-497"></a>
-</span><span id="__span-0-498"><a id="__codelineno-0-498" name="__codelineno-0-498"></a><span class="sd">    The values must have been computed with the same algorithm. An exception</span>
-</span><span id="__span-0-499"><a id="__codelineno-0-499" name="__codelineno-0-499"></a><span class="sd">    to this is if one argument has empty values, in which case the other</span>
-</span><span id="__span-0-500"><a id="__codelineno-0-500" name="__codelineno-0-500"></a><span class="sd">    argument is returned.</span>
-</span><span id="__span-0-501"><a id="__codelineno-0-501" name="__codelineno-0-501"></a>
-</span><span id="__span-0-502"><a id="__codelineno-0-502" name="__codelineno-0-502"></a><span class="sd">    !!! Warning</span>
-</span><span id="__span-0-503"><a id="__codelineno-0-503" name="__codelineno-0-503"></a><span class="sd">        Abusing this will introduce numerical errors.</span>
-</span><span id="__span-0-504"><a id="__codelineno-0-504" name="__codelineno-0-504"></a>
-</span><span id="__span-0-505"><a id="__codelineno-0-505" name="__codelineno-0-505"></a><span class="sd">    Means and standard errors are correctly handled. Statuses are added with</span>
-</span><span id="__span-0-506"><a id="__codelineno-0-506" name="__codelineno-0-506"></a><span class="sd">    bit-wise `&amp;`, see [Status][pydvl.value.result.Status].</span>
-</span><span id="__span-0-507"><a id="__codelineno-0-507" name="__codelineno-0-507"></a><span class="sd">    `data_names` are taken from the left summand, or if unavailable from</span>
-</span><span id="__span-0-508"><a id="__codelineno-0-508" name="__codelineno-0-508"></a><span class="sd">    the right one. The `algorithm` string is carried over if both terms</span>
-</span><span id="__span-0-509"><a id="__codelineno-0-509" name="__codelineno-0-509"></a><span class="sd">    have the same one or concatenated.</span>
+</span><span id="__span-0-498"><a id="__codelineno-0-498" name="__codelineno-0-498"></a><span class="sd">    !!! Warning</span>
+</span><span id="__span-0-499"><a id="__codelineno-0-499" name="__codelineno-0-499"></a><span class="sd">        Abusing this will introduce numerical errors.</span>
+</span><span id="__span-0-500"><a id="__codelineno-0-500" name="__codelineno-0-500"></a>
+</span><span id="__span-0-501"><a id="__codelineno-0-501" name="__codelineno-0-501"></a><span class="sd">    Means and standard errors are correctly handled. Statuses are added with</span>
+</span><span id="__span-0-502"><a id="__codelineno-0-502" name="__codelineno-0-502"></a><span class="sd">    bit-wise `&amp;`, see [Status][pydvl.value.result.Status].</span>
+</span><span id="__span-0-503"><a id="__codelineno-0-503" name="__codelineno-0-503"></a><span class="sd">    `data_names` are taken from the left summand, or if unavailable from</span>
+</span><span id="__span-0-504"><a id="__codelineno-0-504" name="__codelineno-0-504"></a><span class="sd">    the right one. The `algorithm` string is carried over if both terms</span>
+</span><span id="__span-0-505"><a id="__codelineno-0-505" name="__codelineno-0-505"></a><span class="sd">    have the same one or concatenated.</span>
+</span><span id="__span-0-506"><a id="__codelineno-0-506" name="__codelineno-0-506"></a>
+</span><span id="__span-0-507"><a id="__codelineno-0-507" name="__codelineno-0-507"></a><span class="sd">    It is possible to add ValuationResults of different lengths, and with</span>
+</span><span id="__span-0-508"><a id="__codelineno-0-508" name="__codelineno-0-508"></a><span class="sd">    different or overlapping indices. The result will have the union of</span>
+</span><span id="__span-0-509"><a id="__codelineno-0-509" name="__codelineno-0-509"></a><span class="sd">    indices, and the values.</span>
 </span><span id="__span-0-510"><a id="__codelineno-0-510" name="__codelineno-0-510"></a>
-</span><span id="__span-0-511"><a id="__codelineno-0-511" name="__codelineno-0-511"></a><span class="sd">    It is possible to add ValuationResults of different lengths, and with</span>
-</span><span id="__span-0-512"><a id="__codelineno-0-512" name="__codelineno-0-512"></a><span class="sd">    different or overlapping indices. The result will have the union of</span>
-</span><span id="__span-0-513"><a id="__codelineno-0-513" name="__codelineno-0-513"></a><span class="sd">    indices, and the values.</span>
-</span><span id="__span-0-514"><a id="__codelineno-0-514" name="__codelineno-0-514"></a>
-</span><span id="__span-0-515"><a id="__codelineno-0-515" name="__codelineno-0-515"></a><span class="sd">    !!! Warning</span>
-</span><span id="__span-0-516"><a id="__codelineno-0-516" name="__codelineno-0-516"></a><span class="sd">        FIXME: Arbitrary `extra_values` aren&#39;t handled.</span>
-</span><span id="__span-0-517"><a id="__codelineno-0-517" name="__codelineno-0-517"></a>
-</span><span id="__span-0-518"><a id="__codelineno-0-518" name="__codelineno-0-518"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-519"><a id="__codelineno-0-519" name="__codelineno-0-519"></a>    <span class="c1"># empty results</span>
-</span><span id="__span-0-520"><a id="__codelineno-0-520" name="__codelineno-0-520"></a>    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
-</span><span id="__span-0-521"><a id="__codelineno-0-521" name="__codelineno-0-521"></a>        <span class="k">return</span> <span class="n">other</span>
-</span><span id="__span-0-522"><a id="__codelineno-0-522" name="__codelineno-0-522"></a>    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">values</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
-</span><span id="__span-0-523"><a id="__codelineno-0-523" name="__codelineno-0-523"></a>        <span class="k">return</span> <span class="bp">self</span>
-</span><span id="__span-0-524"><a id="__codelineno-0-524" name="__codelineno-0-524"></a>
-</span><span id="__span-0-525"><a id="__codelineno-0-525" name="__codelineno-0-525"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_check_compatible</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+</span><span id="__span-0-511"><a id="__codelineno-0-511" name="__codelineno-0-511"></a><span class="sd">    !!! Warning</span>
+</span><span id="__span-0-512"><a id="__codelineno-0-512" name="__codelineno-0-512"></a><span class="sd">        FIXME: Arbitrary `extra_values` aren&#39;t handled.</span>
+</span><span id="__span-0-513"><a id="__codelineno-0-513" name="__codelineno-0-513"></a>
+</span><span id="__span-0-514"><a id="__codelineno-0-514" name="__codelineno-0-514"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-515"><a id="__codelineno-0-515" name="__codelineno-0-515"></a>    <span class="c1"># empty results</span>
+</span><span id="__span-0-516"><a id="__codelineno-0-516" name="__codelineno-0-516"></a>    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+</span><span id="__span-0-517"><a id="__codelineno-0-517" name="__codelineno-0-517"></a>        <span class="k">return</span> <span class="n">other</span>
+</span><span id="__span-0-518"><a id="__codelineno-0-518" name="__codelineno-0-518"></a>    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">values</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+</span><span id="__span-0-519"><a id="__codelineno-0-519" name="__codelineno-0-519"></a>        <span class="k">return</span> <span class="bp">self</span>
+</span><span id="__span-0-520"><a id="__codelineno-0-520" name="__codelineno-0-520"></a>
+</span><span id="__span-0-521"><a id="__codelineno-0-521" name="__codelineno-0-521"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_check_compatible</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+</span><span id="__span-0-522"><a id="__codelineno-0-522" name="__codelineno-0-522"></a>
+</span><span id="__span-0-523"><a id="__codelineno-0-523" name="__codelineno-0-523"></a>    <span class="n">indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">union1d</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_indices</span><span class="p">,</span> <span class="n">other</span><span class="o">.</span><span class="n">_indices</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_indices</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
+</span><span id="__span-0-524"><a id="__codelineno-0-524" name="__codelineno-0-524"></a>    <span class="n">this_pos</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">searchsorted</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span><span class="p">)</span>
+</span><span id="__span-0-525"><a id="__codelineno-0-525" name="__codelineno-0-525"></a>    <span class="n">other_pos</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">searchsorted</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">other</span><span class="o">.</span><span class="n">_indices</span><span class="p">)</span>
 </span><span id="__span-0-526"><a id="__codelineno-0-526" name="__codelineno-0-526"></a>
-</span><span id="__span-0-527"><a id="__codelineno-0-527" name="__codelineno-0-527"></a>    <span class="n">indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">union1d</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_indices</span><span class="p">,</span> <span class="n">other</span><span class="o">.</span><span class="n">_indices</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_indices</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
-</span><span id="__span-0-528"><a id="__codelineno-0-528" name="__codelineno-0-528"></a>    <span class="n">this_pos</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">searchsorted</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span><span class="p">)</span>
-</span><span id="__span-0-529"><a id="__codelineno-0-529" name="__codelineno-0-529"></a>    <span class="n">other_pos</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">searchsorted</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">other</span><span class="o">.</span><span class="n">_indices</span><span class="p">)</span>
-</span><span id="__span-0-530"><a id="__codelineno-0-530" name="__codelineno-0-530"></a>
-</span><span id="__span-0-531"><a id="__codelineno-0-531" name="__codelineno-0-531"></a>    <span class="n">n</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">int</span><span class="p">)</span>
-</span><span id="__span-0-532"><a id="__codelineno-0-532" name="__codelineno-0-532"></a>    <span class="n">m</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">int</span><span class="p">)</span>
-</span><span id="__span-0-533"><a id="__codelineno-0-533" name="__codelineno-0-533"></a>    <span class="n">xn</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">float</span><span class="p">)</span>
-</span><span id="__span-0-534"><a id="__codelineno-0-534" name="__codelineno-0-534"></a>    <span class="n">xm</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">float</span><span class="p">)</span>
-</span><span id="__span-0-535"><a id="__codelineno-0-535" name="__codelineno-0-535"></a>    <span class="n">vn</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">float</span><span class="p">)</span>
-</span><span id="__span-0-536"><a id="__codelineno-0-536" name="__codelineno-0-536"></a>    <span class="n">vm</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">float</span><span class="p">)</span>
-</span><span id="__span-0-537"><a id="__codelineno-0-537" name="__codelineno-0-537"></a>
-</span><span id="__span-0-538"><a id="__codelineno-0-538" name="__codelineno-0-538"></a>    <span class="n">n</span><span class="p">[</span><span class="n">this_pos</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_counts</span>
-</span><span id="__span-0-539"><a id="__codelineno-0-539" name="__codelineno-0-539"></a>    <span class="n">xn</span><span class="p">[</span><span class="n">this_pos</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_values</span>
-</span><span id="__span-0-540"><a id="__codelineno-0-540" name="__codelineno-0-540"></a>    <span class="n">vn</span><span class="p">[</span><span class="n">this_pos</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_variances</span>
-</span><span id="__span-0-541"><a id="__codelineno-0-541" name="__codelineno-0-541"></a>    <span class="n">m</span><span class="p">[</span><span class="n">other_pos</span><span class="p">]</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">_counts</span>
-</span><span id="__span-0-542"><a id="__codelineno-0-542" name="__codelineno-0-542"></a>    <span class="n">xm</span><span class="p">[</span><span class="n">other_pos</span><span class="p">]</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">_values</span>
-</span><span id="__span-0-543"><a id="__codelineno-0-543" name="__codelineno-0-543"></a>    <span class="n">vm</span><span class="p">[</span><span class="n">other_pos</span><span class="p">]</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">_variances</span>
-</span><span id="__span-0-544"><a id="__codelineno-0-544" name="__codelineno-0-544"></a>
-</span><span id="__span-0-545"><a id="__codelineno-0-545" name="__codelineno-0-545"></a>    <span class="c1"># Sample mean of n+m samples from two means of n and m samples</span>
-</span><span id="__span-0-546"><a id="__codelineno-0-546" name="__codelineno-0-546"></a>    <span class="n">xnm</span> <span class="o">=</span> <span class="p">(</span><span class="n">n</span> <span class="o">*</span> <span class="n">xn</span> <span class="o">+</span> <span class="n">m</span> <span class="o">*</span> <span class="n">xm</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="n">n</span> <span class="o">+</span> <span class="n">m</span><span class="p">)</span>
-</span><span id="__span-0-547"><a id="__codelineno-0-547" name="__codelineno-0-547"></a>    <span class="c1"># Sample variance of n+m samples from two sample variances of n and m samples</span>
-</span><span id="__span-0-548"><a id="__codelineno-0-548" name="__codelineno-0-548"></a>    <span class="n">vnm</span> <span class="o">=</span> <span class="p">(</span><span class="n">n</span> <span class="o">*</span> <span class="p">(</span><span class="n">vn</span> <span class="o">+</span> <span class="n">xn</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="n">m</span> <span class="o">*</span> <span class="p">(</span><span class="n">vm</span> <span class="o">+</span> <span class="n">xm</span><span class="o">**</span><span class="mi">2</span><span class="p">))</span> <span class="o">/</span> <span class="p">(</span><span class="n">n</span> <span class="o">+</span> <span class="n">m</span><span class="p">)</span> <span class="o">-</span> <span class="n">xnm</span><span class="o">**</span><span class="mi">2</span>
-</span><span id="__span-0-549"><a id="__codelineno-0-549" name="__codelineno-0-549"></a>
-</span><span id="__span-0-550"><a id="__codelineno-0-550" name="__codelineno-0-550"></a>    <span class="k">if</span> <span class="n">np</span><span class="o">.</span><span class="n">any</span><span class="p">(</span><span class="n">vnm</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">):</span>
-</span><span id="__span-0-551"><a id="__codelineno-0-551" name="__codelineno-0-551"></a>        <span class="k">if</span> <span class="n">np</span><span class="o">.</span><span class="n">any</span><span class="p">(</span><span class="n">vnm</span> <span class="o">&lt;</span> <span class="o">-</span><span class="mf">1e-6</span><span class="p">):</span>
-</span><span id="__span-0-552"><a id="__codelineno-0-552" name="__codelineno-0-552"></a>            <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
-</span><span id="__span-0-553"><a id="__codelineno-0-553" name="__codelineno-0-553"></a>                <span class="s2">&quot;Numerical error in variance computation. &quot;</span>
-</span><span id="__span-0-554"><a id="__codelineno-0-554" name="__codelineno-0-554"></a>                <span class="sa">f</span><span class="s2">&quot;Negative sample variances clipped to 0 in </span><span class="si">{</span><span class="n">vnm</span><span class="si">}</span><span class="s2">&quot;</span>
-</span><span id="__span-0-555"><a id="__codelineno-0-555" name="__codelineno-0-555"></a>            <span class="p">)</span>
-</span><span id="__span-0-556"><a id="__codelineno-0-556" name="__codelineno-0-556"></a>        <span class="n">vnm</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">vnm</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">)]</span> <span class="o">=</span> <span class="mi">0</span>
-</span><span id="__span-0-557"><a id="__codelineno-0-557" name="__codelineno-0-557"></a>
-</span><span id="__span-0-558"><a id="__codelineno-0-558" name="__codelineno-0-558"></a>    <span class="c1"># Merging of names:</span>
-</span><span id="__span-0-559"><a id="__codelineno-0-559" name="__codelineno-0-559"></a>    <span class="c1"># If an index has the same name in both results, it must be the same.</span>
-</span><span id="__span-0-560"><a id="__codelineno-0-560" name="__codelineno-0-560"></a>    <span class="c1"># If an index has a name in one result but not the other, the name is</span>
-</span><span id="__span-0-561"><a id="__codelineno-0-561" name="__codelineno-0-561"></a>    <span class="c1"># taken from the result with the name.</span>
-</span><span id="__span-0-562"><a id="__codelineno-0-562" name="__codelineno-0-562"></a>    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span> <span class="o">!=</span> <span class="n">other</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="p">:</span>
-</span><span id="__span-0-563"><a id="__codelineno-0-563" name="__codelineno-0-563"></a>        <span class="k">if</span> <span class="n">np</span><span class="o">.</span><span class="n">can_cast</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="p">,</span> <span class="n">casting</span><span class="o">=</span><span class="s2">&quot;safe&quot;</span><span class="p">):</span>
-</span><span id="__span-0-564"><a id="__codelineno-0-564" name="__codelineno-0-564"></a>            <span class="n">other</span><span class="o">.</span><span class="n">_names</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
-</span><span id="__span-0-565"><a id="__codelineno-0-565" name="__codelineno-0-565"></a>            <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
-</span><span id="__span-0-566"><a id="__codelineno-0-566" name="__codelineno-0-566"></a>                <span class="sa">f</span><span class="s2">&quot;Casting ValuationResult.names from </span><span class="si">{</span><span class="n">other</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="si">}</span><span class="s2">&quot;</span>
-</span><span id="__span-0-567"><a id="__codelineno-0-567" name="__codelineno-0-567"></a>            <span class="p">)</span>
-</span><span id="__span-0-568"><a id="__codelineno-0-568" name="__codelineno-0-568"></a>        <span class="k">else</span><span class="p">:</span>
-</span><span id="__span-0-569"><a id="__codelineno-0-569" name="__codelineno-0-569"></a>            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span>
-</span><span id="__span-0-570"><a id="__codelineno-0-570" name="__codelineno-0-570"></a>                <span class="sa">f</span><span class="s2">&quot;Cannot cast ValuationResult.names from &quot;</span>
-</span><span id="__span-0-571"><a id="__codelineno-0-571" name="__codelineno-0-571"></a>                <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">other</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="si">}</span><span class="s2">&quot;</span>
-</span><span id="__span-0-572"><a id="__codelineno-0-572" name="__codelineno-0-572"></a>            <span class="p">)</span>
-</span><span id="__span-0-573"><a id="__codelineno-0-573" name="__codelineno-0-573"></a>
-</span><span id="__span-0-574"><a id="__codelineno-0-574" name="__codelineno-0-574"></a>    <span class="n">both_pos</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">intersect1d</span><span class="p">(</span><span class="n">this_pos</span><span class="p">,</span> <span class="n">other_pos</span><span class="p">)</span>
-</span><span id="__span-0-575"><a id="__codelineno-0-575" name="__codelineno-0-575"></a>
-</span><span id="__span-0-576"><a id="__codelineno-0-576" name="__codelineno-0-576"></a>    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">both_pos</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
-</span><span id="__span-0-577"><a id="__codelineno-0-577" name="__codelineno-0-577"></a>        <span class="n">this_names</span><span class="p">:</span> <span class="n">NDArray</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">empty_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">object</span><span class="p">)</span>
-</span><span id="__span-0-578"><a id="__codelineno-0-578" name="__codelineno-0-578"></a>        <span class="n">other_names</span><span class="p">:</span> <span class="n">NDArray</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">empty_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">object</span><span class="p">)</span>
-</span><span id="__span-0-579"><a id="__codelineno-0-579" name="__codelineno-0-579"></a>        <span class="n">this_names</span><span class="p">[</span><span class="n">this_pos</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_names</span>
-</span><span id="__span-0-580"><a id="__codelineno-0-580" name="__codelineno-0-580"></a>        <span class="n">other_names</span><span class="p">[</span><span class="n">other_pos</span><span class="p">]</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">_names</span>
-</span><span id="__span-0-581"><a id="__codelineno-0-581" name="__codelineno-0-581"></a>
-</span><span id="__span-0-582"><a id="__codelineno-0-582" name="__codelineno-0-582"></a>        <span class="n">this_shared_names</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">take</span><span class="p">(</span><span class="n">this_names</span><span class="p">,</span> <span class="n">both_pos</span><span class="p">)</span>
-</span><span id="__span-0-583"><a id="__codelineno-0-583" name="__codelineno-0-583"></a>        <span class="n">other_shared_names</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">take</span><span class="p">(</span><span class="n">other_names</span><span class="p">,</span> <span class="n">both_pos</span><span class="p">)</span>
-</span><span id="__span-0-584"><a id="__codelineno-0-584" name="__codelineno-0-584"></a>
-</span><span id="__span-0-585"><a id="__codelineno-0-585" name="__codelineno-0-585"></a>        <span class="k">if</span> <span class="n">np</span><span class="o">.</span><span class="n">any</span><span class="p">(</span><span class="n">this_shared_names</span> <span class="o">!=</span> <span class="n">other_shared_names</span><span class="p">):</span>
-</span><span id="__span-0-586"><a id="__codelineno-0-586" name="__codelineno-0-586"></a>            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Mismatching names in ValuationResults&quot;</span><span class="p">)</span>
+</span><span id="__span-0-527"><a id="__codelineno-0-527" name="__codelineno-0-527"></a>    <span class="n">n</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">int</span><span class="p">)</span>
+</span><span id="__span-0-528"><a id="__codelineno-0-528" name="__codelineno-0-528"></a>    <span class="n">m</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">int</span><span class="p">)</span>
+</span><span id="__span-0-529"><a id="__codelineno-0-529" name="__codelineno-0-529"></a>    <span class="n">xn</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">float</span><span class="p">)</span>
+</span><span id="__span-0-530"><a id="__codelineno-0-530" name="__codelineno-0-530"></a>    <span class="n">xm</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">float</span><span class="p">)</span>
+</span><span id="__span-0-531"><a id="__codelineno-0-531" name="__codelineno-0-531"></a>    <span class="n">vn</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">float</span><span class="p">)</span>
+</span><span id="__span-0-532"><a id="__codelineno-0-532" name="__codelineno-0-532"></a>    <span class="n">vm</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">float</span><span class="p">)</span>
+</span><span id="__span-0-533"><a id="__codelineno-0-533" name="__codelineno-0-533"></a>
+</span><span id="__span-0-534"><a id="__codelineno-0-534" name="__codelineno-0-534"></a>    <span class="n">n</span><span class="p">[</span><span class="n">this_pos</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_counts</span>
+</span><span id="__span-0-535"><a id="__codelineno-0-535" name="__codelineno-0-535"></a>    <span class="n">xn</span><span class="p">[</span><span class="n">this_pos</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_values</span>
+</span><span id="__span-0-536"><a id="__codelineno-0-536" name="__codelineno-0-536"></a>    <span class="n">vn</span><span class="p">[</span><span class="n">this_pos</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_variances</span>
+</span><span id="__span-0-537"><a id="__codelineno-0-537" name="__codelineno-0-537"></a>    <span class="n">m</span><span class="p">[</span><span class="n">other_pos</span><span class="p">]</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">_counts</span>
+</span><span id="__span-0-538"><a id="__codelineno-0-538" name="__codelineno-0-538"></a>    <span class="n">xm</span><span class="p">[</span><span class="n">other_pos</span><span class="p">]</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">_values</span>
+</span><span id="__span-0-539"><a id="__codelineno-0-539" name="__codelineno-0-539"></a>    <span class="n">vm</span><span class="p">[</span><span class="n">other_pos</span><span class="p">]</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">_variances</span>
+</span><span id="__span-0-540"><a id="__codelineno-0-540" name="__codelineno-0-540"></a>
+</span><span id="__span-0-541"><a id="__codelineno-0-541" name="__codelineno-0-541"></a>    <span class="c1"># Sample mean of n+m samples from two means of n and m samples</span>
+</span><span id="__span-0-542"><a id="__codelineno-0-542" name="__codelineno-0-542"></a>    <span class="n">xnm</span> <span class="o">=</span> <span class="p">(</span><span class="n">n</span> <span class="o">*</span> <span class="n">xn</span> <span class="o">+</span> <span class="n">m</span> <span class="o">*</span> <span class="n">xm</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="n">n</span> <span class="o">+</span> <span class="n">m</span><span class="p">)</span>
+</span><span id="__span-0-543"><a id="__codelineno-0-543" name="__codelineno-0-543"></a>    <span class="c1"># Sample variance of n+m samples from two sample variances of n and m samples</span>
+</span><span id="__span-0-544"><a id="__codelineno-0-544" name="__codelineno-0-544"></a>    <span class="n">vnm</span> <span class="o">=</span> <span class="p">(</span><span class="n">n</span> <span class="o">*</span> <span class="p">(</span><span class="n">vn</span> <span class="o">+</span> <span class="n">xn</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="n">m</span> <span class="o">*</span> <span class="p">(</span><span class="n">vm</span> <span class="o">+</span> <span class="n">xm</span><span class="o">**</span><span class="mi">2</span><span class="p">))</span> <span class="o">/</span> <span class="p">(</span><span class="n">n</span> <span class="o">+</span> <span class="n">m</span><span class="p">)</span> <span class="o">-</span> <span class="n">xnm</span><span class="o">**</span><span class="mi">2</span>
+</span><span id="__span-0-545"><a id="__codelineno-0-545" name="__codelineno-0-545"></a>
+</span><span id="__span-0-546"><a id="__codelineno-0-546" name="__codelineno-0-546"></a>    <span class="k">if</span> <span class="n">np</span><span class="o">.</span><span class="n">any</span><span class="p">(</span><span class="n">vnm</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">):</span>
+</span><span id="__span-0-547"><a id="__codelineno-0-547" name="__codelineno-0-547"></a>        <span class="k">if</span> <span class="n">np</span><span class="o">.</span><span class="n">any</span><span class="p">(</span><span class="n">vnm</span> <span class="o">&lt;</span> <span class="o">-</span><span class="mf">1e-6</span><span class="p">):</span>
+</span><span id="__span-0-548"><a id="__codelineno-0-548" name="__codelineno-0-548"></a>            <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
+</span><span id="__span-0-549"><a id="__codelineno-0-549" name="__codelineno-0-549"></a>                <span class="s2">&quot;Numerical error in variance computation. &quot;</span>
+</span><span id="__span-0-550"><a id="__codelineno-0-550" name="__codelineno-0-550"></a>                <span class="sa">f</span><span class="s2">&quot;Negative sample variances clipped to 0 in </span><span class="si">{</span><span class="n">vnm</span><span class="si">}</span><span class="s2">&quot;</span>
+</span><span id="__span-0-551"><a id="__codelineno-0-551" name="__codelineno-0-551"></a>            <span class="p">)</span>
+</span><span id="__span-0-552"><a id="__codelineno-0-552" name="__codelineno-0-552"></a>        <span class="n">vnm</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">vnm</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">)]</span> <span class="o">=</span> <span class="mi">0</span>
+</span><span id="__span-0-553"><a id="__codelineno-0-553" name="__codelineno-0-553"></a>
+</span><span id="__span-0-554"><a id="__codelineno-0-554" name="__codelineno-0-554"></a>    <span class="c1"># Merging of names:</span>
+</span><span id="__span-0-555"><a id="__codelineno-0-555" name="__codelineno-0-555"></a>    <span class="c1"># If an index has the same name in both results, it must be the same.</span>
+</span><span id="__span-0-556"><a id="__codelineno-0-556" name="__codelineno-0-556"></a>    <span class="c1"># If an index has a name in one result but not the other, the name is</span>
+</span><span id="__span-0-557"><a id="__codelineno-0-557" name="__codelineno-0-557"></a>    <span class="c1"># taken from the result with the name.</span>
+</span><span id="__span-0-558"><a id="__codelineno-0-558" name="__codelineno-0-558"></a>    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span> <span class="o">!=</span> <span class="n">other</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="p">:</span>
+</span><span id="__span-0-559"><a id="__codelineno-0-559" name="__codelineno-0-559"></a>        <span class="k">if</span> <span class="n">np</span><span class="o">.</span><span class="n">can_cast</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="p">,</span> <span class="n">casting</span><span class="o">=</span><span class="s2">&quot;safe&quot;</span><span class="p">):</span>
+</span><span id="__span-0-560"><a id="__codelineno-0-560" name="__codelineno-0-560"></a>            <span class="n">other</span><span class="o">.</span><span class="n">_names</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
+</span><span id="__span-0-561"><a id="__codelineno-0-561" name="__codelineno-0-561"></a>            <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
+</span><span id="__span-0-562"><a id="__codelineno-0-562" name="__codelineno-0-562"></a>                <span class="sa">f</span><span class="s2">&quot;Casting ValuationResult.names from </span><span class="si">{</span><span class="n">other</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="si">}</span><span class="s2">&quot;</span>
+</span><span id="__span-0-563"><a id="__codelineno-0-563" name="__codelineno-0-563"></a>            <span class="p">)</span>
+</span><span id="__span-0-564"><a id="__codelineno-0-564" name="__codelineno-0-564"></a>        <span class="k">else</span><span class="p">:</span>
+</span><span id="__span-0-565"><a id="__codelineno-0-565" name="__codelineno-0-565"></a>            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span>
+</span><span id="__span-0-566"><a id="__codelineno-0-566" name="__codelineno-0-566"></a>                <span class="sa">f</span><span class="s2">&quot;Cannot cast ValuationResult.names from &quot;</span>
+</span><span id="__span-0-567"><a id="__codelineno-0-567" name="__codelineno-0-567"></a>                <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">other</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="si">}</span><span class="s2">&quot;</span>
+</span><span id="__span-0-568"><a id="__codelineno-0-568" name="__codelineno-0-568"></a>            <span class="p">)</span>
+</span><span id="__span-0-569"><a id="__codelineno-0-569" name="__codelineno-0-569"></a>
+</span><span id="__span-0-570"><a id="__codelineno-0-570" name="__codelineno-0-570"></a>    <span class="n">both_pos</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">intersect1d</span><span class="p">(</span><span class="n">this_pos</span><span class="p">,</span> <span class="n">other_pos</span><span class="p">)</span>
+</span><span id="__span-0-571"><a id="__codelineno-0-571" name="__codelineno-0-571"></a>
+</span><span id="__span-0-572"><a id="__codelineno-0-572" name="__codelineno-0-572"></a>    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">both_pos</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+</span><span id="__span-0-573"><a id="__codelineno-0-573" name="__codelineno-0-573"></a>        <span class="n">this_names</span><span class="p">:</span> <span class="n">NDArray</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">empty_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">object</span><span class="p">)</span>
+</span><span id="__span-0-574"><a id="__codelineno-0-574" name="__codelineno-0-574"></a>        <span class="n">other_names</span><span class="p">:</span> <span class="n">NDArray</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">empty_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">object</span><span class="p">)</span>
+</span><span id="__span-0-575"><a id="__codelineno-0-575" name="__codelineno-0-575"></a>        <span class="n">this_names</span><span class="p">[</span><span class="n">this_pos</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_names</span>
+</span><span id="__span-0-576"><a id="__codelineno-0-576" name="__codelineno-0-576"></a>        <span class="n">other_names</span><span class="p">[</span><span class="n">other_pos</span><span class="p">]</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">_names</span>
+</span><span id="__span-0-577"><a id="__codelineno-0-577" name="__codelineno-0-577"></a>
+</span><span id="__span-0-578"><a id="__codelineno-0-578" name="__codelineno-0-578"></a>        <span class="n">this_shared_names</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">take</span><span class="p">(</span><span class="n">this_names</span><span class="p">,</span> <span class="n">both_pos</span><span class="p">)</span>
+</span><span id="__span-0-579"><a id="__codelineno-0-579" name="__codelineno-0-579"></a>        <span class="n">other_shared_names</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">take</span><span class="p">(</span><span class="n">other_names</span><span class="p">,</span> <span class="n">both_pos</span><span class="p">)</span>
+</span><span id="__span-0-580"><a id="__codelineno-0-580" name="__codelineno-0-580"></a>
+</span><span id="__span-0-581"><a id="__codelineno-0-581" name="__codelineno-0-581"></a>        <span class="k">if</span> <span class="n">np</span><span class="o">.</span><span class="n">any</span><span class="p">(</span><span class="n">this_shared_names</span> <span class="o">!=</span> <span class="n">other_shared_names</span><span class="p">):</span>
+</span><span id="__span-0-582"><a id="__codelineno-0-582" name="__codelineno-0-582"></a>            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Mismatching names in ValuationResults&quot;</span><span class="p">)</span>
+</span><span id="__span-0-583"><a id="__codelineno-0-583" name="__codelineno-0-583"></a>
+</span><span id="__span-0-584"><a id="__codelineno-0-584" name="__codelineno-0-584"></a>    <span class="n">names</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">empty_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
+</span><span id="__span-0-585"><a id="__codelineno-0-585" name="__codelineno-0-585"></a>    <span class="n">names</span><span class="p">[</span><span class="n">this_pos</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_names</span>
+</span><span id="__span-0-586"><a id="__codelineno-0-586" name="__codelineno-0-586"></a>    <span class="n">names</span><span class="p">[</span><span class="n">other_pos</span><span class="p">]</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">_names</span>
 </span><span id="__span-0-587"><a id="__codelineno-0-587" name="__codelineno-0-587"></a>
-</span><span id="__span-0-588"><a id="__codelineno-0-588" name="__codelineno-0-588"></a>    <span class="n">names</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">empty_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
-</span><span id="__span-0-589"><a id="__codelineno-0-589" name="__codelineno-0-589"></a>    <span class="n">names</span><span class="p">[</span><span class="n">this_pos</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_names</span>
-</span><span id="__span-0-590"><a id="__codelineno-0-590" name="__codelineno-0-590"></a>    <span class="n">names</span><span class="p">[</span><span class="n">other_pos</span><span class="p">]</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">_names</span>
-</span><span id="__span-0-591"><a id="__codelineno-0-591" name="__codelineno-0-591"></a>
-</span><span id="__span-0-592"><a id="__codelineno-0-592" name="__codelineno-0-592"></a>    <span class="k">return</span> <span class="n">ValuationResult</span><span class="p">(</span>
-</span><span id="__span-0-593"><a id="__codelineno-0-593" name="__codelineno-0-593"></a>        <span class="n">algorithm</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">algorithm</span> <span class="ow">or</span> <span class="n">other</span><span class="o">.</span><span class="n">algorithm</span> <span class="ow">or</span> <span class="s2">&quot;&quot;</span><span class="p">,</span>
-</span><span id="__span-0-594"><a id="__codelineno-0-594" name="__codelineno-0-594"></a>        <span class="n">status</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">status</span> <span class="o">&amp;</span> <span class="n">other</span><span class="o">.</span><span class="n">status</span><span class="p">,</span>
-</span><span id="__span-0-595"><a id="__codelineno-0-595" name="__codelineno-0-595"></a>        <span class="n">indices</span><span class="o">=</span><span class="n">indices</span><span class="p">,</span>
-</span><span id="__span-0-596"><a id="__codelineno-0-596" name="__codelineno-0-596"></a>        <span class="n">values</span><span class="o">=</span><span class="n">xnm</span><span class="p">,</span>
-</span><span id="__span-0-597"><a id="__codelineno-0-597" name="__codelineno-0-597"></a>        <span class="n">variances</span><span class="o">=</span><span class="n">vnm</span><span class="p">,</span>
-</span><span id="__span-0-598"><a id="__codelineno-0-598" name="__codelineno-0-598"></a>        <span class="n">counts</span><span class="o">=</span><span class="n">n</span> <span class="o">+</span> <span class="n">m</span><span class="p">,</span>
-</span><span id="__span-0-599"><a id="__codelineno-0-599" name="__codelineno-0-599"></a>        <span class="n">data_names</span><span class="o">=</span><span class="n">names</span><span class="p">,</span>
-</span><span id="__span-0-600"><a id="__codelineno-0-600" name="__codelineno-0-600"></a>        <span class="c1"># FIXME: What to do with extra_values? This is not commutative:</span>
-</span><span id="__span-0-601"><a id="__codelineno-0-601" name="__codelineno-0-601"></a>        <span class="c1"># extra_values=self._extra_values.update(other._extra_values),</span>
-</span><span id="__span-0-602"><a id="__codelineno-0-602" name="__codelineno-0-602"></a>    <span class="p">)</span>
+</span><span id="__span-0-588"><a id="__codelineno-0-588" name="__codelineno-0-588"></a>    <span class="k">return</span> <span class="n">ValuationResult</span><span class="p">(</span>
+</span><span id="__span-0-589"><a id="__codelineno-0-589" name="__codelineno-0-589"></a>        <span class="n">algorithm</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">algorithm</span> <span class="ow">or</span> <span class="n">other</span><span class="o">.</span><span class="n">algorithm</span> <span class="ow">or</span> <span class="s2">&quot;&quot;</span><span class="p">,</span>
+</span><span id="__span-0-590"><a id="__codelineno-0-590" name="__codelineno-0-590"></a>        <span class="n">status</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">status</span> <span class="o">&amp;</span> <span class="n">other</span><span class="o">.</span><span class="n">status</span><span class="p">,</span>
+</span><span id="__span-0-591"><a id="__codelineno-0-591" name="__codelineno-0-591"></a>        <span class="n">indices</span><span class="o">=</span><span class="n">indices</span><span class="p">,</span>
+</span><span id="__span-0-592"><a id="__codelineno-0-592" name="__codelineno-0-592"></a>        <span class="n">values</span><span class="o">=</span><span class="n">xnm</span><span class="p">,</span>
+</span><span id="__span-0-593"><a id="__codelineno-0-593" name="__codelineno-0-593"></a>        <span class="n">variances</span><span class="o">=</span><span class="n">vnm</span><span class="p">,</span>
+</span><span id="__span-0-594"><a id="__codelineno-0-594" name="__codelineno-0-594"></a>        <span class="n">counts</span><span class="o">=</span><span class="n">n</span> <span class="o">+</span> <span class="n">m</span><span class="p">,</span>
+</span><span id="__span-0-595"><a id="__codelineno-0-595" name="__codelineno-0-595"></a>        <span class="n">data_names</span><span class="o">=</span><span class="n">names</span><span class="p">,</span>
+</span><span id="__span-0-596"><a id="__codelineno-0-596" name="__codelineno-0-596"></a>        <span class="c1"># FIXME: What to do with extra_values? This is not commutative:</span>
+</span><span id="__span-0-597"><a id="__codelineno-0-597" name="__codelineno-0-597"></a>        <span class="c1"># extra_values=self._extra_values.update(other._extra_values),</span>
+</span><span id="__span-0-598"><a id="__codelineno-0-598" name="__codelineno-0-598"></a>    <span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -4108,7 +4112,7 @@ <h3 id="pydvl.value.result.ValuationResult.update" class="doc doc-heading">
         <tr>
           <td>
               <span class="doc-returns-annotation">
-                  <code>&#39;ValuationResult&#39;</code>
+                  <code><a class="autorefs autorefs-internal" title="pydvl.value.result.ValuationResult" href="#pydvl.value.result.ValuationResult">ValuationResult</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>, <span title="pydvl.utils.types.NameT">NameT</span>]</code>
               </span>
           </td>
           <td class="doc-returns-details">
@@ -4147,7 +4151,11 @@ <h3 id="pydvl.value.result.ValuationResult.update" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/result.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-604">604</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-600">600</a></span>
+<span class="normal"><a href="#__codelineno-0-601">601</a></span>
+<span class="normal"><a href="#__codelineno-0-602">602</a></span>
+<span class="normal"><a href="#__codelineno-0-603">603</a></span>
+<span class="normal"><a href="#__codelineno-0-604">604</a></span>
 <span class="normal"><a href="#__codelineno-0-605">605</a></span>
 <span class="normal"><a href="#__codelineno-0-606">606</a></span>
 <span class="normal"><a href="#__codelineno-0-607">607</a></span>
@@ -4171,39 +4179,35 @@ <h3 id="pydvl.value.result.ValuationResult.update" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-625">625</a></span>
 <span class="normal"><a href="#__codelineno-0-626">626</a></span>
 <span class="normal"><a href="#__codelineno-0-627">627</a></span>
-<span class="normal"><a href="#__codelineno-0-628">628</a></span>
-<span class="normal"><a href="#__codelineno-0-629">629</a></span>
-<span class="normal"><a href="#__codelineno-0-630">630</a></span>
-<span class="normal"><a href="#__codelineno-0-631">631</a></span>
-<span class="normal"><a href="#__codelineno-0-632">632</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-604"><a id="__codelineno-0-604" name="__codelineno-0-604"></a><span class="k">def</span> <span class="nf">update</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">idx</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">new_value</span><span class="p">:</span> <span class="nb">float</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ValuationResult&quot;</span><span class="p">:</span>
-</span><span id="__span-0-605"><a id="__codelineno-0-605" name="__codelineno-0-605"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Updates the result in place with a new value, using running mean</span>
-</span><span id="__span-0-606"><a id="__codelineno-0-606" name="__codelineno-0-606"></a><span class="sd">    and variance.</span>
+<span class="normal"><a href="#__codelineno-0-628">628</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-600"><a id="__codelineno-0-600" name="__codelineno-0-600"></a><span class="k">def</span> <span class="nf">update</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">idx</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">new_value</span><span class="p">:</span> <span class="nb">float</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">[</span><span class="n">IndexT</span><span class="p">,</span> <span class="n">NameT</span><span class="p">]:</span>
+</span><span id="__span-0-601"><a id="__codelineno-0-601" name="__codelineno-0-601"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Updates the result in place with a new value, using running mean</span>
+</span><span id="__span-0-602"><a id="__codelineno-0-602" name="__codelineno-0-602"></a><span class="sd">    and variance.</span>
+</span><span id="__span-0-603"><a id="__codelineno-0-603" name="__codelineno-0-603"></a>
+</span><span id="__span-0-604"><a id="__codelineno-0-604" name="__codelineno-0-604"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-605"><a id="__codelineno-0-605" name="__codelineno-0-605"></a><span class="sd">        idx: Data index of the value to update.</span>
+</span><span id="__span-0-606"><a id="__codelineno-0-606" name="__codelineno-0-606"></a><span class="sd">        new_value: New value to add to the result.</span>
 </span><span id="__span-0-607"><a id="__codelineno-0-607" name="__codelineno-0-607"></a>
-</span><span id="__span-0-608"><a id="__codelineno-0-608" name="__codelineno-0-608"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-609"><a id="__codelineno-0-609" name="__codelineno-0-609"></a><span class="sd">        idx: Data index of the value to update.</span>
-</span><span id="__span-0-610"><a id="__codelineno-0-610" name="__codelineno-0-610"></a><span class="sd">        new_value: New value to add to the result.</span>
-</span><span id="__span-0-611"><a id="__codelineno-0-611" name="__codelineno-0-611"></a>
-</span><span id="__span-0-612"><a id="__codelineno-0-612" name="__codelineno-0-612"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-613"><a id="__codelineno-0-613" name="__codelineno-0-613"></a><span class="sd">        A reference to the same, modified result.</span>
-</span><span id="__span-0-614"><a id="__codelineno-0-614" name="__codelineno-0-614"></a>
-</span><span id="__span-0-615"><a id="__codelineno-0-615" name="__codelineno-0-615"></a><span class="sd">    Raises:</span>
-</span><span id="__span-0-616"><a id="__codelineno-0-616" name="__codelineno-0-616"></a><span class="sd">        IndexError: If the index is not found.</span>
-</span><span id="__span-0-617"><a id="__codelineno-0-617" name="__codelineno-0-617"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-618"><a id="__codelineno-0-618" name="__codelineno-0-618"></a>    <span class="k">try</span><span class="p">:</span>
-</span><span id="__span-0-619"><a id="__codelineno-0-619" name="__codelineno-0-619"></a>        <span class="n">pos</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_positions</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span>
-</span><span id="__span-0-620"><a id="__codelineno-0-620" name="__codelineno-0-620"></a>    <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
-</span><span id="__span-0-621"><a id="__codelineno-0-621" name="__codelineno-0-621"></a>        <span class="k">raise</span> <span class="ne">IndexError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Index </span><span class="si">{</span><span class="n">idx</span><span class="si">}</span><span class="s2"> not found in ValuationResult&quot;</span><span class="p">)</span>
-</span><span id="__span-0-622"><a id="__codelineno-0-622" name="__codelineno-0-622"></a>    <span class="n">val</span><span class="p">,</span> <span class="n">var</span> <span class="o">=</span> <span class="n">running_moments</span><span class="p">(</span>
-</span><span id="__span-0-623"><a id="__codelineno-0-623" name="__codelineno-0-623"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_values</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">_variances</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">_counts</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span> <span class="n">new_value</span>
-</span><span id="__span-0-624"><a id="__codelineno-0-624" name="__codelineno-0-624"></a>    <span class="p">)</span>
-</span><span id="__span-0-625"><a id="__codelineno-0-625" name="__codelineno-0-625"></a>    <span class="bp">self</span><span class="p">[</span><span class="n">pos</span><span class="p">]</span> <span class="o">=</span> <span class="n">ValueItem</span><span class="p">(</span>
-</span><span id="__span-0-626"><a id="__codelineno-0-626" name="__codelineno-0-626"></a>        <span class="n">index</span><span class="o">=</span><span class="n">cast</span><span class="p">(</span><span class="n">IndexT</span><span class="p">,</span> <span class="n">idx</span><span class="p">),</span>
-</span><span id="__span-0-627"><a id="__codelineno-0-627" name="__codelineno-0-627"></a>        <span class="n">name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
-</span><span id="__span-0-628"><a id="__codelineno-0-628" name="__codelineno-0-628"></a>        <span class="n">value</span><span class="o">=</span><span class="n">val</span><span class="p">,</span>
-</span><span id="__span-0-629"><a id="__codelineno-0-629" name="__codelineno-0-629"></a>        <span class="n">variance</span><span class="o">=</span><span class="n">var</span><span class="p">,</span>
-</span><span id="__span-0-630"><a id="__codelineno-0-630" name="__codelineno-0-630"></a>        <span class="n">count</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_counts</span><span class="p">[</span><span class="n">pos</span><span class="p">]</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
-</span><span id="__span-0-631"><a id="__codelineno-0-631" name="__codelineno-0-631"></a>    <span class="p">)</span>
-</span><span id="__span-0-632"><a id="__codelineno-0-632" name="__codelineno-0-632"></a>    <span class="k">return</span> <span class="bp">self</span>
+</span><span id="__span-0-608"><a id="__codelineno-0-608" name="__codelineno-0-608"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-609"><a id="__codelineno-0-609" name="__codelineno-0-609"></a><span class="sd">        A reference to the same, modified result.</span>
+</span><span id="__span-0-610"><a id="__codelineno-0-610" name="__codelineno-0-610"></a>
+</span><span id="__span-0-611"><a id="__codelineno-0-611" name="__codelineno-0-611"></a><span class="sd">    Raises:</span>
+</span><span id="__span-0-612"><a id="__codelineno-0-612" name="__codelineno-0-612"></a><span class="sd">        IndexError: If the index is not found.</span>
+</span><span id="__span-0-613"><a id="__codelineno-0-613" name="__codelineno-0-613"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-614"><a id="__codelineno-0-614" name="__codelineno-0-614"></a>    <span class="k">try</span><span class="p">:</span>
+</span><span id="__span-0-615"><a id="__codelineno-0-615" name="__codelineno-0-615"></a>        <span class="n">pos</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_positions</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span>
+</span><span id="__span-0-616"><a id="__codelineno-0-616" name="__codelineno-0-616"></a>    <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
+</span><span id="__span-0-617"><a id="__codelineno-0-617" name="__codelineno-0-617"></a>        <span class="k">raise</span> <span class="ne">IndexError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Index </span><span class="si">{</span><span class="n">idx</span><span class="si">}</span><span class="s2"> not found in ValuationResult&quot;</span><span class="p">)</span>
+</span><span id="__span-0-618"><a id="__codelineno-0-618" name="__codelineno-0-618"></a>    <span class="n">val</span><span class="p">,</span> <span class="n">var</span> <span class="o">=</span> <span class="n">running_moments</span><span class="p">(</span>
+</span><span id="__span-0-619"><a id="__codelineno-0-619" name="__codelineno-0-619"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_values</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">_variances</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">_counts</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span> <span class="n">new_value</span>
+</span><span id="__span-0-620"><a id="__codelineno-0-620" name="__codelineno-0-620"></a>    <span class="p">)</span>
+</span><span id="__span-0-621"><a id="__codelineno-0-621" name="__codelineno-0-621"></a>    <span class="bp">self</span><span class="p">[</span><span class="n">pos</span><span class="p">]</span> <span class="o">=</span> <span class="n">ValueItem</span><span class="p">(</span>
+</span><span id="__span-0-622"><a id="__codelineno-0-622" name="__codelineno-0-622"></a>        <span class="n">index</span><span class="o">=</span><span class="n">cast</span><span class="p">(</span><span class="n">IndexT</span><span class="p">,</span> <span class="n">idx</span><span class="p">),</span>  <span class="c1"># FIXME</span>
+</span><span id="__span-0-623"><a id="__codelineno-0-623" name="__codelineno-0-623"></a>        <span class="n">name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
+</span><span id="__span-0-624"><a id="__codelineno-0-624" name="__codelineno-0-624"></a>        <span class="n">value</span><span class="o">=</span><span class="n">val</span><span class="p">,</span>
+</span><span id="__span-0-625"><a id="__codelineno-0-625" name="__codelineno-0-625"></a>        <span class="n">variance</span><span class="o">=</span><span class="n">var</span><span class="p">,</span>
+</span><span id="__span-0-626"><a id="__codelineno-0-626" name="__codelineno-0-626"></a>        <span class="n">count</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_counts</span><span class="p">[</span><span class="n">pos</span><span class="p">]</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
+</span><span id="__span-0-627"><a id="__codelineno-0-627" name="__codelineno-0-627"></a>    <span class="p">)</span>
+</span><span id="__span-0-628"><a id="__codelineno-0-628" name="__codelineno-0-628"></a>    <span class="k">return</span> <span class="bp">self</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -4254,7 +4258,11 @@ <h3 id="pydvl.value.result.ValuationResult.get" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/result.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-634">634</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-630">630</a></span>
+<span class="normal"><a href="#__codelineno-0-631">631</a></span>
+<span class="normal"><a href="#__codelineno-0-632">632</a></span>
+<span class="normal"><a href="#__codelineno-0-633">633</a></span>
+<span class="normal"><a href="#__codelineno-0-634">634</a></span>
 <span class="normal"><a href="#__codelineno-0-635">635</a></span>
 <span class="normal"><a href="#__codelineno-0-636">636</a></span>
 <span class="normal"><a href="#__codelineno-0-637">637</a></span>
@@ -4268,29 +4276,25 @@ <h3 id="pydvl.value.result.ValuationResult.get" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-645">645</a></span>
 <span class="normal"><a href="#__codelineno-0-646">646</a></span>
 <span class="normal"><a href="#__codelineno-0-647">647</a></span>
-<span class="normal"><a href="#__codelineno-0-648">648</a></span>
-<span class="normal"><a href="#__codelineno-0-649">649</a></span>
-<span class="normal"><a href="#__codelineno-0-650">650</a></span>
-<span class="normal"><a href="#__codelineno-0-651">651</a></span>
-<span class="normal"><a href="#__codelineno-0-652">652</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-634"><a id="__codelineno-0-634" name="__codelineno-0-634"></a><span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">idx</span><span class="p">:</span> <span class="n">Integral</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValueItem</span><span class="p">:</span>
-</span><span id="__span-0-635"><a id="__codelineno-0-635" name="__codelineno-0-635"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Retrieves a ValueItem by data index, as opposed to sort index, like</span>
-</span><span id="__span-0-636"><a id="__codelineno-0-636" name="__codelineno-0-636"></a><span class="sd">    the indexing operator.</span>
-</span><span id="__span-0-637"><a id="__codelineno-0-637" name="__codelineno-0-637"></a>
-</span><span id="__span-0-638"><a id="__codelineno-0-638" name="__codelineno-0-638"></a><span class="sd">    Raises:</span>
-</span><span id="__span-0-639"><a id="__codelineno-0-639" name="__codelineno-0-639"></a><span class="sd">         IndexError: If the index is not found.</span>
-</span><span id="__span-0-640"><a id="__codelineno-0-640" name="__codelineno-0-640"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-641"><a id="__codelineno-0-641" name="__codelineno-0-641"></a>    <span class="k">try</span><span class="p">:</span>
-</span><span id="__span-0-642"><a id="__codelineno-0-642" name="__codelineno-0-642"></a>        <span class="n">pos</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_positions</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span>
-</span><span id="__span-0-643"><a id="__codelineno-0-643" name="__codelineno-0-643"></a>    <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
-</span><span id="__span-0-644"><a id="__codelineno-0-644" name="__codelineno-0-644"></a>        <span class="k">raise</span> <span class="ne">IndexError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Index </span><span class="si">{</span><span class="n">idx</span><span class="si">}</span><span class="s2"> not found in ValuationResult&quot;</span><span class="p">)</span>
-</span><span id="__span-0-645"><a id="__codelineno-0-645" name="__codelineno-0-645"></a>
-</span><span id="__span-0-646"><a id="__codelineno-0-646" name="__codelineno-0-646"></a>    <span class="k">return</span> <span class="n">ValueItem</span><span class="p">(</span>
-</span><span id="__span-0-647"><a id="__codelineno-0-647" name="__codelineno-0-647"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
-</span><span id="__span-0-648"><a id="__codelineno-0-648" name="__codelineno-0-648"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
-</span><span id="__span-0-649"><a id="__codelineno-0-649" name="__codelineno-0-649"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_values</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
-</span><span id="__span-0-650"><a id="__codelineno-0-650" name="__codelineno-0-650"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_variances</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
-</span><span id="__span-0-651"><a id="__codelineno-0-651" name="__codelineno-0-651"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_counts</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
-</span><span id="__span-0-652"><a id="__codelineno-0-652" name="__codelineno-0-652"></a>    <span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-648">648</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-630"><a id="__codelineno-0-630" name="__codelineno-0-630"></a><span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">idx</span><span class="p">:</span> <span class="n">Integral</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValueItem</span><span class="p">:</span>
+</span><span id="__span-0-631"><a id="__codelineno-0-631" name="__codelineno-0-631"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Retrieves a ValueItem by data index, as opposed to sort index, like</span>
+</span><span id="__span-0-632"><a id="__codelineno-0-632" name="__codelineno-0-632"></a><span class="sd">    the indexing operator.</span>
+</span><span id="__span-0-633"><a id="__codelineno-0-633" name="__codelineno-0-633"></a>
+</span><span id="__span-0-634"><a id="__codelineno-0-634" name="__codelineno-0-634"></a><span class="sd">    Raises:</span>
+</span><span id="__span-0-635"><a id="__codelineno-0-635" name="__codelineno-0-635"></a><span class="sd">         IndexError: If the index is not found.</span>
+</span><span id="__span-0-636"><a id="__codelineno-0-636" name="__codelineno-0-636"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-637"><a id="__codelineno-0-637" name="__codelineno-0-637"></a>    <span class="k">try</span><span class="p">:</span>
+</span><span id="__span-0-638"><a id="__codelineno-0-638" name="__codelineno-0-638"></a>        <span class="n">pos</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_positions</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span>
+</span><span id="__span-0-639"><a id="__codelineno-0-639" name="__codelineno-0-639"></a>    <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
+</span><span id="__span-0-640"><a id="__codelineno-0-640" name="__codelineno-0-640"></a>        <span class="k">raise</span> <span class="ne">IndexError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Index </span><span class="si">{</span><span class="n">idx</span><span class="si">}</span><span class="s2"> not found in ValuationResult&quot;</span><span class="p">)</span>
+</span><span id="__span-0-641"><a id="__codelineno-0-641" name="__codelineno-0-641"></a>
+</span><span id="__span-0-642"><a id="__codelineno-0-642" name="__codelineno-0-642"></a>    <span class="k">return</span> <span class="n">ValueItem</span><span class="p">(</span>
+</span><span id="__span-0-643"><a id="__codelineno-0-643" name="__codelineno-0-643"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
+</span><span id="__span-0-644"><a id="__codelineno-0-644" name="__codelineno-0-644"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
+</span><span id="__span-0-645"><a id="__codelineno-0-645" name="__codelineno-0-645"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_values</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
+</span><span id="__span-0-646"><a id="__codelineno-0-646" name="__codelineno-0-646"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_variances</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
+</span><span id="__span-0-647"><a id="__codelineno-0-647" name="__codelineno-0-647"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_counts</span><span class="p">[</span><span class="n">pos</span><span class="p">],</span>
+</span><span id="__span-0-648"><a id="__codelineno-0-648" name="__codelineno-0-648"></a>    <span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -4395,7 +4399,11 @@ <h3 id="pydvl.value.result.ValuationResult.to_dataframe" class="doc doc-heading"
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/result.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-654">654</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-650">650</a></span>
+<span class="normal"><a href="#__codelineno-0-651">651</a></span>
+<span class="normal"><a href="#__codelineno-0-652">652</a></span>
+<span class="normal"><a href="#__codelineno-0-653">653</a></span>
+<span class="normal"><a href="#__codelineno-0-654">654</a></span>
 <span class="normal"><a href="#__codelineno-0-655">655</a></span>
 <span class="normal"><a href="#__codelineno-0-656">656</a></span>
 <span class="normal"><a href="#__codelineno-0-657">657</a></span>
@@ -4420,40 +4428,36 @@ <h3 id="pydvl.value.result.ValuationResult.to_dataframe" class="doc doc-heading"
 <span class="normal"><a href="#__codelineno-0-676">676</a></span>
 <span class="normal"><a href="#__codelineno-0-677">677</a></span>
 <span class="normal"><a href="#__codelineno-0-678">678</a></span>
-<span class="normal"><a href="#__codelineno-0-679">679</a></span>
-<span class="normal"><a href="#__codelineno-0-680">680</a></span>
-<span class="normal"><a href="#__codelineno-0-681">681</a></span>
-<span class="normal"><a href="#__codelineno-0-682">682</a></span>
-<span class="normal"><a href="#__codelineno-0-683">683</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-654"><a id="__codelineno-0-654" name="__codelineno-0-654"></a><span class="k">def</span> <span class="nf">to_dataframe</span><span class="p">(</span>
-</span><span id="__span-0-655"><a id="__codelineno-0-655" name="__codelineno-0-655"></a>    <span class="bp">self</span><span class="p">,</span> <span class="n">column</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">use_names</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
-</span><span id="__span-0-656"><a id="__codelineno-0-656" name="__codelineno-0-656"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pandas</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
-</span><span id="__span-0-657"><a id="__codelineno-0-657" name="__codelineno-0-657"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns values as a dataframe.</span>
-</span><span id="__span-0-658"><a id="__codelineno-0-658" name="__codelineno-0-658"></a>
-</span><span id="__span-0-659"><a id="__codelineno-0-659" name="__codelineno-0-659"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-660"><a id="__codelineno-0-660" name="__codelineno-0-660"></a><span class="sd">        column: Name for the column holding the data value. Defaults to</span>
-</span><span id="__span-0-661"><a id="__codelineno-0-661" name="__codelineno-0-661"></a><span class="sd">            the name of the algorithm used.</span>
-</span><span id="__span-0-662"><a id="__codelineno-0-662" name="__codelineno-0-662"></a><span class="sd">        use_names: Whether to use data names instead of indices for the</span>
-</span><span id="__span-0-663"><a id="__codelineno-0-663" name="__codelineno-0-663"></a><span class="sd">            DataFrame&#39;s index.</span>
-</span><span id="__span-0-664"><a id="__codelineno-0-664" name="__codelineno-0-664"></a>
-</span><span id="__span-0-665"><a id="__codelineno-0-665" name="__codelineno-0-665"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-666"><a id="__codelineno-0-666" name="__codelineno-0-666"></a><span class="sd">        A dataframe with two columns, one for the values, with name</span>
-</span><span id="__span-0-667"><a id="__codelineno-0-667" name="__codelineno-0-667"></a><span class="sd">            given as explained in `column`, and another with standard errors for</span>
-</span><span id="__span-0-668"><a id="__codelineno-0-668" name="__codelineno-0-668"></a><span class="sd">            approximate algorithms. The latter will be named `column+&#39;_stderr&#39;`.</span>
-</span><span id="__span-0-669"><a id="__codelineno-0-669" name="__codelineno-0-669"></a><span class="sd">    Raises:</span>
-</span><span id="__span-0-670"><a id="__codelineno-0-670" name="__codelineno-0-670"></a><span class="sd">         ImportError: If pandas is not installed</span>
-</span><span id="__span-0-671"><a id="__codelineno-0-671" name="__codelineno-0-671"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-672"><a id="__codelineno-0-672" name="__codelineno-0-672"></a>    <span class="k">if</span> <span class="ow">not</span> <span class="n">pandas</span><span class="p">:</span>
-</span><span id="__span-0-673"><a id="__codelineno-0-673" name="__codelineno-0-673"></a>        <span class="k">raise</span> <span class="ne">ImportError</span><span class="p">(</span><span class="s2">&quot;Pandas required for DataFrame export&quot;</span><span class="p">)</span>
-</span><span id="__span-0-674"><a id="__codelineno-0-674" name="__codelineno-0-674"></a>    <span class="n">column</span> <span class="o">=</span> <span class="n">column</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_algorithm</span>
-</span><span id="__span-0-675"><a id="__codelineno-0-675" name="__codelineno-0-675"></a>    <span class="n">df</span> <span class="o">=</span> <span class="n">pandas</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span>
-</span><span id="__span-0-676"><a id="__codelineno-0-676" name="__codelineno-0-676"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_values</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span><span class="p">],</span>
-</span><span id="__span-0-677"><a id="__codelineno-0-677" name="__codelineno-0-677"></a>        <span class="n">index</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span><span class="p">]</span>
-</span><span id="__span-0-678"><a id="__codelineno-0-678" name="__codelineno-0-678"></a>        <span class="k">if</span> <span class="n">use_names</span>
-</span><span id="__span-0-679"><a id="__codelineno-0-679" name="__codelineno-0-679"></a>        <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span><span class="p">],</span>
-</span><span id="__span-0-680"><a id="__codelineno-0-680" name="__codelineno-0-680"></a>        <span class="n">columns</span><span class="o">=</span><span class="p">[</span><span class="n">column</span><span class="p">],</span>
-</span><span id="__span-0-681"><a id="__codelineno-0-681" name="__codelineno-0-681"></a>    <span class="p">)</span>
-</span><span id="__span-0-682"><a id="__codelineno-0-682" name="__codelineno-0-682"></a>    <span class="n">df</span><span class="p">[</span><span class="n">column</span> <span class="o">+</span> <span class="s2">&quot;_stderr&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">stderr</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span><span class="p">]</span>
-</span><span id="__span-0-683"><a id="__codelineno-0-683" name="__codelineno-0-683"></a>    <span class="k">return</span> <span class="n">df</span>
+<span class="normal"><a href="#__codelineno-0-679">679</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-650"><a id="__codelineno-0-650" name="__codelineno-0-650"></a><span class="k">def</span> <span class="nf">to_dataframe</span><span class="p">(</span>
+</span><span id="__span-0-651"><a id="__codelineno-0-651" name="__codelineno-0-651"></a>    <span class="bp">self</span><span class="p">,</span> <span class="n">column</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">use_names</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
+</span><span id="__span-0-652"><a id="__codelineno-0-652" name="__codelineno-0-652"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pandas</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+</span><span id="__span-0-653"><a id="__codelineno-0-653" name="__codelineno-0-653"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns values as a dataframe.</span>
+</span><span id="__span-0-654"><a id="__codelineno-0-654" name="__codelineno-0-654"></a>
+</span><span id="__span-0-655"><a id="__codelineno-0-655" name="__codelineno-0-655"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-656"><a id="__codelineno-0-656" name="__codelineno-0-656"></a><span class="sd">        column: Name for the column holding the data value. Defaults to</span>
+</span><span id="__span-0-657"><a id="__codelineno-0-657" name="__codelineno-0-657"></a><span class="sd">            the name of the algorithm used.</span>
+</span><span id="__span-0-658"><a id="__codelineno-0-658" name="__codelineno-0-658"></a><span class="sd">        use_names: Whether to use data names instead of indices for the</span>
+</span><span id="__span-0-659"><a id="__codelineno-0-659" name="__codelineno-0-659"></a><span class="sd">            DataFrame&#39;s index.</span>
+</span><span id="__span-0-660"><a id="__codelineno-0-660" name="__codelineno-0-660"></a>
+</span><span id="__span-0-661"><a id="__codelineno-0-661" name="__codelineno-0-661"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-662"><a id="__codelineno-0-662" name="__codelineno-0-662"></a><span class="sd">        A dataframe with two columns, one for the values, with name</span>
+</span><span id="__span-0-663"><a id="__codelineno-0-663" name="__codelineno-0-663"></a><span class="sd">            given as explained in `column`, and another with standard errors for</span>
+</span><span id="__span-0-664"><a id="__codelineno-0-664" name="__codelineno-0-664"></a><span class="sd">            approximate algorithms. The latter will be named `column+&#39;_stderr&#39;`.</span>
+</span><span id="__span-0-665"><a id="__codelineno-0-665" name="__codelineno-0-665"></a><span class="sd">    Raises:</span>
+</span><span id="__span-0-666"><a id="__codelineno-0-666" name="__codelineno-0-666"></a><span class="sd">         ImportError: If pandas is not installed</span>
+</span><span id="__span-0-667"><a id="__codelineno-0-667" name="__codelineno-0-667"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-668"><a id="__codelineno-0-668" name="__codelineno-0-668"></a>    <span class="k">if</span> <span class="ow">not</span> <span class="n">pandas</span><span class="p">:</span>
+</span><span id="__span-0-669"><a id="__codelineno-0-669" name="__codelineno-0-669"></a>        <span class="k">raise</span> <span class="ne">ImportError</span><span class="p">(</span><span class="s2">&quot;Pandas required for DataFrame export&quot;</span><span class="p">)</span>
+</span><span id="__span-0-670"><a id="__codelineno-0-670" name="__codelineno-0-670"></a>    <span class="n">column</span> <span class="o">=</span> <span class="n">column</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_algorithm</span>
+</span><span id="__span-0-671"><a id="__codelineno-0-671" name="__codelineno-0-671"></a>    <span class="n">df</span> <span class="o">=</span> <span class="n">pandas</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span>
+</span><span id="__span-0-672"><a id="__codelineno-0-672" name="__codelineno-0-672"></a>        <span class="bp">self</span><span class="o">.</span><span class="n">_values</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span><span class="p">],</span>
+</span><span id="__span-0-673"><a id="__codelineno-0-673" name="__codelineno-0-673"></a>        <span class="n">index</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_names</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span><span class="p">]</span>
+</span><span id="__span-0-674"><a id="__codelineno-0-674" name="__codelineno-0-674"></a>        <span class="k">if</span> <span class="n">use_names</span>
+</span><span id="__span-0-675"><a id="__codelineno-0-675" name="__codelineno-0-675"></a>        <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span><span class="p">],</span>
+</span><span id="__span-0-676"><a id="__codelineno-0-676" name="__codelineno-0-676"></a>        <span class="n">columns</span><span class="o">=</span><span class="p">[</span><span class="n">column</span><span class="p">],</span>
+</span><span id="__span-0-677"><a id="__codelineno-0-677" name="__codelineno-0-677"></a>    <span class="p">)</span>
+</span><span id="__span-0-678"><a id="__codelineno-0-678" name="__codelineno-0-678"></a>    <span class="n">df</span><span class="p">[</span><span class="n">column</span> <span class="o">+</span> <span class="s2">&quot;_stderr&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">stderr</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_sort_positions</span><span class="p">]</span>
+</span><span id="__span-0-679"><a id="__codelineno-0-679" name="__codelineno-0-679"></a>    <span class="k">return</span> <span class="n">df</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -4611,7 +4615,11 @@ <h3 id="pydvl.value.result.ValuationResult.from_random" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/result.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-685">685</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-681">681</a></span>
+<span class="normal"><a href="#__codelineno-0-682">682</a></span>
+<span class="normal"><a href="#__codelineno-0-683">683</a></span>
+<span class="normal"><a href="#__codelineno-0-684">684</a></span>
+<span class="normal"><a href="#__codelineno-0-685">685</a></span>
 <span class="normal"><a href="#__codelineno-0-686">686</a></span>
 <span class="normal"><a href="#__codelineno-0-687">687</a></span>
 <span class="normal"><a href="#__codelineno-0-688">688</a></span>
@@ -4646,50 +4654,46 @@ <h3 id="pydvl.value.result.ValuationResult.from_random" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-717">717</a></span>
 <span class="normal"><a href="#__codelineno-0-718">718</a></span>
 <span class="normal"><a href="#__codelineno-0-719">719</a></span>
-<span class="normal"><a href="#__codelineno-0-720">720</a></span>
-<span class="normal"><a href="#__codelineno-0-721">721</a></span>
-<span class="normal"><a href="#__codelineno-0-722">722</a></span>
-<span class="normal"><a href="#__codelineno-0-723">723</a></span>
-<span class="normal"><a href="#__codelineno-0-724">724</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-685"><a id="__codelineno-0-685" name="__codelineno-0-685"></a><span class="nd">@classmethod</span>
-</span><span id="__span-0-686"><a id="__codelineno-0-686" name="__codelineno-0-686"></a><span class="k">def</span> <span class="nf">from_random</span><span class="p">(</span>
-</span><span id="__span-0-687"><a id="__codelineno-0-687" name="__codelineno-0-687"></a>    <span class="bp">cls</span><span class="p">,</span>
-</span><span id="__span-0-688"><a id="__codelineno-0-688" name="__codelineno-0-688"></a>    <span class="n">size</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-</span><span id="__span-0-689"><a id="__codelineno-0-689" name="__codelineno-0-689"></a>    <span class="n">total</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-690"><a id="__codelineno-0-690" name="__codelineno-0-690"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-691"><a id="__codelineno-0-691" name="__codelineno-0-691"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-</span><span id="__span-0-692"><a id="__codelineno-0-692" name="__codelineno-0-692"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ValuationResult&quot;</span><span class="p">:</span>
-</span><span id="__span-0-693"><a id="__codelineno-0-693" name="__codelineno-0-693"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Creates a [ValuationResult][pydvl.value.result.ValuationResult] object and fills it with an array</span>
-</span><span id="__span-0-694"><a id="__codelineno-0-694" name="__codelineno-0-694"></a><span class="sd">    of random values from a uniform distribution in [-1,1]. The values can</span>
-</span><span id="__span-0-695"><a id="__codelineno-0-695" name="__codelineno-0-695"></a><span class="sd">    be made to sum up to a given total number (doing so will change their range).</span>
-</span><span id="__span-0-696"><a id="__codelineno-0-696" name="__codelineno-0-696"></a>
-</span><span id="__span-0-697"><a id="__codelineno-0-697" name="__codelineno-0-697"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-698"><a id="__codelineno-0-698" name="__codelineno-0-698"></a><span class="sd">        size: Number of values to generate</span>
-</span><span id="__span-0-699"><a id="__codelineno-0-699" name="__codelineno-0-699"></a><span class="sd">        total: If set, the values are normalized to sum to this number</span>
-</span><span id="__span-0-700"><a id="__codelineno-0-700" name="__codelineno-0-700"></a><span class="sd">            (&quot;efficiency&quot; property of Shapley values).</span>
-</span><span id="__span-0-701"><a id="__codelineno-0-701" name="__codelineno-0-701"></a><span class="sd">        kwargs: Additional options to pass to the constructor of</span>
-</span><span id="__span-0-702"><a id="__codelineno-0-702" name="__codelineno-0-702"></a><span class="sd">            [ValuationResult][pydvl.value.result.ValuationResult]. Use to override status, names, etc.</span>
+<span class="normal"><a href="#__codelineno-0-720">720</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-681"><a id="__codelineno-0-681" name="__codelineno-0-681"></a><span class="nd">@classmethod</span>
+</span><span id="__span-0-682"><a id="__codelineno-0-682" name="__codelineno-0-682"></a><span class="k">def</span> <span class="nf">from_random</span><span class="p">(</span>
+</span><span id="__span-0-683"><a id="__codelineno-0-683" name="__codelineno-0-683"></a>    <span class="bp">cls</span><span class="p">,</span>
+</span><span id="__span-0-684"><a id="__codelineno-0-684" name="__codelineno-0-684"></a>    <span class="n">size</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+</span><span id="__span-0-685"><a id="__codelineno-0-685" name="__codelineno-0-685"></a>    <span class="n">total</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-686"><a id="__codelineno-0-686" name="__codelineno-0-686"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-687"><a id="__codelineno-0-687" name="__codelineno-0-687"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+</span><span id="__span-0-688"><a id="__codelineno-0-688" name="__codelineno-0-688"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ValuationResult&quot;</span><span class="p">:</span>
+</span><span id="__span-0-689"><a id="__codelineno-0-689" name="__codelineno-0-689"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Creates a [ValuationResult][pydvl.value.result.ValuationResult] object and fills it with an array</span>
+</span><span id="__span-0-690"><a id="__codelineno-0-690" name="__codelineno-0-690"></a><span class="sd">    of random values from a uniform distribution in [-1,1]. The values can</span>
+</span><span id="__span-0-691"><a id="__codelineno-0-691" name="__codelineno-0-691"></a><span class="sd">    be made to sum up to a given total number (doing so will change their range).</span>
+</span><span id="__span-0-692"><a id="__codelineno-0-692" name="__codelineno-0-692"></a>
+</span><span id="__span-0-693"><a id="__codelineno-0-693" name="__codelineno-0-693"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-694"><a id="__codelineno-0-694" name="__codelineno-0-694"></a><span class="sd">        size: Number of values to generate</span>
+</span><span id="__span-0-695"><a id="__codelineno-0-695" name="__codelineno-0-695"></a><span class="sd">        total: If set, the values are normalized to sum to this number</span>
+</span><span id="__span-0-696"><a id="__codelineno-0-696" name="__codelineno-0-696"></a><span class="sd">            (&quot;efficiency&quot; property of Shapley values).</span>
+</span><span id="__span-0-697"><a id="__codelineno-0-697" name="__codelineno-0-697"></a><span class="sd">        kwargs: Additional options to pass to the constructor of</span>
+</span><span id="__span-0-698"><a id="__codelineno-0-698" name="__codelineno-0-698"></a><span class="sd">            [ValuationResult][pydvl.value.result.ValuationResult]. Use to override status, names, etc.</span>
+</span><span id="__span-0-699"><a id="__codelineno-0-699" name="__codelineno-0-699"></a>
+</span><span id="__span-0-700"><a id="__codelineno-0-700" name="__codelineno-0-700"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-701"><a id="__codelineno-0-701" name="__codelineno-0-701"></a><span class="sd">        A valuation result with its status set to</span>
+</span><span id="__span-0-702"><a id="__codelineno-0-702" name="__codelineno-0-702"></a><span class="sd">        [Status.Converged][pydvl.utils.status.Status] by default.</span>
 </span><span id="__span-0-703"><a id="__codelineno-0-703" name="__codelineno-0-703"></a>
-</span><span id="__span-0-704"><a id="__codelineno-0-704" name="__codelineno-0-704"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-705"><a id="__codelineno-0-705" name="__codelineno-0-705"></a><span class="sd">        A valuation result with its status set to</span>
-</span><span id="__span-0-706"><a id="__codelineno-0-706" name="__codelineno-0-706"></a><span class="sd">        [Status.Converged][pydvl.utils.status.Status] by default.</span>
-</span><span id="__span-0-707"><a id="__codelineno-0-707" name="__codelineno-0-707"></a>
-</span><span id="__span-0-708"><a id="__codelineno-0-708" name="__codelineno-0-708"></a><span class="sd">    Raises:</span>
-</span><span id="__span-0-709"><a id="__codelineno-0-709" name="__codelineno-0-709"></a><span class="sd">         ValueError: If `size` is less than 1.</span>
-</span><span id="__span-0-710"><a id="__codelineno-0-710" name="__codelineno-0-710"></a>
-</span><span id="__span-0-711"><a id="__codelineno-0-711" name="__codelineno-0-711"></a><span class="sd">    !!! tip &quot;Changed in version 0.6.0&quot;</span>
-</span><span id="__span-0-712"><a id="__codelineno-0-712" name="__codelineno-0-712"></a><span class="sd">        Added parameter `total`. Check for zero size</span>
-</span><span id="__span-0-713"><a id="__codelineno-0-713" name="__codelineno-0-713"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-714"><a id="__codelineno-0-714" name="__codelineno-0-714"></a>    <span class="k">if</span> <span class="n">size</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
-</span><span id="__span-0-715"><a id="__codelineno-0-715" name="__codelineno-0-715"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Size must be a positive integer&quot;</span><span class="p">)</span>
-</span><span id="__span-0-716"><a id="__codelineno-0-716" name="__codelineno-0-716"></a>
-</span><span id="__span-0-717"><a id="__codelineno-0-717" name="__codelineno-0-717"></a>    <span class="n">rng</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">default_rng</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
-</span><span id="__span-0-718"><a id="__codelineno-0-718" name="__codelineno-0-718"></a>    <span class="n">values</span> <span class="o">=</span> <span class="n">rng</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="n">low</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> <span class="n">high</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="n">size</span><span class="p">)</span>
-</span><span id="__span-0-719"><a id="__codelineno-0-719" name="__codelineno-0-719"></a>    <span class="k">if</span> <span class="n">total</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-</span><span id="__span-0-720"><a id="__codelineno-0-720" name="__codelineno-0-720"></a>        <span class="n">values</span> <span class="o">*=</span> <span class="n">total</span> <span class="o">/</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">values</span><span class="p">)</span>
-</span><span id="__span-0-721"><a id="__codelineno-0-721" name="__codelineno-0-721"></a>
-</span><span id="__span-0-722"><a id="__codelineno-0-722" name="__codelineno-0-722"></a>    <span class="n">options</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">values</span><span class="o">=</span><span class="n">values</span><span class="p">,</span> <span class="n">status</span><span class="o">=</span><span class="n">Status</span><span class="o">.</span><span class="n">Converged</span><span class="p">,</span> <span class="n">algorithm</span><span class="o">=</span><span class="s2">&quot;random&quot;</span><span class="p">)</span>
-</span><span id="__span-0-723"><a id="__codelineno-0-723" name="__codelineno-0-723"></a>    <span class="n">options</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>
-</span><span id="__span-0-724"><a id="__codelineno-0-724" name="__codelineno-0-724"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="o">**</span><span class="n">options</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
+</span><span id="__span-0-704"><a id="__codelineno-0-704" name="__codelineno-0-704"></a><span class="sd">    Raises:</span>
+</span><span id="__span-0-705"><a id="__codelineno-0-705" name="__codelineno-0-705"></a><span class="sd">         ValueError: If `size` is less than 1.</span>
+</span><span id="__span-0-706"><a id="__codelineno-0-706" name="__codelineno-0-706"></a>
+</span><span id="__span-0-707"><a id="__codelineno-0-707" name="__codelineno-0-707"></a><span class="sd">    !!! tip &quot;Changed in version 0.6.0&quot;</span>
+</span><span id="__span-0-708"><a id="__codelineno-0-708" name="__codelineno-0-708"></a><span class="sd">        Added parameter `total`. Check for zero size</span>
+</span><span id="__span-0-709"><a id="__codelineno-0-709" name="__codelineno-0-709"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-710"><a id="__codelineno-0-710" name="__codelineno-0-710"></a>    <span class="k">if</span> <span class="n">size</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
+</span><span id="__span-0-711"><a id="__codelineno-0-711" name="__codelineno-0-711"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Size must be a positive integer&quot;</span><span class="p">)</span>
+</span><span id="__span-0-712"><a id="__codelineno-0-712" name="__codelineno-0-712"></a>
+</span><span id="__span-0-713"><a id="__codelineno-0-713" name="__codelineno-0-713"></a>    <span class="n">rng</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">default_rng</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
+</span><span id="__span-0-714"><a id="__codelineno-0-714" name="__codelineno-0-714"></a>    <span class="n">values</span> <span class="o">=</span> <span class="n">rng</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="n">low</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> <span class="n">high</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="n">size</span><span class="p">)</span>
+</span><span id="__span-0-715"><a id="__codelineno-0-715" name="__codelineno-0-715"></a>    <span class="k">if</span> <span class="n">total</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+</span><span id="__span-0-716"><a id="__codelineno-0-716" name="__codelineno-0-716"></a>        <span class="n">values</span> <span class="o">*=</span> <span class="n">total</span> <span class="o">/</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">values</span><span class="p">)</span>
+</span><span id="__span-0-717"><a id="__codelineno-0-717" name="__codelineno-0-717"></a>
+</span><span id="__span-0-718"><a id="__codelineno-0-718" name="__codelineno-0-718"></a>    <span class="n">options</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">values</span><span class="o">=</span><span class="n">values</span><span class="p">,</span> <span class="n">status</span><span class="o">=</span><span class="n">Status</span><span class="o">.</span><span class="n">Converged</span><span class="p">,</span> <span class="n">algorithm</span><span class="o">=</span><span class="s2">&quot;random&quot;</span><span class="p">)</span>
+</span><span id="__span-0-719"><a id="__codelineno-0-719" name="__codelineno-0-719"></a>    <span class="n">options</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>
+</span><span id="__span-0-720"><a id="__codelineno-0-720" name="__codelineno-0-720"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="o">**</span><span class="n">options</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -4762,7 +4766,7 @@ <h3 id="pydvl.value.result.ValuationResult.empty" class="doc doc-heading">
         <tr>
           <td>
               <span class="doc-returns-annotation">
-                  <code>&#39;ValuationResult&#39;</code>
+                  <code><a class="autorefs autorefs-internal" title="pydvl.value.result.ValuationResult" href="#pydvl.value.result.ValuationResult">ValuationResult</a></code>
               </span>
           </td>
           <td class="doc-returns-details">
@@ -4776,7 +4780,11 @@ <h3 id="pydvl.value.result.ValuationResult.empty" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/result.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-726">726</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-722">722</a></span>
+<span class="normal"><a href="#__codelineno-0-723">723</a></span>
+<span class="normal"><a href="#__codelineno-0-724">724</a></span>
+<span class="normal"><a href="#__codelineno-0-725">725</a></span>
+<span class="normal"><a href="#__codelineno-0-726">726</a></span>
 <span class="normal"><a href="#__codelineno-0-727">727</a></span>
 <span class="normal"><a href="#__codelineno-0-728">728</a></span>
 <span class="normal"><a href="#__codelineno-0-729">729</a></span>
@@ -4806,45 +4814,41 @@ <h3 id="pydvl.value.result.ValuationResult.empty" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-753">753</a></span>
 <span class="normal"><a href="#__codelineno-0-754">754</a></span>
 <span class="normal"><a href="#__codelineno-0-755">755</a></span>
-<span class="normal"><a href="#__codelineno-0-756">756</a></span>
-<span class="normal"><a href="#__codelineno-0-757">757</a></span>
-<span class="normal"><a href="#__codelineno-0-758">758</a></span>
-<span class="normal"><a href="#__codelineno-0-759">759</a></span>
-<span class="normal"><a href="#__codelineno-0-760">760</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-726"><a id="__codelineno-0-726" name="__codelineno-0-726"></a><span class="nd">@classmethod</span>
-</span><span id="__span-0-727"><a id="__codelineno-0-727" name="__codelineno-0-727"></a><span class="nd">@deprecated</span><span class="p">(</span>
-</span><span id="__span-0-728"><a id="__codelineno-0-728" name="__codelineno-0-728"></a>    <span class="n">target</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-</span><span id="__span-0-729"><a id="__codelineno-0-729" name="__codelineno-0-729"></a>    <span class="n">deprecated_in</span><span class="o">=</span><span class="s2">&quot;0.6.0&quot;</span><span class="p">,</span>
-</span><span id="__span-0-730"><a id="__codelineno-0-730" name="__codelineno-0-730"></a>    <span class="n">remove_in</span><span class="o">=</span><span class="s2">&quot;0.8.0&quot;</span><span class="p">,</span>
-</span><span id="__span-0-731"><a id="__codelineno-0-731" name="__codelineno-0-731"></a>    <span class="n">args_mapping</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">indices</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">data_names</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">n_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span>
-</span><span id="__span-0-732"><a id="__codelineno-0-732" name="__codelineno-0-732"></a>    <span class="n">template_mgs</span><span class="o">=</span><span class="s2">&quot;`</span><span class="si">%(source_name)s</span><span class="s2">` is deprecated for generating zero-filled &quot;</span>
-</span><span id="__span-0-733"><a id="__codelineno-0-733" name="__codelineno-0-733"></a>    <span class="s2">&quot;results, use `ValuationResult.zeros()` instead.&quot;</span><span class="p">,</span>
-</span><span id="__span-0-734"><a id="__codelineno-0-734" name="__codelineno-0-734"></a><span class="p">)</span>
-</span><span id="__span-0-735"><a id="__codelineno-0-735" name="__codelineno-0-735"></a><span class="k">def</span> <span class="nf">empty</span><span class="p">(</span>
-</span><span id="__span-0-736"><a id="__codelineno-0-736" name="__codelineno-0-736"></a>    <span class="bp">cls</span><span class="p">,</span>
-</span><span id="__span-0-737"><a id="__codelineno-0-737" name="__codelineno-0-737"></a>    <span class="n">algorithm</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="p">,</span>
-</span><span id="__span-0-738"><a id="__codelineno-0-738" name="__codelineno-0-738"></a>    <span class="n">indices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="n">IndexT</span><span class="p">]</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">IndexT</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-739"><a id="__codelineno-0-739" name="__codelineno-0-739"></a>    <span class="n">data_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="n">NameT</span><span class="p">]</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">NameT</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-740"><a id="__codelineno-0-740" name="__codelineno-0-740"></a>    <span class="n">n_samples</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
-</span><span id="__span-0-741"><a id="__codelineno-0-741" name="__codelineno-0-741"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ValuationResult&quot;</span><span class="p">:</span>
-</span><span id="__span-0-742"><a id="__codelineno-0-742" name="__codelineno-0-742"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Creates an empty [ValuationResult][pydvl.value.result.ValuationResult] object.</span>
-</span><span id="__span-0-743"><a id="__codelineno-0-743" name="__codelineno-0-743"></a>
-</span><span id="__span-0-744"><a id="__codelineno-0-744" name="__codelineno-0-744"></a><span class="sd">    Empty results are characterised by having an empty array of values. When</span>
-</span><span id="__span-0-745"><a id="__codelineno-0-745" name="__codelineno-0-745"></a><span class="sd">    another result is added to an empty one, the empty one is discarded.</span>
-</span><span id="__span-0-746"><a id="__codelineno-0-746" name="__codelineno-0-746"></a>
-</span><span id="__span-0-747"><a id="__codelineno-0-747" name="__codelineno-0-747"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-748"><a id="__codelineno-0-748" name="__codelineno-0-748"></a><span class="sd">        algorithm: Name of the algorithm used to compute the values</span>
-</span><span id="__span-0-749"><a id="__codelineno-0-749" name="__codelineno-0-749"></a>
-</span><span id="__span-0-750"><a id="__codelineno-0-750" name="__codelineno-0-750"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-751"><a id="__codelineno-0-751" name="__codelineno-0-751"></a><span class="sd">        Object with the results.</span>
-</span><span id="__span-0-752"><a id="__codelineno-0-752" name="__codelineno-0-752"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-753"><a id="__codelineno-0-753" name="__codelineno-0-753"></a>    <span class="k">if</span> <span class="n">indices</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">data_names</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">n_samples</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
-</span><span id="__span-0-754"><a id="__codelineno-0-754" name="__codelineno-0-754"></a>        <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span>
-</span><span id="__span-0-755"><a id="__codelineno-0-755" name="__codelineno-0-755"></a>            <span class="n">algorithm</span><span class="o">=</span><span class="n">algorithm</span><span class="p">,</span>
-</span><span id="__span-0-756"><a id="__codelineno-0-756" name="__codelineno-0-756"></a>            <span class="n">indices</span><span class="o">=</span><span class="n">indices</span><span class="p">,</span>
-</span><span id="__span-0-757"><a id="__codelineno-0-757" name="__codelineno-0-757"></a>            <span class="n">data_names</span><span class="o">=</span><span class="n">data_names</span><span class="p">,</span>
-</span><span id="__span-0-758"><a id="__codelineno-0-758" name="__codelineno-0-758"></a>            <span class="n">n_samples</span><span class="o">=</span><span class="n">n_samples</span><span class="p">,</span>
-</span><span id="__span-0-759"><a id="__codelineno-0-759" name="__codelineno-0-759"></a>        <span class="p">)</span>
-</span><span id="__span-0-760"><a id="__codelineno-0-760" name="__codelineno-0-760"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="n">algorithm</span><span class="o">=</span><span class="n">algorithm</span><span class="p">,</span> <span class="n">status</span><span class="o">=</span><span class="n">Status</span><span class="o">.</span><span class="n">Pending</span><span class="p">,</span> <span class="n">values</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([]))</span>
+<span class="normal"><a href="#__codelineno-0-756">756</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-722"><a id="__codelineno-0-722" name="__codelineno-0-722"></a><span class="nd">@classmethod</span>
+</span><span id="__span-0-723"><a id="__codelineno-0-723" name="__codelineno-0-723"></a><span class="nd">@deprecated</span><span class="p">(</span>
+</span><span id="__span-0-724"><a id="__codelineno-0-724" name="__codelineno-0-724"></a>    <span class="n">target</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+</span><span id="__span-0-725"><a id="__codelineno-0-725" name="__codelineno-0-725"></a>    <span class="n">deprecated_in</span><span class="o">=</span><span class="s2">&quot;0.6.0&quot;</span><span class="p">,</span>
+</span><span id="__span-0-726"><a id="__codelineno-0-726" name="__codelineno-0-726"></a>    <span class="n">remove_in</span><span class="o">=</span><span class="s2">&quot;0.8.0&quot;</span><span class="p">,</span>
+</span><span id="__span-0-727"><a id="__codelineno-0-727" name="__codelineno-0-727"></a>    <span class="n">args_mapping</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">indices</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">data_names</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">n_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span>
+</span><span id="__span-0-728"><a id="__codelineno-0-728" name="__codelineno-0-728"></a>    <span class="n">template_mgs</span><span class="o">=</span><span class="s2">&quot;`</span><span class="si">%(source_name)s</span><span class="s2">` is deprecated for generating zero-filled &quot;</span>
+</span><span id="__span-0-729"><a id="__codelineno-0-729" name="__codelineno-0-729"></a>    <span class="s2">&quot;results, use `ValuationResult.zeros()` instead.&quot;</span><span class="p">,</span>
+</span><span id="__span-0-730"><a id="__codelineno-0-730" name="__codelineno-0-730"></a><span class="p">)</span>
+</span><span id="__span-0-731"><a id="__codelineno-0-731" name="__codelineno-0-731"></a><span class="k">def</span> <span class="nf">empty</span><span class="p">(</span>
+</span><span id="__span-0-732"><a id="__codelineno-0-732" name="__codelineno-0-732"></a>    <span class="bp">cls</span><span class="p">,</span>
+</span><span id="__span-0-733"><a id="__codelineno-0-733" name="__codelineno-0-733"></a>    <span class="n">algorithm</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="p">,</span>
+</span><span id="__span-0-734"><a id="__codelineno-0-734" name="__codelineno-0-734"></a>    <span class="n">indices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="n">IndexT</span><span class="p">]</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">IndexT</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-735"><a id="__codelineno-0-735" name="__codelineno-0-735"></a>    <span class="n">data_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="n">NameT</span><span class="p">]</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">NameT</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-736"><a id="__codelineno-0-736" name="__codelineno-0-736"></a>    <span class="n">n_samples</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
+</span><span id="__span-0-737"><a id="__codelineno-0-737" name="__codelineno-0-737"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">:</span>
+</span><span id="__span-0-738"><a id="__codelineno-0-738" name="__codelineno-0-738"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Creates an empty [ValuationResult][pydvl.value.result.ValuationResult] object.</span>
+</span><span id="__span-0-739"><a id="__codelineno-0-739" name="__codelineno-0-739"></a>
+</span><span id="__span-0-740"><a id="__codelineno-0-740" name="__codelineno-0-740"></a><span class="sd">    Empty results are characterised by having an empty array of values. When</span>
+</span><span id="__span-0-741"><a id="__codelineno-0-741" name="__codelineno-0-741"></a><span class="sd">    another result is added to an empty one, the empty one is discarded.</span>
+</span><span id="__span-0-742"><a id="__codelineno-0-742" name="__codelineno-0-742"></a>
+</span><span id="__span-0-743"><a id="__codelineno-0-743" name="__codelineno-0-743"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-744"><a id="__codelineno-0-744" name="__codelineno-0-744"></a><span class="sd">        algorithm: Name of the algorithm used to compute the values</span>
+</span><span id="__span-0-745"><a id="__codelineno-0-745" name="__codelineno-0-745"></a>
+</span><span id="__span-0-746"><a id="__codelineno-0-746" name="__codelineno-0-746"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-747"><a id="__codelineno-0-747" name="__codelineno-0-747"></a><span class="sd">        Object with the results.</span>
+</span><span id="__span-0-748"><a id="__codelineno-0-748" name="__codelineno-0-748"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-749"><a id="__codelineno-0-749" name="__codelineno-0-749"></a>    <span class="k">if</span> <span class="n">indices</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">data_names</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">n_samples</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
+</span><span id="__span-0-750"><a id="__codelineno-0-750" name="__codelineno-0-750"></a>        <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span>
+</span><span id="__span-0-751"><a id="__codelineno-0-751" name="__codelineno-0-751"></a>            <span class="n">algorithm</span><span class="o">=</span><span class="n">algorithm</span><span class="p">,</span>
+</span><span id="__span-0-752"><a id="__codelineno-0-752" name="__codelineno-0-752"></a>            <span class="n">indices</span><span class="o">=</span><span class="n">indices</span><span class="p">,</span>
+</span><span id="__span-0-753"><a id="__codelineno-0-753" name="__codelineno-0-753"></a>            <span class="n">data_names</span><span class="o">=</span><span class="n">data_names</span><span class="p">,</span>
+</span><span id="__span-0-754"><a id="__codelineno-0-754" name="__codelineno-0-754"></a>            <span class="n">n_samples</span><span class="o">=</span><span class="n">n_samples</span><span class="p">,</span>
+</span><span id="__span-0-755"><a id="__codelineno-0-755" name="__codelineno-0-755"></a>        <span class="p">)</span>
+</span><span id="__span-0-756"><a id="__codelineno-0-756" name="__codelineno-0-756"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="n">algorithm</span><span class="o">=</span><span class="n">algorithm</span><span class="p">,</span> <span class="n">status</span><span class="o">=</span><span class="n">Status</span><span class="o">.</span><span class="n">Pending</span><span class="p">,</span> <span class="n">values</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([]))</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -4911,7 +4915,7 @@ <h3 id="pydvl.value.result.ValuationResult.zeros" class="doc doc-heading">
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-external" title="typing.Optional" href="https://docs.python.org/3/library/typing.html#typing.Optional">Optional</a>[<a class="autorefs autorefs-external" title="typing.Sequence" href="https://docs.python.org/3/library/typing.html#typing.Sequence">Sequence</a>[<span title="pydvl.value.result.IndexT">IndexT</span>] | <a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a>[<span title="pydvl.value.result.IndexT">IndexT</span>]]</code>
+                    <code><a class="autorefs autorefs-external" title="typing.Optional" href="https://docs.python.org/3/library/typing.html#typing.Optional">Optional</a>[<a class="autorefs autorefs-external" title="typing.Sequence" href="https://docs.python.org/3/library/typing.html#typing.Sequence">Sequence</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>] | <a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>]]</code>
                 </span>
                 <span class="doc-param-default">
                   <b>DEFAULT:</b> 
@@ -4930,7 +4934,7 @@ <h3 id="pydvl.value.result.ValuationResult.zeros" class="doc doc-heading">
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-external" title="typing.Optional" href="https://docs.python.org/3/library/typing.html#typing.Optional">Optional</a>[<a class="autorefs autorefs-external" title="typing.Sequence" href="https://docs.python.org/3/library/typing.html#typing.Sequence">Sequence</a>[<span title="pydvl.value.result.NameT">NameT</span>] | <a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a>[<span title="pydvl.value.result.NameT">NameT</span>]]</code>
+                    <code><a class="autorefs autorefs-external" title="typing.Optional" href="https://docs.python.org/3/library/typing.html#typing.Optional">Optional</a>[<a class="autorefs autorefs-external" title="typing.Sequence" href="https://docs.python.org/3/library/typing.html#typing.Sequence">Sequence</a>[<span title="pydvl.utils.types.NameT">NameT</span>] | <a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a>[<span title="pydvl.utils.types.NameT">NameT</span>]]</code>
                 </span>
                 <span class="doc-param-default">
                   <b>DEFAULT:</b> 
@@ -4974,7 +4978,7 @@ <h3 id="pydvl.value.result.ValuationResult.zeros" class="doc doc-heading">
         <tr>
           <td>
               <span class="doc-returns-annotation">
-                  <code>&#39;ValuationResult&#39;</code>
+                  <code><a class="autorefs autorefs-internal" title="pydvl.value.result.ValuationResult" href="#pydvl.value.result.ValuationResult">ValuationResult</a></code>
               </span>
           </td>
           <td class="doc-returns-details">
@@ -4988,7 +4992,11 @@ <h3 id="pydvl.value.result.ValuationResult.zeros" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/result.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-762">762</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-758">758</a></span>
+<span class="normal"><a href="#__codelineno-0-759">759</a></span>
+<span class="normal"><a href="#__codelineno-0-760">760</a></span>
+<span class="normal"><a href="#__codelineno-0-761">761</a></span>
+<span class="normal"><a href="#__codelineno-0-762">762</a></span>
 <span class="normal"><a href="#__codelineno-0-763">763</a></span>
 <span class="normal"><a href="#__codelineno-0-764">764</a></span>
 <span class="normal"><a href="#__codelineno-0-765">765</a></span>
@@ -5023,50 +5031,46 @@ <h3 id="pydvl.value.result.ValuationResult.zeros" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-794">794</a></span>
 <span class="normal"><a href="#__codelineno-0-795">795</a></span>
 <span class="normal"><a href="#__codelineno-0-796">796</a></span>
-<span class="normal"><a href="#__codelineno-0-797">797</a></span>
-<span class="normal"><a href="#__codelineno-0-798">798</a></span>
-<span class="normal"><a href="#__codelineno-0-799">799</a></span>
-<span class="normal"><a href="#__codelineno-0-800">800</a></span>
-<span class="normal"><a href="#__codelineno-0-801">801</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-762"><a id="__codelineno-0-762" name="__codelineno-0-762"></a><span class="nd">@classmethod</span>
-</span><span id="__span-0-763"><a id="__codelineno-0-763" name="__codelineno-0-763"></a><span class="k">def</span> <span class="nf">zeros</span><span class="p">(</span>
-</span><span id="__span-0-764"><a id="__codelineno-0-764" name="__codelineno-0-764"></a>    <span class="bp">cls</span><span class="p">,</span>
-</span><span id="__span-0-765"><a id="__codelineno-0-765" name="__codelineno-0-765"></a>    <span class="n">algorithm</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="p">,</span>
-</span><span id="__span-0-766"><a id="__codelineno-0-766" name="__codelineno-0-766"></a>    <span class="n">indices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="n">IndexT</span><span class="p">]</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">IndexT</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-767"><a id="__codelineno-0-767" name="__codelineno-0-767"></a>    <span class="n">data_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="n">NameT</span><span class="p">]</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">NameT</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-768"><a id="__codelineno-0-768" name="__codelineno-0-768"></a>    <span class="n">n_samples</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
-</span><span id="__span-0-769"><a id="__codelineno-0-769" name="__codelineno-0-769"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ValuationResult&quot;</span><span class="p">:</span>
-</span><span id="__span-0-770"><a id="__codelineno-0-770" name="__codelineno-0-770"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Creates an empty [ValuationResult][pydvl.value.result.ValuationResult] object.</span>
-</span><span id="__span-0-771"><a id="__codelineno-0-771" name="__codelineno-0-771"></a>
-</span><span id="__span-0-772"><a id="__codelineno-0-772" name="__codelineno-0-772"></a><span class="sd">    Empty results are characterised by having an empty array of values. When</span>
-</span><span id="__span-0-773"><a id="__codelineno-0-773" name="__codelineno-0-773"></a><span class="sd">    another result is added to an empty one, the empty one is ignored.</span>
-</span><span id="__span-0-774"><a id="__codelineno-0-774" name="__codelineno-0-774"></a>
-</span><span id="__span-0-775"><a id="__codelineno-0-775" name="__codelineno-0-775"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-776"><a id="__codelineno-0-776" name="__codelineno-0-776"></a><span class="sd">        algorithm: Name of the algorithm used to compute the values</span>
-</span><span id="__span-0-777"><a id="__codelineno-0-777" name="__codelineno-0-777"></a><span class="sd">        indices: Data indices to use. A copy will be made. If not given,</span>
-</span><span id="__span-0-778"><a id="__codelineno-0-778" name="__codelineno-0-778"></a><span class="sd">            the indices will be set to the range `[0, n_samples)`.</span>
-</span><span id="__span-0-779"><a id="__codelineno-0-779" name="__codelineno-0-779"></a><span class="sd">        data_names: Data names to use. A copy will be made. If not given,</span>
-</span><span id="__span-0-780"><a id="__codelineno-0-780" name="__codelineno-0-780"></a><span class="sd">            the names will be set to the string representation of the indices.</span>
-</span><span id="__span-0-781"><a id="__codelineno-0-781" name="__codelineno-0-781"></a><span class="sd">        n_samples: Number of data points whose values are computed. If</span>
-</span><span id="__span-0-782"><a id="__codelineno-0-782" name="__codelineno-0-782"></a><span class="sd">            not given, the length of `indices` will be used.</span>
-</span><span id="__span-0-783"><a id="__codelineno-0-783" name="__codelineno-0-783"></a>
-</span><span id="__span-0-784"><a id="__codelineno-0-784" name="__codelineno-0-784"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-785"><a id="__codelineno-0-785" name="__codelineno-0-785"></a><span class="sd">        Object with the results.</span>
-</span><span id="__span-0-786"><a id="__codelineno-0-786" name="__codelineno-0-786"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-787"><a id="__codelineno-0-787" name="__codelineno-0-787"></a>    <span class="k">if</span> <span class="n">indices</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-</span><span id="__span-0-788"><a id="__codelineno-0-788" name="__codelineno-0-788"></a>        <span class="n">indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="n">n_samples</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">)</span>
-</span><span id="__span-0-789"><a id="__codelineno-0-789" name="__codelineno-0-789"></a>    <span class="k">else</span><span class="p">:</span>
-</span><span id="__span-0-790"><a id="__codelineno-0-790" name="__codelineno-0-790"></a>        <span class="n">indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">indices</span><span class="p">)</span>
-</span><span id="__span-0-791"><a id="__codelineno-0-791" name="__codelineno-0-791"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span>
-</span><span id="__span-0-792"><a id="__codelineno-0-792" name="__codelineno-0-792"></a>        <span class="n">algorithm</span><span class="o">=</span><span class="n">algorithm</span><span class="p">,</span>
-</span><span id="__span-0-793"><a id="__codelineno-0-793" name="__codelineno-0-793"></a>        <span class="n">status</span><span class="o">=</span><span class="n">Status</span><span class="o">.</span><span class="n">Pending</span><span class="p">,</span>
-</span><span id="__span-0-794"><a id="__codelineno-0-794" name="__codelineno-0-794"></a>        <span class="n">indices</span><span class="o">=</span><span class="n">indices</span><span class="p">,</span>
-</span><span id="__span-0-795"><a id="__codelineno-0-795" name="__codelineno-0-795"></a>        <span class="n">data_names</span><span class="o">=</span><span class="n">data_names</span>
-</span><span id="__span-0-796"><a id="__codelineno-0-796" name="__codelineno-0-796"></a>        <span class="k">if</span> <span class="n">data_names</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-</span><span id="__span-0-797"><a id="__codelineno-0-797" name="__codelineno-0-797"></a>        <span class="k">else</span> <span class="n">np</span><span class="o">.</span><span class="n">empty_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">object</span><span class="p">),</span>
-</span><span id="__span-0-798"><a id="__codelineno-0-798" name="__codelineno-0-798"></a>        <span class="n">values</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">indices</span><span class="p">)),</span>
-</span><span id="__span-0-799"><a id="__codelineno-0-799" name="__codelineno-0-799"></a>        <span class="n">variances</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">indices</span><span class="p">)),</span>
-</span><span id="__span-0-800"><a id="__codelineno-0-800" name="__codelineno-0-800"></a>        <span class="n">counts</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">indices</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">),</span>
-</span><span id="__span-0-801"><a id="__codelineno-0-801" name="__codelineno-0-801"></a>    <span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-797">797</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-758"><a id="__codelineno-0-758" name="__codelineno-0-758"></a><span class="nd">@classmethod</span>
+</span><span id="__span-0-759"><a id="__codelineno-0-759" name="__codelineno-0-759"></a><span class="k">def</span> <span class="nf">zeros</span><span class="p">(</span>
+</span><span id="__span-0-760"><a id="__codelineno-0-760" name="__codelineno-0-760"></a>    <span class="bp">cls</span><span class="p">,</span>
+</span><span id="__span-0-761"><a id="__codelineno-0-761" name="__codelineno-0-761"></a>    <span class="n">algorithm</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="p">,</span>
+</span><span id="__span-0-762"><a id="__codelineno-0-762" name="__codelineno-0-762"></a>    <span class="n">indices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="n">IndexT</span><span class="p">]</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">IndexT</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-763"><a id="__codelineno-0-763" name="__codelineno-0-763"></a>    <span class="n">data_names</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="n">NameT</span><span class="p">]</span> <span class="o">|</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">NameT</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-764"><a id="__codelineno-0-764" name="__codelineno-0-764"></a>    <span class="n">n_samples</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
+</span><span id="__span-0-765"><a id="__codelineno-0-765" name="__codelineno-0-765"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">:</span>
+</span><span id="__span-0-766"><a id="__codelineno-0-766" name="__codelineno-0-766"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Creates an empty [ValuationResult][pydvl.value.result.ValuationResult] object.</span>
+</span><span id="__span-0-767"><a id="__codelineno-0-767" name="__codelineno-0-767"></a>
+</span><span id="__span-0-768"><a id="__codelineno-0-768" name="__codelineno-0-768"></a><span class="sd">    Empty results are characterised by having an empty array of values. When</span>
+</span><span id="__span-0-769"><a id="__codelineno-0-769" name="__codelineno-0-769"></a><span class="sd">    another result is added to an empty one, the empty one is ignored.</span>
+</span><span id="__span-0-770"><a id="__codelineno-0-770" name="__codelineno-0-770"></a>
+</span><span id="__span-0-771"><a id="__codelineno-0-771" name="__codelineno-0-771"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-772"><a id="__codelineno-0-772" name="__codelineno-0-772"></a><span class="sd">        algorithm: Name of the algorithm used to compute the values</span>
+</span><span id="__span-0-773"><a id="__codelineno-0-773" name="__codelineno-0-773"></a><span class="sd">        indices: Data indices to use. A copy will be made. If not given,</span>
+</span><span id="__span-0-774"><a id="__codelineno-0-774" name="__codelineno-0-774"></a><span class="sd">            the indices will be set to the range `[0, n_samples)`.</span>
+</span><span id="__span-0-775"><a id="__codelineno-0-775" name="__codelineno-0-775"></a><span class="sd">        data_names: Data names to use. A copy will be made. If not given,</span>
+</span><span id="__span-0-776"><a id="__codelineno-0-776" name="__codelineno-0-776"></a><span class="sd">            the names will be set to the string representation of the indices.</span>
+</span><span id="__span-0-777"><a id="__codelineno-0-777" name="__codelineno-0-777"></a><span class="sd">        n_samples: Number of data points whose values are computed. If</span>
+</span><span id="__span-0-778"><a id="__codelineno-0-778" name="__codelineno-0-778"></a><span class="sd">            not given, the length of `indices` will be used.</span>
+</span><span id="__span-0-779"><a id="__codelineno-0-779" name="__codelineno-0-779"></a>
+</span><span id="__span-0-780"><a id="__codelineno-0-780" name="__codelineno-0-780"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-781"><a id="__codelineno-0-781" name="__codelineno-0-781"></a><span class="sd">        Object with the results.</span>
+</span><span id="__span-0-782"><a id="__codelineno-0-782" name="__codelineno-0-782"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-783"><a id="__codelineno-0-783" name="__codelineno-0-783"></a>    <span class="k">if</span> <span class="n">indices</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+</span><span id="__span-0-784"><a id="__codelineno-0-784" name="__codelineno-0-784"></a>        <span class="n">indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="n">n_samples</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">)</span>
+</span><span id="__span-0-785"><a id="__codelineno-0-785" name="__codelineno-0-785"></a>    <span class="k">else</span><span class="p">:</span>
+</span><span id="__span-0-786"><a id="__codelineno-0-786" name="__codelineno-0-786"></a>        <span class="n">indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">)</span>
+</span><span id="__span-0-787"><a id="__codelineno-0-787" name="__codelineno-0-787"></a>    <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span>
+</span><span id="__span-0-788"><a id="__codelineno-0-788" name="__codelineno-0-788"></a>        <span class="n">algorithm</span><span class="o">=</span><span class="n">algorithm</span><span class="p">,</span>
+</span><span id="__span-0-789"><a id="__codelineno-0-789" name="__codelineno-0-789"></a>        <span class="n">status</span><span class="o">=</span><span class="n">Status</span><span class="o">.</span><span class="n">Pending</span><span class="p">,</span>
+</span><span id="__span-0-790"><a id="__codelineno-0-790" name="__codelineno-0-790"></a>        <span class="n">indices</span><span class="o">=</span><span class="n">indices</span><span class="p">,</span>
+</span><span id="__span-0-791"><a id="__codelineno-0-791" name="__codelineno-0-791"></a>        <span class="n">data_names</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">data_names</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">object</span><span class="p">)</span>
+</span><span id="__span-0-792"><a id="__codelineno-0-792" name="__codelineno-0-792"></a>        <span class="k">if</span> <span class="n">data_names</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+</span><span id="__span-0-793"><a id="__codelineno-0-793" name="__codelineno-0-793"></a>        <span class="k">else</span> <span class="n">np</span><span class="o">.</span><span class="n">empty_like</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">object</span><span class="p">),</span>
+</span><span id="__span-0-794"><a id="__codelineno-0-794" name="__codelineno-0-794"></a>        <span class="n">values</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">indices</span><span class="p">)),</span>
+</span><span id="__span-0-795"><a id="__codelineno-0-795" name="__codelineno-0-795"></a>        <span class="n">variances</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">indices</span><span class="p">)),</span>
+</span><span id="__span-0-796"><a id="__codelineno-0-796" name="__codelineno-0-796"></a>        <span class="n">counts</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">indices</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int_</span><span class="p">),</span>
+</span><span id="__span-0-797"><a id="__codelineno-0-797" name="__codelineno-0-797"></a>    <span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -5095,11 +5099,11 @@ <h3 id="pydvl.value.result.ValuationResult.zeros" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/sampler/index.html b/devel/api/pydvl/value/sampler/index.html
index 92e4cfc9b..79b936c36 100644
--- a/devel/api/pydvl/value/sampler/index.html
+++ b/devel/api/pydvl/value/sampler/index.html
@@ -2827,7 +2827,7 @@ <h2 id="pydvl.value.sampler.PowersetSampler" class="doc doc-heading">
 
   <div class="doc doc-contents ">
           <p class="doc doc-class-bases">
-            Bases: <code><a class="autorefs autorefs-external" title="abc.ABC" href="https://docs.python.org/3/library/abc.html#abc.ABC">ABC</a></code>, <code><a class="autorefs autorefs-external" title="typing.Iterable" href="https://docs.python.org/3/library/typing.html#typing.Iterable">Iterable</a>[<span title="pydvl.value.sampler.SampleT">SampleT</span>]</code>, <code><a class="autorefs autorefs-external" title="typing.Generic" href="https://docs.python.org/3/library/typing.html#typing.Generic">Generic</a>[<span title="pydvl.value.sampler.T">T</span>]</code></p>
+            Bases: <code><a class="autorefs autorefs-external" title="abc.ABC" href="https://docs.python.org/3/library/abc.html#abc.ABC">ABC</a></code>, <code><a class="autorefs autorefs-external" title="typing.Iterable" href="https://docs.python.org/3/library/typing.html#typing.Iterable">Iterable</a>[<span title="pydvl.value.sampler.SampleT">SampleT</span>]</code>, <code><a class="autorefs autorefs-external" title="typing.Generic" href="https://docs.python.org/3/library/typing.html#typing.Generic">Generic</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>]</code></p>
 
   
       <p>Samplers are custom iterables over subsets of indices.</p>
@@ -2867,7 +2867,10 @@ <h3 id="pydvl.value.sampler.PowersetSampler--slicing-of-samplers">Slicing of sam
 
               <details class="quote">
                 <summary>Source code in <code>src/pydvl/value/sampler.py</code></summary>
-                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-122">122</a></span>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-119">119</a></span>
+<span class="normal"><a href="#__codelineno-0-120">120</a></span>
+<span class="normal"><a href="#__codelineno-0-121">121</a></span>
+<span class="normal"><a href="#__codelineno-0-122">122</a></span>
 <span class="normal"><a href="#__codelineno-0-123">123</a></span>
 <span class="normal"><a href="#__codelineno-0-124">124</a></span>
 <span class="normal"><a href="#__codelineno-0-125">125</a></span>
@@ -2883,29 +2886,26 @@ <h3 id="pydvl.value.sampler.PowersetSampler--slicing-of-samplers">Slicing of sam
 <span class="normal"><a href="#__codelineno-0-135">135</a></span>
 <span class="normal"><a href="#__codelineno-0-136">136</a></span>
 <span class="normal"><a href="#__codelineno-0-137">137</a></span>
-<span class="normal"><a href="#__codelineno-0-138">138</a></span>
-<span class="normal"><a href="#__codelineno-0-139">139</a></span>
-<span class="normal"><a href="#__codelineno-0-140">140</a></span>
-<span class="normal"><a href="#__codelineno-0-141">141</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-122"><a id="__codelineno-0-122" name="__codelineno-0-122"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-</span><span id="__span-0-123"><a id="__codelineno-0-123" name="__codelineno-0-123"></a>    <span class="bp">self</span><span class="p">,</span>
-</span><span id="__span-0-124"><a id="__codelineno-0-124" name="__codelineno-0-124"></a>    <span class="n">indices</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">],</span>
-</span><span id="__span-0-125"><a id="__codelineno-0-125" name="__codelineno-0-125"></a>    <span class="n">index_iteration</span><span class="p">:</span> <span class="n">IndexIteration</span> <span class="o">=</span> <span class="n">IndexIteration</span><span class="o">.</span><span class="n">Sequential</span><span class="p">,</span>
-</span><span id="__span-0-126"><a id="__codelineno-0-126" name="__codelineno-0-126"></a>    <span class="n">outer_indices</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-127"><a id="__codelineno-0-127" name="__codelineno-0-127"></a><span class="p">):</span>
-</span><span id="__span-0-128"><a id="__codelineno-0-128" name="__codelineno-0-128"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-</span><span id="__span-0-129"><a id="__codelineno-0-129" name="__codelineno-0-129"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-130"><a id="__codelineno-0-130" name="__codelineno-0-130"></a><span class="sd">        indices: The set of items (indices) to sample from.</span>
-</span><span id="__span-0-131"><a id="__codelineno-0-131" name="__codelineno-0-131"></a><span class="sd">        index_iteration: the order in which indices are iterated over</span>
-</span><span id="__span-0-132"><a id="__codelineno-0-132" name="__codelineno-0-132"></a><span class="sd">        outer_indices: The set of items (indices) over which to iterate</span>
-</span><span id="__span-0-133"><a id="__codelineno-0-133" name="__codelineno-0-133"></a><span class="sd">            when sampling. Subsets are taken from the complement of each index</span>
-</span><span id="__span-0-134"><a id="__codelineno-0-134" name="__codelineno-0-134"></a><span class="sd">            in succession. For embarrassingly parallel computations, this set</span>
-</span><span id="__span-0-135"><a id="__codelineno-0-135" name="__codelineno-0-135"></a><span class="sd">            is sliced and the samplers are used to iterate over the slices.</span>
-</span><span id="__span-0-136"><a id="__codelineno-0-136" name="__codelineno-0-136"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-137"><a id="__codelineno-0-137" name="__codelineno-0-137"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span> <span class="o">=</span> <span class="n">indices</span>
-</span><span id="__span-0-138"><a id="__codelineno-0-138" name="__codelineno-0-138"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_index_iteration</span> <span class="o">=</span> <span class="n">index_iteration</span>
-</span><span id="__span-0-139"><a id="__codelineno-0-139" name="__codelineno-0-139"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_outer_indices</span> <span class="o">=</span> <span class="n">outer_indices</span> <span class="k">if</span> <span class="n">outer_indices</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">indices</span>
-</span><span id="__span-0-140"><a id="__codelineno-0-140" name="__codelineno-0-140"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_n</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">indices</span><span class="p">)</span>
-</span><span id="__span-0-141"><a id="__codelineno-0-141" name="__codelineno-0-141"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_n_samples</span> <span class="o">=</span> <span class="mi">0</span>
+<span class="normal"><a href="#__codelineno-0-138">138</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-119"><a id="__codelineno-0-119" name="__codelineno-0-119"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+</span><span id="__span-0-120"><a id="__codelineno-0-120" name="__codelineno-0-120"></a>    <span class="bp">self</span><span class="p">,</span>
+</span><span id="__span-0-121"><a id="__codelineno-0-121" name="__codelineno-0-121"></a>    <span class="n">indices</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">IndexT</span><span class="p">],</span>
+</span><span id="__span-0-122"><a id="__codelineno-0-122" name="__codelineno-0-122"></a>    <span class="n">index_iteration</span><span class="p">:</span> <span class="n">IndexIteration</span> <span class="o">=</span> <span class="n">IndexIteration</span><span class="o">.</span><span class="n">Sequential</span><span class="p">,</span>
+</span><span id="__span-0-123"><a id="__codelineno-0-123" name="__codelineno-0-123"></a>    <span class="n">outer_indices</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">IndexT</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-124"><a id="__codelineno-0-124" name="__codelineno-0-124"></a><span class="p">):</span>
+</span><span id="__span-0-125"><a id="__codelineno-0-125" name="__codelineno-0-125"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+</span><span id="__span-0-126"><a id="__codelineno-0-126" name="__codelineno-0-126"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-127"><a id="__codelineno-0-127" name="__codelineno-0-127"></a><span class="sd">        indices: The set of items (indices) to sample from.</span>
+</span><span id="__span-0-128"><a id="__codelineno-0-128" name="__codelineno-0-128"></a><span class="sd">        index_iteration: the order in which indices are iterated over</span>
+</span><span id="__span-0-129"><a id="__codelineno-0-129" name="__codelineno-0-129"></a><span class="sd">        outer_indices: The set of items (indices) over which to iterate</span>
+</span><span id="__span-0-130"><a id="__codelineno-0-130" name="__codelineno-0-130"></a><span class="sd">            when sampling. Subsets are taken from the complement of each index</span>
+</span><span id="__span-0-131"><a id="__codelineno-0-131" name="__codelineno-0-131"></a><span class="sd">            in succession. For embarrassingly parallel computations, this set</span>
+</span><span id="__span-0-132"><a id="__codelineno-0-132" name="__codelineno-0-132"></a><span class="sd">            is sliced and the samplers are used to iterate over the slices.</span>
+</span><span id="__span-0-133"><a id="__codelineno-0-133" name="__codelineno-0-133"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-134"><a id="__codelineno-0-134" name="__codelineno-0-134"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_indices</span> <span class="o">=</span> <span class="n">indices</span>
+</span><span id="__span-0-135"><a id="__codelineno-0-135" name="__codelineno-0-135"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_index_iteration</span> <span class="o">=</span> <span class="n">index_iteration</span>
+</span><span id="__span-0-136"><a id="__codelineno-0-136" name="__codelineno-0-136"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_outer_indices</span> <span class="o">=</span> <span class="n">outer_indices</span> <span class="k">if</span> <span class="n">outer_indices</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">indices</span>
+</span><span id="__span-0-137"><a id="__codelineno-0-137" name="__codelineno-0-137"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_n</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">indices</span><span class="p">)</span>
+</span><span id="__span-0-138"><a id="__codelineno-0-138" name="__codelineno-0-138"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_n_samples</span> <span class="o">=</span> <span class="mi">0</span>
 </span></code></pre></div></td></tr></table></div>
               </details>
 
@@ -2943,7 +2943,10 @@ <h3 id="pydvl.value.sampler.PowersetSampler.iterindices" class="doc doc-heading"
 </details>
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/sampler.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-162">162</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-159">159</a></span>
+<span class="normal"><a href="#__codelineno-0-160">160</a></span>
+<span class="normal"><a href="#__codelineno-0-161">161</a></span>
+<span class="normal"><a href="#__codelineno-0-162">162</a></span>
 <span class="normal"><a href="#__codelineno-0-163">163</a></span>
 <span class="normal"><a href="#__codelineno-0-164">164</a></span>
 <span class="normal"><a href="#__codelineno-0-165">165</a></span>
@@ -2951,21 +2954,18 @@ <h3 id="pydvl.value.sampler.PowersetSampler.iterindices" class="doc doc-heading"
 <span class="normal"><a href="#__codelineno-0-167">167</a></span>
 <span class="normal"><a href="#__codelineno-0-168">168</a></span>
 <span class="normal"><a href="#__codelineno-0-169">169</a></span>
-<span class="normal"><a href="#__codelineno-0-170">170</a></span>
-<span class="normal"><a href="#__codelineno-0-171">171</a></span>
-<span class="normal"><a href="#__codelineno-0-172">172</a></span>
-<span class="normal"><a href="#__codelineno-0-173">173</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-162"><a id="__codelineno-0-162" name="__codelineno-0-162"></a><span class="k">def</span> <span class="nf">iterindices</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Iterator</span><span class="p">[</span><span class="n">T</span><span class="p">]:</span>
-</span><span id="__span-0-163"><a id="__codelineno-0-163" name="__codelineno-0-163"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Iterates over indices in the order specified at construction.</span>
-</span><span id="__span-0-164"><a id="__codelineno-0-164" name="__codelineno-0-164"></a>
-</span><span id="__span-0-165"><a id="__codelineno-0-165" name="__codelineno-0-165"></a><span class="sd">    FIXME: this is probably not very useful, but I couldn&#39;t decide</span>
-</span><span id="__span-0-166"><a id="__codelineno-0-166" name="__codelineno-0-166"></a><span class="sd">      which method is better</span>
-</span><span id="__span-0-167"><a id="__codelineno-0-167" name="__codelineno-0-167"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-168"><a id="__codelineno-0-168" name="__codelineno-0-168"></a>    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_index_iteration</span> <span class="ow">is</span> <span class="n">PowersetSampler</span><span class="o">.</span><span class="n">IndexIteration</span><span class="o">.</span><span class="n">Sequential</span><span class="p">:</span>
-</span><span id="__span-0-169"><a id="__codelineno-0-169" name="__codelineno-0-169"></a>        <span class="k">for</span> <span class="n">idx</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_outer_indices</span><span class="p">:</span>
-</span><span id="__span-0-170"><a id="__codelineno-0-170" name="__codelineno-0-170"></a>            <span class="k">yield</span> <span class="n">idx</span>
-</span><span id="__span-0-171"><a id="__codelineno-0-171" name="__codelineno-0-171"></a>    <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_index_iteration</span> <span class="ow">is</span> <span class="n">PowersetSampler</span><span class="o">.</span><span class="n">IndexIteration</span><span class="o">.</span><span class="n">Random</span><span class="p">:</span>
-</span><span id="__span-0-172"><a id="__codelineno-0-172" name="__codelineno-0-172"></a>        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
-</span><span id="__span-0-173"><a id="__codelineno-0-173" name="__codelineno-0-173"></a>            <span class="k">yield</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_outer_indices</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
+<span class="normal"><a href="#__codelineno-0-170">170</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-159"><a id="__codelineno-0-159" name="__codelineno-0-159"></a><span class="k">def</span> <span class="nf">iterindices</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Iterator</span><span class="p">[</span><span class="n">IndexT</span><span class="p">]:</span>
+</span><span id="__span-0-160"><a id="__codelineno-0-160" name="__codelineno-0-160"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Iterates over indices in the order specified at construction.</span>
+</span><span id="__span-0-161"><a id="__codelineno-0-161" name="__codelineno-0-161"></a>
+</span><span id="__span-0-162"><a id="__codelineno-0-162" name="__codelineno-0-162"></a><span class="sd">    FIXME: this is probably not very useful, but I couldn&#39;t decide</span>
+</span><span id="__span-0-163"><a id="__codelineno-0-163" name="__codelineno-0-163"></a><span class="sd">      which method is better</span>
+</span><span id="__span-0-164"><a id="__codelineno-0-164" name="__codelineno-0-164"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-165"><a id="__codelineno-0-165" name="__codelineno-0-165"></a>    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_index_iteration</span> <span class="ow">is</span> <span class="n">PowersetSampler</span><span class="o">.</span><span class="n">IndexIteration</span><span class="o">.</span><span class="n">Sequential</span><span class="p">:</span>
+</span><span id="__span-0-166"><a id="__codelineno-0-166" name="__codelineno-0-166"></a>        <span class="k">for</span> <span class="n">idx</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_outer_indices</span><span class="p">:</span>
+</span><span id="__span-0-167"><a id="__codelineno-0-167" name="__codelineno-0-167"></a>            <span class="k">yield</span> <span class="n">idx</span>
+</span><span id="__span-0-168"><a id="__codelineno-0-168" name="__codelineno-0-168"></a>    <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_index_iteration</span> <span class="ow">is</span> <span class="n">PowersetSampler</span><span class="o">.</span><span class="n">IndexIteration</span><span class="o">.</span><span class="n">Random</span><span class="p">:</span>
+</span><span id="__span-0-169"><a id="__codelineno-0-169" name="__codelineno-0-169"></a>        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+</span><span id="__span-0-170"><a id="__codelineno-0-170" name="__codelineno-0-170"></a>            <span class="k">yield</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_outer_indices</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -2990,11 +2990,11 @@ <h3 id="pydvl.value.sampler.PowersetSampler.__len__" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/sampler.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-192">192</a></span>
-<span class="normal"><a href="#__codelineno-0-193">193</a></span>
-<span class="normal"><a href="#__codelineno-0-194">194</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-192"><a id="__codelineno-0-192" name="__codelineno-0-192"></a><span class="k">def</span> <span class="fm">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
-</span><span id="__span-0-193"><a id="__codelineno-0-193" name="__codelineno-0-193"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the number of outer indices over which the sampler iterates.&quot;&quot;&quot;</span>
-</span><span id="__span-0-194"><a id="__codelineno-0-194" name="__codelineno-0-194"></a>    <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_outer_indices</span><span class="p">)</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-189">189</a></span>
+<span class="normal"><a href="#__codelineno-0-190">190</a></span>
+<span class="normal"><a href="#__codelineno-0-191">191</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-189"><a id="__codelineno-0-189" name="__codelineno-0-189"></a><span class="k">def</span> <span class="fm">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+</span><span id="__span-0-190"><a id="__codelineno-0-190" name="__codelineno-0-190"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the number of outer indices over which the sampler iterates.&quot;&quot;&quot;</span>
+</span><span id="__span-0-191"><a id="__codelineno-0-191" name="__codelineno-0-191"></a>    <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_outer_indices</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3033,7 +3033,10 @@ <h3 id="pydvl.value.sampler.PowersetSampler.weight" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/sampler.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-206">206</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-203">203</a></span>
+<span class="normal"><a href="#__codelineno-0-204">204</a></span>
+<span class="normal"><a href="#__codelineno-0-205">205</a></span>
+<span class="normal"><a href="#__codelineno-0-206">206</a></span>
 <span class="normal"><a href="#__codelineno-0-207">207</a></span>
 <span class="normal"><a href="#__codelineno-0-208">208</a></span>
 <span class="normal"><a href="#__codelineno-0-209">209</a></span>
@@ -3047,27 +3050,24 @@ <h3 id="pydvl.value.sampler.PowersetSampler.weight" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-217">217</a></span>
 <span class="normal"><a href="#__codelineno-0-218">218</a></span>
 <span class="normal"><a href="#__codelineno-0-219">219</a></span>
-<span class="normal"><a href="#__codelineno-0-220">220</a></span>
-<span class="normal"><a href="#__codelineno-0-221">221</a></span>
-<span class="normal"><a href="#__codelineno-0-222">222</a></span>
-<span class="normal"><a href="#__codelineno-0-223">223</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-206"><a id="__codelineno-0-206" name="__codelineno-0-206"></a><span class="nd">@classmethod</span>
-</span><span id="__span-0-207"><a id="__codelineno-0-207" name="__codelineno-0-207"></a><span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
-</span><span id="__span-0-208"><a id="__codelineno-0-208" name="__codelineno-0-208"></a><span class="k">def</span> <span class="nf">weight</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">n</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">subset_len</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
-</span><span id="__span-0-209"><a id="__codelineno-0-209" name="__codelineno-0-209"></a><span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Factor by which to multiply Monte Carlo samples, so that the</span>
-</span><span id="__span-0-210"><a id="__codelineno-0-210" name="__codelineno-0-210"></a><span class="sd">    mean converges to the desired expression.</span>
-</span><span id="__span-0-211"><a id="__codelineno-0-211" name="__codelineno-0-211"></a>
-</span><span id="__span-0-212"><a id="__codelineno-0-212" name="__codelineno-0-212"></a><span class="sd">    By the Law of Large Numbers, the sample mean of $\delta_i(S_j)$</span>
-</span><span id="__span-0-213"><a id="__codelineno-0-213" name="__codelineno-0-213"></a><span class="sd">    converges to the expectation under the distribution from which $S_j$ is</span>
-</span><span id="__span-0-214"><a id="__codelineno-0-214" name="__codelineno-0-214"></a><span class="sd">    sampled.</span>
-</span><span id="__span-0-215"><a id="__codelineno-0-215" name="__codelineno-0-215"></a>
-</span><span id="__span-0-216"><a id="__codelineno-0-216" name="__codelineno-0-216"></a><span class="sd">    $$ \frac{1}{m}  \sum_{j = 1}^m \delta_i (S_j) c (S_j) \longrightarrow</span>
-</span><span id="__span-0-217"><a id="__codelineno-0-217" name="__codelineno-0-217"></a><span class="sd">       \underset{S \sim \mathcal{D}_{- i}}{\mathbb{E}} [\delta_i (S) c (</span>
-</span><span id="__span-0-218"><a id="__codelineno-0-218" name="__codelineno-0-218"></a><span class="sd">       S)]$$</span>
-</span><span id="__span-0-219"><a id="__codelineno-0-219" name="__codelineno-0-219"></a>
-</span><span id="__span-0-220"><a id="__codelineno-0-220" name="__codelineno-0-220"></a><span class="sd">    We add a factor $c(S_j)$ in order to have this expectation coincide with</span>
-</span><span id="__span-0-221"><a id="__codelineno-0-221" name="__codelineno-0-221"></a><span class="sd">    the desired expression.</span>
-</span><span id="__span-0-222"><a id="__codelineno-0-222" name="__codelineno-0-222"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-223"><a id="__codelineno-0-223" name="__codelineno-0-223"></a>    <span class="o">...</span>
+<span class="normal"><a href="#__codelineno-0-220">220</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-203"><a id="__codelineno-0-203" name="__codelineno-0-203"></a><span class="nd">@classmethod</span>
+</span><span id="__span-0-204"><a id="__codelineno-0-204" name="__codelineno-0-204"></a><span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
+</span><span id="__span-0-205"><a id="__codelineno-0-205" name="__codelineno-0-205"></a><span class="k">def</span> <span class="nf">weight</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">n</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">subset_len</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
+</span><span id="__span-0-206"><a id="__codelineno-0-206" name="__codelineno-0-206"></a><span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Factor by which to multiply Monte Carlo samples, so that the</span>
+</span><span id="__span-0-207"><a id="__codelineno-0-207" name="__codelineno-0-207"></a><span class="sd">    mean converges to the desired expression.</span>
+</span><span id="__span-0-208"><a id="__codelineno-0-208" name="__codelineno-0-208"></a>
+</span><span id="__span-0-209"><a id="__codelineno-0-209" name="__codelineno-0-209"></a><span class="sd">    By the Law of Large Numbers, the sample mean of $\delta_i(S_j)$</span>
+</span><span id="__span-0-210"><a id="__codelineno-0-210" name="__codelineno-0-210"></a><span class="sd">    converges to the expectation under the distribution from which $S_j$ is</span>
+</span><span id="__span-0-211"><a id="__codelineno-0-211" name="__codelineno-0-211"></a><span class="sd">    sampled.</span>
+</span><span id="__span-0-212"><a id="__codelineno-0-212" name="__codelineno-0-212"></a>
+</span><span id="__span-0-213"><a id="__codelineno-0-213" name="__codelineno-0-213"></a><span class="sd">    $$ \frac{1}{m}  \sum_{j = 1}^m \delta_i (S_j) c (S_j) \longrightarrow</span>
+</span><span id="__span-0-214"><a id="__codelineno-0-214" name="__codelineno-0-214"></a><span class="sd">       \underset{S \sim \mathcal{D}_{- i}}{\mathbb{E}} [\delta_i (S) c (</span>
+</span><span id="__span-0-215"><a id="__codelineno-0-215" name="__codelineno-0-215"></a><span class="sd">       S)]$$</span>
+</span><span id="__span-0-216"><a id="__codelineno-0-216" name="__codelineno-0-216"></a>
+</span><span id="__span-0-217"><a id="__codelineno-0-217" name="__codelineno-0-217"></a><span class="sd">    We add a factor $c(S_j)$ in order to have this expectation coincide with</span>
+</span><span id="__span-0-218"><a id="__codelineno-0-218" name="__codelineno-0-218"></a><span class="sd">    the desired expression.</span>
+</span><span id="__span-0-219"><a id="__codelineno-0-219" name="__codelineno-0-219"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-220"><a id="__codelineno-0-220" name="__codelineno-0-220"></a>    <span class="o">...</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3100,11 +3100,11 @@ <h2 id="pydvl.value.sampler.StochasticSamplerMixin" class="doc doc-heading">
 
               <details class="quote">
                 <summary>Source code in <code>src/pydvl/value/sampler.py</code></summary>
-                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-229">229</a></span>
-<span class="normal"><a href="#__codelineno-0-230">230</a></span>
-<span class="normal"><a href="#__codelineno-0-231">231</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-229"><a id="__codelineno-0-229" name="__codelineno-0-229"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-</span><span id="__span-0-230"><a id="__codelineno-0-230" name="__codelineno-0-230"></a>    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
-</span><span id="__span-0-231"><a id="__codelineno-0-231" name="__codelineno-0-231"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_rng</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">default_rng</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-226">226</a></span>
+<span class="normal"><a href="#__codelineno-0-227">227</a></span>
+<span class="normal"><a href="#__codelineno-0-228">228</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-226"><a id="__codelineno-0-226" name="__codelineno-0-226"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+</span><span id="__span-0-227"><a id="__codelineno-0-227" name="__codelineno-0-227"></a>    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+</span><span id="__span-0-228"><a id="__codelineno-0-228" name="__codelineno-0-228"></a>    <span class="bp">self</span><span class="o">.</span><span class="n">_rng</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">default_rng</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
               </details>
 
@@ -3141,7 +3141,7 @@ <h2 id="pydvl.value.sampler.DeterministicUniformSampler" class="doc doc-heading"
 
   <div class="doc doc-contents ">
           <p class="doc doc-class-bases">
-            Bases: <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PowersetSampler" href="#pydvl.value.sampler.PowersetSampler">PowersetSampler</a>[<span title="pydvl.value.sampler.T">T</span>]</code></p>
+            Bases: <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PowersetSampler" href="#pydvl.value.sampler.PowersetSampler">PowersetSampler</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>]</code></p>
 
   
       <p>For every index <span class="arithmatex">\(i\)</span>, each subset of the complement <code>indices - {i}</code> is
@@ -3178,7 +3178,7 @@ <h2 id="pydvl.value.sampler.DeterministicUniformSampler" class="doc doc-heading"
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a>[<span title="pydvl.value.sampler.T">T</span>]</code>
+                    <code><a class="autorefs autorefs-external" title="numpy.typing.NDArray" href="https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray">NDArray</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>]</code>
                 </span>
             </p>
           </td>
@@ -3188,7 +3188,10 @@ <h2 id="pydvl.value.sampler.DeterministicUniformSampler" class="doc doc-heading"
 
               <details class="quote">
                 <summary>Source code in <code>src/pydvl/value/sampler.py</code></summary>
-                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-235">235</a></span>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-232">232</a></span>
+<span class="normal"><a href="#__codelineno-0-233">233</a></span>
+<span class="normal"><a href="#__codelineno-0-234">234</a></span>
+<span class="normal"><a href="#__codelineno-0-235">235</a></span>
 <span class="normal"><a href="#__codelineno-0-236">236</a></span>
 <span class="normal"><a href="#__codelineno-0-237">237</a></span>
 <span class="normal"><a href="#__codelineno-0-238">238</a></span>
@@ -3207,32 +3210,29 @@ <h2 id="pydvl.value.sampler.DeterministicUniformSampler" class="doc doc-heading"
 <span class="normal"><a href="#__codelineno-0-251">251</a></span>
 <span class="normal"><a href="#__codelineno-0-252">252</a></span>
 <span class="normal"><a href="#__codelineno-0-253">253</a></span>
-<span class="normal"><a href="#__codelineno-0-254">254</a></span>
-<span class="normal"><a href="#__codelineno-0-255">255</a></span>
-<span class="normal"><a href="#__codelineno-0-256">256</a></span>
-<span class="normal"><a href="#__codelineno-0-257">257</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-235"><a id="__codelineno-0-235" name="__codelineno-0-235"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">indices</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">T</span><span class="p">],</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-</span><span id="__span-0-236"><a id="__codelineno-0-236" name="__codelineno-0-236"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;An iterator to perform uniform deterministic sampling of subsets.</span>
+<span class="normal"><a href="#__codelineno-0-254">254</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-232"><a id="__codelineno-0-232" name="__codelineno-0-232"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">indices</span><span class="p">:</span> <span class="n">NDArray</span><span class="p">[</span><span class="n">IndexT</span><span class="p">],</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+</span><span id="__span-0-233"><a id="__codelineno-0-233" name="__codelineno-0-233"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;An iterator to perform uniform deterministic sampling of subsets.</span>
+</span><span id="__span-0-234"><a id="__codelineno-0-234" name="__codelineno-0-234"></a>
+</span><span id="__span-0-235"><a id="__codelineno-0-235" name="__codelineno-0-235"></a><span class="sd">    For every index $i$, each subset of the complement `indices - {i}` is</span>
+</span><span id="__span-0-236"><a id="__codelineno-0-236" name="__codelineno-0-236"></a><span class="sd">    returned.</span>
 </span><span id="__span-0-237"><a id="__codelineno-0-237" name="__codelineno-0-237"></a>
-</span><span id="__span-0-238"><a id="__codelineno-0-238" name="__codelineno-0-238"></a><span class="sd">    For every index $i$, each subset of the complement `indices - {i}` is</span>
-</span><span id="__span-0-239"><a id="__codelineno-0-239" name="__codelineno-0-239"></a><span class="sd">    returned.</span>
-</span><span id="__span-0-240"><a id="__codelineno-0-240" name="__codelineno-0-240"></a>
-</span><span id="__span-0-241"><a id="__codelineno-0-241" name="__codelineno-0-241"></a><span class="sd">    !!! Note</span>
-</span><span id="__span-0-242"><a id="__codelineno-0-242" name="__codelineno-0-242"></a><span class="sd">        Indices are always iterated over sequentially, irrespective of</span>
-</span><span id="__span-0-243"><a id="__codelineno-0-243" name="__codelineno-0-243"></a><span class="sd">        the value of `index_iteration` upon construction.</span>
-</span><span id="__span-0-244"><a id="__codelineno-0-244" name="__codelineno-0-244"></a>
-</span><span id="__span-0-245"><a id="__codelineno-0-245" name="__codelineno-0-245"></a><span class="sd">    ??? Example</span>
-</span><span id="__span-0-246"><a id="__codelineno-0-246" name="__codelineno-0-246"></a><span class="sd">        ``` pycon</span>
-</span><span id="__span-0-247"><a id="__codelineno-0-247" name="__codelineno-0-247"></a><span class="sd">        &gt;&gt;&gt; for idx, s in DeterministicUniformSampler(np.arange(2)):</span>
-</span><span id="__span-0-248"><a id="__codelineno-0-248" name="__codelineno-0-248"></a><span class="sd">        &gt;&gt;&gt;    print(f&quot;{idx} - {s}&quot;, end=&quot;, &quot;)</span>
-</span><span id="__span-0-249"><a id="__codelineno-0-249" name="__codelineno-0-249"></a><span class="sd">        1 - [], 1 - [2], 2 - [], 2 - [1],</span>
-</span><span id="__span-0-250"><a id="__codelineno-0-250" name="__codelineno-0-250"></a><span class="sd">        ```</span>
-</span><span id="__span-0-251"><a id="__codelineno-0-251" name="__codelineno-0-251"></a>
-</span><span id="__span-0-252"><a id="__codelineno-0-252" name="__codelineno-0-252"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-253"><a id="__codelineno-0-253" name="__codelineno-0-253"></a><span class="sd">        indices: The set of items (indices) to sample from.</span>
-</span><span id="__span-0-254"><a id="__codelineno-0-254" name="__codelineno-0-254"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-255"><a id="__codelineno-0-255" name="__codelineno-0-255"></a>    <span class="c1"># Force sequential iteration</span>
-</span><span id="__span-0-256"><a id="__codelineno-0-256" name="__codelineno-0-256"></a>    <span class="n">kwargs</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s2">&quot;index_iteration&quot;</span><span class="p">:</span> <span class="n">PowersetSampler</span><span class="o">.</span><span class="n">IndexIteration</span><span class="o">.</span><span class="n">Sequential</span><span class="p">})</span>
-</span><span id="__span-0-257"><a id="__codelineno-0-257" name="__codelineno-0-257"></a>    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+</span><span id="__span-0-238"><a id="__codelineno-0-238" name="__codelineno-0-238"></a><span class="sd">    !!! Note</span>
+</span><span id="__span-0-239"><a id="__codelineno-0-239" name="__codelineno-0-239"></a><span class="sd">        Indices are always iterated over sequentially, irrespective of</span>
+</span><span id="__span-0-240"><a id="__codelineno-0-240" name="__codelineno-0-240"></a><span class="sd">        the value of `index_iteration` upon construction.</span>
+</span><span id="__span-0-241"><a id="__codelineno-0-241" name="__codelineno-0-241"></a>
+</span><span id="__span-0-242"><a id="__codelineno-0-242" name="__codelineno-0-242"></a><span class="sd">    ??? Example</span>
+</span><span id="__span-0-243"><a id="__codelineno-0-243" name="__codelineno-0-243"></a><span class="sd">        ``` pycon</span>
+</span><span id="__span-0-244"><a id="__codelineno-0-244" name="__codelineno-0-244"></a><span class="sd">        &gt;&gt;&gt; for idx, s in DeterministicUniformSampler(np.arange(2)):</span>
+</span><span id="__span-0-245"><a id="__codelineno-0-245" name="__codelineno-0-245"></a><span class="sd">        &gt;&gt;&gt;    print(f&quot;{idx} - {s}&quot;, end=&quot;, &quot;)</span>
+</span><span id="__span-0-246"><a id="__codelineno-0-246" name="__codelineno-0-246"></a><span class="sd">        1 - [], 1 - [2], 2 - [], 2 - [1],</span>
+</span><span id="__span-0-247"><a id="__codelineno-0-247" name="__codelineno-0-247"></a><span class="sd">        ```</span>
+</span><span id="__span-0-248"><a id="__codelineno-0-248" name="__codelineno-0-248"></a>
+</span><span id="__span-0-249"><a id="__codelineno-0-249" name="__codelineno-0-249"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-250"><a id="__codelineno-0-250" name="__codelineno-0-250"></a><span class="sd">        indices: The set of items (indices) to sample from.</span>
+</span><span id="__span-0-251"><a id="__codelineno-0-251" name="__codelineno-0-251"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-252"><a id="__codelineno-0-252" name="__codelineno-0-252"></a>    <span class="c1"># Force sequential iteration</span>
+</span><span id="__span-0-253"><a id="__codelineno-0-253" name="__codelineno-0-253"></a>    <span class="n">kwargs</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s2">&quot;index_iteration&quot;</span><span class="p">:</span> <span class="n">PowersetSampler</span><span class="o">.</span><span class="n">IndexIteration</span><span class="o">.</span><span class="n">Sequential</span><span class="p">})</span>
+</span><span id="__span-0-254"><a id="__codelineno-0-254" name="__codelineno-0-254"></a>    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
               </details>
 
@@ -3270,7 +3270,7 @@ <h2 id="pydvl.value.sampler.UniformSampler" class="doc doc-heading">
 
   <div class="doc doc-contents ">
           <p class="doc doc-class-bases">
-            Bases: <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.StochasticSamplerMixin" href="#pydvl.value.sampler.StochasticSamplerMixin">StochasticSamplerMixin</a></code>, <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PowersetSampler" href="#pydvl.value.sampler.PowersetSampler">PowersetSampler</a>[<span title="pydvl.value.sampler.T">T</span>]</code></p>
+            Bases: <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.StochasticSamplerMixin" href="#pydvl.value.sampler.StochasticSamplerMixin">StochasticSamplerMixin</a></code>, <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PowersetSampler" href="#pydvl.value.sampler.PowersetSampler">PowersetSampler</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>]</code></p>
 
   
       <p>An iterator to perform uniform random sampling of subsets.</p>
@@ -3327,17 +3327,17 @@ <h3 id="pydvl.value.sampler.UniformSampler.weight" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/sampler.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-301">301</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-298">298</a></span>
+<span class="normal"><a href="#__codelineno-0-299">299</a></span>
+<span class="normal"><a href="#__codelineno-0-300">300</a></span>
+<span class="normal"><a href="#__codelineno-0-301">301</a></span>
 <span class="normal"><a href="#__codelineno-0-302">302</a></span>
-<span class="normal"><a href="#__codelineno-0-303">303</a></span>
-<span class="normal"><a href="#__codelineno-0-304">304</a></span>
-<span class="normal"><a href="#__codelineno-0-305">305</a></span>
-<span class="normal"><a href="#__codelineno-0-306">306</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-301"><a id="__codelineno-0-301" name="__codelineno-0-301"></a><span class="nd">@classmethod</span>
-</span><span id="__span-0-302"><a id="__codelineno-0-302" name="__codelineno-0-302"></a><span class="k">def</span> <span class="nf">weight</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">n</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">subset_len</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
-</span><span id="__span-0-303"><a id="__codelineno-0-303" name="__codelineno-0-303"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Correction coming from Monte Carlo integration so that the mean of</span>
-</span><span id="__span-0-304"><a id="__codelineno-0-304" name="__codelineno-0-304"></a><span class="sd">    the marginals converges to the value: the uniform distribution over the</span>
-</span><span id="__span-0-305"><a id="__codelineno-0-305" name="__codelineno-0-305"></a><span class="sd">    powerset of a set with n-1 elements has mass 2^{n-1} over each subset.&quot;&quot;&quot;</span>
-</span><span id="__span-0-306"><a id="__codelineno-0-306" name="__codelineno-0-306"></a>    <span class="k">return</span> <span class="nb">float</span><span class="p">(</span><span class="mi">2</span> <span class="o">**</span> <span class="p">(</span><span class="n">n</span> <span class="o">-</span> <span class="mi">1</span><span class="p">))</span> <span class="k">if</span> <span class="n">n</span> <span class="o">&gt;</span> <span class="mi">0</span> <span class="k">else</span> <span class="mf">1.0</span>
+<span class="normal"><a href="#__codelineno-0-303">303</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-298"><a id="__codelineno-0-298" name="__codelineno-0-298"></a><span class="nd">@classmethod</span>
+</span><span id="__span-0-299"><a id="__codelineno-0-299" name="__codelineno-0-299"></a><span class="k">def</span> <span class="nf">weight</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">n</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">subset_len</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
+</span><span id="__span-0-300"><a id="__codelineno-0-300" name="__codelineno-0-300"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Correction coming from Monte Carlo integration so that the mean of</span>
+</span><span id="__span-0-301"><a id="__codelineno-0-301" name="__codelineno-0-301"></a><span class="sd">    the marginals converges to the value: the uniform distribution over the</span>
+</span><span id="__span-0-302"><a id="__codelineno-0-302" name="__codelineno-0-302"></a><span class="sd">    powerset of a set with n-1 elements has mass 2^{n-1} over each subset.&quot;&quot;&quot;</span>
+</span><span id="__span-0-303"><a id="__codelineno-0-303" name="__codelineno-0-303"></a>    <span class="k">return</span> <span class="nb">float</span><span class="p">(</span><span class="mi">2</span> <span class="o">**</span> <span class="p">(</span><span class="n">n</span> <span class="o">-</span> <span class="mi">1</span><span class="p">))</span> <span class="k">if</span> <span class="n">n</span> <span class="o">&gt;</span> <span class="mi">0</span> <span class="k">else</span> <span class="mf">1.0</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3366,7 +3366,7 @@ <h2 id="pydvl.value.sampler.AntitheticSampler" class="doc doc-heading">
 
   <div class="doc doc-contents ">
           <p class="doc doc-class-bases">
-            Bases: <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.StochasticSamplerMixin" href="#pydvl.value.sampler.StochasticSamplerMixin">StochasticSamplerMixin</a></code>, <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PowersetSampler" href="#pydvl.value.sampler.PowersetSampler">PowersetSampler</a>[<span title="pydvl.value.sampler.T">T</span>]</code></p>
+            Bases: <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.StochasticSamplerMixin" href="#pydvl.value.sampler.StochasticSamplerMixin">StochasticSamplerMixin</a></code>, <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PowersetSampler" href="#pydvl.value.sampler.PowersetSampler">PowersetSampler</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>]</code></p>
 
   
       <p>An iterator to perform uniform random sampling of subsets, and their
@@ -3410,7 +3410,7 @@ <h2 id="pydvl.value.sampler.PermutationSampler" class="doc doc-heading">
 
   <div class="doc doc-contents ">
           <p class="doc doc-class-bases">
-            Bases: <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.StochasticSamplerMixin" href="#pydvl.value.sampler.StochasticSamplerMixin">StochasticSamplerMixin</a></code>, <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PowersetSampler" href="#pydvl.value.sampler.PowersetSampler">PowersetSampler</a>[<span title="pydvl.value.sampler.T">T</span>]</code></p>
+            Bases: <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.StochasticSamplerMixin" href="#pydvl.value.sampler.StochasticSamplerMixin">StochasticSamplerMixin</a></code>, <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PowersetSampler" href="#pydvl.value.sampler.PowersetSampler">PowersetSampler</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>]</code></p>
 
   
       <p>Sample permutations of indices and iterate through each returning
@@ -3459,13 +3459,13 @@ <h3 id="pydvl.value.sampler.PermutationSampler.__getitem__" class="doc doc-headi
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/sampler.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-368">368</a></span>
-<span class="normal"><a href="#__codelineno-0-369">369</a></span>
-<span class="normal"><a href="#__codelineno-0-370">370</a></span>
-<span class="normal"><a href="#__codelineno-0-371">371</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-368"><a id="__codelineno-0-368" name="__codelineno-0-368"></a><span class="k">def</span> <span class="fm">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">slice</span> <span class="o">|</span> <span class="nb">list</span><span class="p">[</span><span class="nb">int</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">PowersetSampler</span><span class="p">[</span><span class="n">T</span><span class="p">]:</span>
-</span><span id="__span-0-369"><a id="__codelineno-0-369" name="__codelineno-0-369"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Permutation samplers cannot be split across indices, so we return</span>
-</span><span id="__span-0-370"><a id="__codelineno-0-370" name="__codelineno-0-370"></a><span class="sd">    a copy of the full sampler.&quot;&quot;&quot;</span>
-</span><span id="__span-0-371"><a id="__codelineno-0-371" name="__codelineno-0-371"></a>    <span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__getitem__</span><span class="p">(</span><span class="nb">slice</span><span class="p">(</span><span class="kc">None</span><span class="p">))</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-365">365</a></span>
+<span class="normal"><a href="#__codelineno-0-366">366</a></span>
+<span class="normal"><a href="#__codelineno-0-367">367</a></span>
+<span class="normal"><a href="#__codelineno-0-368">368</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-365"><a id="__codelineno-0-365" name="__codelineno-0-365"></a><span class="k">def</span> <span class="fm">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">slice</span> <span class="o">|</span> <span class="nb">list</span><span class="p">[</span><span class="nb">int</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">PowersetSampler</span><span class="p">[</span><span class="n">IndexT</span><span class="p">]:</span>
+</span><span id="__span-0-366"><a id="__codelineno-0-366" name="__codelineno-0-366"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Permutation samplers cannot be split across indices, so we return</span>
+</span><span id="__span-0-367"><a id="__codelineno-0-367" name="__codelineno-0-367"></a><span class="sd">    a copy of the full sampler.&quot;&quot;&quot;</span>
+</span><span id="__span-0-368"><a id="__codelineno-0-368" name="__codelineno-0-368"></a>    <span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__getitem__</span><span class="p">(</span><span class="nb">slice</span><span class="p">(</span><span class="kc">None</span><span class="p">))</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3494,7 +3494,7 @@ <h2 id="pydvl.value.sampler.DeterministicPermutationSampler" class="doc doc-head
 
   <div class="doc doc-contents ">
           <p class="doc doc-class-bases">
-            Bases: <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PermutationSampler" href="#pydvl.value.sampler.PermutationSampler">PermutationSampler</a>[<span title="pydvl.value.sampler.T">T</span>]</code></p>
+            Bases: <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PermutationSampler" href="#pydvl.value.sampler.PermutationSampler">PermutationSampler</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>]</code></p>
 
   
       <p>Samples all n! permutations of the indices deterministically, and
@@ -3547,7 +3547,7 @@ <h2 id="pydvl.value.sampler.RandomHierarchicalSampler" class="doc doc-heading">
 
   <div class="doc doc-contents ">
           <p class="doc doc-class-bases">
-            Bases: <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.StochasticSamplerMixin" href="#pydvl.value.sampler.StochasticSamplerMixin">StochasticSamplerMixin</a></code>, <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PowersetSampler" href="#pydvl.value.sampler.PowersetSampler">PowersetSampler</a>[<span title="pydvl.value.sampler.T">T</span>]</code></p>
+            Bases: <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.StochasticSamplerMixin" href="#pydvl.value.sampler.StochasticSamplerMixin">StochasticSamplerMixin</a></code>, <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PowersetSampler" href="#pydvl.value.sampler.PowersetSampler">PowersetSampler</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>]</code></p>
 
   
       <p>For every index, sample a set size, then a set of that size.</p>
@@ -3591,11 +3591,11 @@ <h2 id="pydvl.value.sampler.RandomHierarchicalSampler" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/semivalues/index.html b/devel/api/pydvl/value/semivalues/index.html
index 485a3a105..b4f8f4434 100644
--- a/devel/api/pydvl/value/semivalues/index.html
+++ b/devel/api/pydvl/value/semivalues/index.html
@@ -1920,6 +1920,20 @@
     <nav class="md-nav" aria-label="semivalues">
       <ul class="md-nav__list">
         
+          <li class="md-nav__item">
+  <a href="#pydvl.value.semivalues--main-components" class="md-nav__link">
+    Main components
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#pydvl.value.semivalues--computing-semi-values" class="md-nav__link">
+    Computing semi-values
+  </a>
+  
+</li>
+        
           <li class="md-nav__item">
   <a href="#pydvl.value.semivalues--references" class="md-nav__link">
     References
@@ -2558,6 +2572,20 @@
     <nav class="md-nav" aria-label="semivalues">
       <ul class="md-nav__list">
         
+          <li class="md-nav__item">
+  <a href="#pydvl.value.semivalues--main-components" class="md-nav__link">
+    Main components
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#pydvl.value.semivalues--computing-semi-values" class="md-nav__link">
+    Computing semi-values
+  </a>
+  
+</li>
+        
           <li class="md-nav__item">
   <a href="#pydvl.value.semivalues--references" class="md-nav__link">
     References
@@ -2665,13 +2693,14 @@ <h1>Semivalues</h1>
 \sum_{S \subset D_{-i}^{(k)}} [U(S_{+i})-U(S)],\]</div>
 <p>where the coefficients <span class="arithmatex">\(w(k)\)</span> satisfy the property:</p>
 <div class="arithmatex">\[\sum_{k=1}^n w(k) = 1.\]</div>
-<div class="admonition note">
-<p class="admonition-title">Note</p>
+<details class="note">
+<summary>Note</summary>
 <p>For implementation consistency, we slightly depart from the common definition
 of semi-values, which includes a factor <span class="arithmatex">\(1/n\)</span> in the sum over subsets.
 Instead, we subsume this factor into the coefficient <span class="arithmatex">\(w(k)\)</span>.</p>
-</div>
-<p>As such, the computation of a semi-value requires two components:</p>
+</details>
+<h3 id="pydvl.value.semivalues--main-components">Main components<a class="headerlink" href="#pydvl.value.semivalues--main-components" title="Permanent link">&para;</a></h3>
+<p>The computation of a semi-value requires two components:</p>
 <ol>
 <li>A <strong>subset sampler</strong> that generates subsets of the set <span class="arithmatex">\(D\)</span> of interest.</li>
 <li>A <strong>coefficient</strong> <span class="arithmatex">\(w(k)\)</span> that assigns a weight to each subset size <span class="arithmatex">\(k\)</span>.</li>
@@ -2697,7 +2726,11 @@ <h1>Semivalues</h1>
 require caching to be enabled or computation will be doubled wrt. a 'direct'
 implementation of permutation MC.</p>
 </div>
-<p>There are several pre-defined coefficients, including the Shapley value of
+<h3 id="pydvl.value.semivalues--computing-semi-values">Computing semi-values<a class="headerlink" href="#pydvl.value.semivalues--computing-semi-values" title="Permanent link">&para;</a></h3>
+<p>Samplers and coefficients can be arbitrarily mixed by means of the main entry
+point of this module,
+<a class="autorefs autorefs-internal" href="#pydvl.value.semivalues.compute_generic_semivalues">compute_generic_semivalues</a>.
+There are several pre-defined coefficients, including the Shapley value of
 (Ghorbani and Zou, 2019)<sup id="pydvl.value.semivalues--fnref:1"><a class="footnote-ref" href="#pydvl.value.semivalues--fn:1">1</a></sup>, the Banzhaf index of (Wang and Jia)<sup id="pydvl.value.semivalues--fnref:3"><a class="footnote-ref" href="#pydvl.value.semivalues--fn:3">3</a></sup>, and the Beta
 coefficient of (Kwon and Zou, 2022)<sup id="pydvl.value.semivalues--fnref:2"><a class="footnote-ref" href="#pydvl.value.semivalues--fn:2">2</a></sup>. For each of these methods, there is a
 convenience wrapper function. Respectively, these are:
@@ -2705,23 +2738,34 @@ <h1>Semivalues</h1>
 <a class="autorefs autorefs-internal" href="#pydvl.value.semivalues.compute_banzhaf_semivalues">compute_banzhaf_semivalues</a>,
 and <a class="autorefs autorefs-internal" href="#pydvl.value.semivalues.compute_beta_shapley_semivalues">compute_beta_shapley_semivalues</a>.
 instead.</p>
+<div class="admonition tip">
+<p class="admonition-title">Parallelization and batching</p>
+<p>In order to ensure reproducibility and fine-grained control of
+parallelization, samples are generated in the main process and then
+distributed to worker processes for evaluation. For small sample sizes, this
+can lead to a significant overhead. To avoid this, we temporarily provide an
+additional argument <code>batch_size</code> to all methods which can improve
+performance with small models up to an order of magnitude. Note that this
+argument will be removed before version 1.0 in favour of a more general
+solution.</p>
+</div>
 <h3 id="pydvl.value.semivalues--references">References<a class="headerlink" href="#pydvl.value.semivalues--references" title="Permanent link">&para;</a></h3>
 <div class="footnote">
 <hr />
 <ol>
 <li id="pydvl.value.semivalues--fn:1">
 <p><a name="ghorbani_data_2019"></a>Ghorbani, A., Zou, J., 2019.
-<a href="http://proceedings.mlr.press/v97/ghorbani19c.html">Data Shapley: Equitable Valuation of Data for Machine Learning</a>.
+<a href="https://proceedings.mlr.press/v97/ghorbani19c.html">Data Shapley: Equitable Valuation of Data for Machine Learning</a>.
 In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242–2251.&#160;<a class="footnote-backref" href="#pydvl.value.semivalues--fnref:1" title="Jump back to footnote 1 in the text">&#8617;</a></p>
 </li>
 <li id="pydvl.value.semivalues--fn:2">
 <p><a name="kwon_beta_2022"></a>Kwon, Y. and Zou, J., 2022.
-<a href="http://arxiv.org/abs/2110.14049">Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning</a>.
+<a href="https://arxiv.org/abs/2110.14049">Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning</a>.
 In: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022, Vol. 151. PMLR, Valencia, Spain.&#160;<a class="footnote-backref" href="#pydvl.value.semivalues--fnref:2" title="Jump back to footnote 2 in the text">&#8617;</a></p>
 </li>
 <li id="pydvl.value.semivalues--fn:3">
 <p><a name="wang_data_2022"></a>Wang, J.T. and Jia, R., 2022.
-<a href="http://arxiv.org/abs/2205.15466">Data Banzhaf: A Robust Data Valuation Framework for Machine Learning</a>.
+<a href="https://arxiv.org/abs/2205.15466">Data Banzhaf: A Robust Data Valuation Framework for Machine Learning</a>.
 ArXiv preprint arXiv:2205.15466.&#160;<a class="footnote-backref" href="#pydvl.value.semivalues--fnref:3" title="Jump back to footnote 3 in the text">&#8617;</a></p>
 </li>
 </ol>
@@ -2755,7 +2799,8 @@ <h2 id="pydvl.value.semivalues.SVCoefficient" class="doc doc-heading">
             Bases: <code><a class="autorefs autorefs-external" title="typing.Protocol" href="https://docs.python.org/3/library/typing.html#typing.Protocol">Protocol</a></code></p>
 
   
-      <p>A coefficient for the computation of semi-values.</p>
+      <p>The protocol that coefficients for the computation of semi-values must
+fulfill.</p>
 
 
   
@@ -2829,21 +2874,21 @@ <h3 id="pydvl.value.semivalues.SVCoefficient.__call__" class="doc doc-heading">
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/semivalues.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-113">113</a></span>
-<span class="normal"><a href="#__codelineno-0-114">114</a></span>
-<span class="normal"><a href="#__codelineno-0-115">115</a></span>
-<span class="normal"><a href="#__codelineno-0-116">116</a></span>
-<span class="normal"><a href="#__codelineno-0-117">117</a></span>
-<span class="normal"><a href="#__codelineno-0-118">118</a></span>
-<span class="normal"><a href="#__codelineno-0-119">119</a></span>
-<span class="normal"><a href="#__codelineno-0-120">120</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-113"><a id="__codelineno-0-113" name="__codelineno-0-113"></a><span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">n</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">k</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
-</span><span id="__span-0-114"><a id="__codelineno-0-114" name="__codelineno-0-114"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Computes the coefficient for a given subset size.</span>
-</span><span id="__span-0-115"><a id="__codelineno-0-115" name="__codelineno-0-115"></a>
-</span><span id="__span-0-116"><a id="__codelineno-0-116" name="__codelineno-0-116"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-117"><a id="__codelineno-0-117" name="__codelineno-0-117"></a><span class="sd">        n: Total number of elements in the set.</span>
-</span><span id="__span-0-118"><a id="__codelineno-0-118" name="__codelineno-0-118"></a><span class="sd">        k: Size of the subset for which the coefficient is being computed</span>
-</span><span id="__span-0-119"><a id="__codelineno-0-119" name="__codelineno-0-119"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-120"><a id="__codelineno-0-120" name="__codelineno-0-120"></a>    <span class="o">...</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-132">132</a></span>
+<span class="normal"><a href="#__codelineno-0-133">133</a></span>
+<span class="normal"><a href="#__codelineno-0-134">134</a></span>
+<span class="normal"><a href="#__codelineno-0-135">135</a></span>
+<span class="normal"><a href="#__codelineno-0-136">136</a></span>
+<span class="normal"><a href="#__codelineno-0-137">137</a></span>
+<span class="normal"><a href="#__codelineno-0-138">138</a></span>
+<span class="normal"><a href="#__codelineno-0-139">139</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-132"><a id="__codelineno-0-132" name="__codelineno-0-132"></a><span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">n</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">k</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
+</span><span id="__span-0-133"><a id="__codelineno-0-133" name="__codelineno-0-133"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Computes the coefficient for a given subset size.</span>
+</span><span id="__span-0-134"><a id="__codelineno-0-134" name="__codelineno-0-134"></a>
+</span><span id="__span-0-135"><a id="__codelineno-0-135" name="__codelineno-0-135"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-136"><a id="__codelineno-0-136" name="__codelineno-0-136"></a><span class="sd">        n: Total number of elements in the set.</span>
+</span><span id="__span-0-137"><a id="__codelineno-0-137" name="__codelineno-0-137"></a><span class="sd">        k: Size of the subset for which the coefficient is being computed</span>
+</span><span id="__span-0-138"><a id="__codelineno-0-138" name="__codelineno-0-138"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-139"><a id="__codelineno-0-139" name="__codelineno-0-139"></a>    <span class="o">...</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -2911,7 +2956,7 @@ <h2 id="pydvl.value.semivalues.SemiValueMode" class="doc doc-heading">
 
 
 <h2 id="pydvl.value.semivalues.compute_generic_semivalues" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">compute_generic_semivalues</span><span class="p">(</span><span class="n">sampler</span><span class="p">,</span> <span class="n">u</span><span class="p">,</span> <span class="n">coefficient</span><span class="p">,</span> <span class="n">done</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">n_jobs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">ParallelConfig</span><span class="p">(),</span> <span class="n">progress</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></code>
+          <code class="highlight language-python"><span class="n">compute_generic_semivalues</span><span class="p">(</span><span class="n">sampler</span><span class="p">,</span> <span class="n">u</span><span class="p">,</span> <span class="n">coefficient</span><span class="p">,</span> <span class="n">done</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">n_jobs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">ParallelConfig</span><span class="p">(),</span> <span class="n">progress</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></code>
 
 <a href="#pydvl.value.semivalues.compute_generic_semivalues" class="headerlink" title="Permanent link">&para;</a></h2>
 
@@ -2939,7 +2984,7 @@ <h2 id="pydvl.value.semivalues.compute_generic_semivalues" class="doc doc-headin
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PowersetSampler" href="../sampler/#pydvl.value.sampler.PowersetSampler">PowersetSampler</a></code>
+                    <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PowersetSampler" href="../sampler/#pydvl.value.sampler.PowersetSampler">PowersetSampler</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>]</code>
                 </span>
             </p>
           </td>
@@ -2986,6 +3031,24 @@ <h2 id="pydvl.value.semivalues.compute_generic_semivalues" class="doc doc-headin
             </p>
           </td>
         </tr>
+        <tr>
+          <td><code>batch_size</code></td>
+          <td class="doc-param-details">
+            <div class="doc-md-description">
+              <p>Number of marginal evaluations per single parallel job.</p>
+            </div>
+            <p>
+                <span class="doc-param-annotation">
+                  <b>TYPE:</b>
+                    <code><a class="autorefs autorefs-external" href="https://docs.python.org/3/library/functions.html#int">int</a></code>
+                </span>
+                <span class="doc-param-default">
+                  <b>DEFAULT:</b> 
+                    <code>1</code>
+                </span>
+            </p>
+          </td>
+        </tr>
         <tr>
           <td><code>n_jobs</code></td>
           <td class="doc-param-details">
@@ -3068,33 +3131,15 @@ <h2 id="pydvl.value.semivalues.compute_generic_semivalues" class="doc doc-headin
         </tr>
     </tbody>
   </table>
+      <div class="admonition warning">
+<p class="admonition-title">Deprecation notice</p>
+<p>Parameter <code>batch_size</code> is for experimental use and will be removed in
+future versions.</p>
+</div>
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/semivalues.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-151">151</a></span>
-<span class="normal"><a href="#__codelineno-0-152">152</a></span>
-<span class="normal"><a href="#__codelineno-0-153">153</a></span>
-<span class="normal"><a href="#__codelineno-0-154">154</a></span>
-<span class="normal"><a href="#__codelineno-0-155">155</a></span>
-<span class="normal"><a href="#__codelineno-0-156">156</a></span>
-<span class="normal"><a href="#__codelineno-0-157">157</a></span>
-<span class="normal"><a href="#__codelineno-0-158">158</a></span>
-<span class="normal"><a href="#__codelineno-0-159">159</a></span>
-<span class="normal"><a href="#__codelineno-0-160">160</a></span>
-<span class="normal"><a href="#__codelineno-0-161">161</a></span>
-<span class="normal"><a href="#__codelineno-0-162">162</a></span>
-<span class="normal"><a href="#__codelineno-0-163">163</a></span>
-<span class="normal"><a href="#__codelineno-0-164">164</a></span>
-<span class="normal"><a href="#__codelineno-0-165">165</a></span>
-<span class="normal"><a href="#__codelineno-0-166">166</a></span>
-<span class="normal"><a href="#__codelineno-0-167">167</a></span>
-<span class="normal"><a href="#__codelineno-0-168">168</a></span>
-<span class="normal"><a href="#__codelineno-0-169">169</a></span>
-<span class="normal"><a href="#__codelineno-0-170">170</a></span>
-<span class="normal"><a href="#__codelineno-0-171">171</a></span>
-<span class="normal"><a href="#__codelineno-0-172">172</a></span>
-<span class="normal"><a href="#__codelineno-0-173">173</a></span>
-<span class="normal"><a href="#__codelineno-0-174">174</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-174">174</a></span>
 <span class="normal"><a href="#__codelineno-0-175">175</a></span>
 <span class="normal"><a href="#__codelineno-0-176">176</a></span>
 <span class="normal"><a href="#__codelineno-0-177">177</a></span>
@@ -3152,88 +3197,155 @@ <h2 id="pydvl.value.semivalues.compute_generic_semivalues" class="doc doc-headin
 <span class="normal"><a href="#__codelineno-0-229">229</a></span>
 <span class="normal"><a href="#__codelineno-0-230">230</a></span>
 <span class="normal"><a href="#__codelineno-0-231">231</a></span>
-<span class="normal"><a href="#__codelineno-0-232">232</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-151"><a id="__codelineno-0-151" name="__codelineno-0-151"></a><span class="k">def</span> <span class="nf">compute_generic_semivalues</span><span class="p">(</span>
-</span><span id="__span-0-152"><a id="__codelineno-0-152" name="__codelineno-0-152"></a>    <span class="n">sampler</span><span class="p">:</span> <span class="n">PowersetSampler</span><span class="p">,</span>
-</span><span id="__span-0-153"><a id="__codelineno-0-153" name="__codelineno-0-153"></a>    <span class="n">u</span><span class="p">:</span> <span class="n">Utility</span><span class="p">,</span>
-</span><span id="__span-0-154"><a id="__codelineno-0-154" name="__codelineno-0-154"></a>    <span class="n">coefficient</span><span class="p">:</span> <span class="n">SVCoefficient</span><span class="p">,</span>
-</span><span id="__span-0-155"><a id="__codelineno-0-155" name="__codelineno-0-155"></a>    <span class="n">done</span><span class="p">:</span> <span class="n">StoppingCriterion</span><span class="p">,</span>
-</span><span id="__span-0-156"><a id="__codelineno-0-156" name="__codelineno-0-156"></a>    <span class="o">*</span><span class="p">,</span>
-</span><span id="__span-0-157"><a id="__codelineno-0-157" name="__codelineno-0-157"></a>    <span class="n">n_jobs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-</span><span id="__span-0-158"><a id="__codelineno-0-158" name="__codelineno-0-158"></a>    <span class="n">config</span><span class="p">:</span> <span class="n">ParallelConfig</span> <span class="o">=</span> <span class="n">ParallelConfig</span><span class="p">(),</span>
-</span><span id="__span-0-159"><a id="__codelineno-0-159" name="__codelineno-0-159"></a>    <span class="n">progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-</span><span id="__span-0-160"><a id="__codelineno-0-160" name="__codelineno-0-160"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">:</span>
-</span><span id="__span-0-161"><a id="__codelineno-0-161" name="__codelineno-0-161"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Computes semi-values for a given utility function and subset sampler.</span>
-</span><span id="__span-0-162"><a id="__codelineno-0-162" name="__codelineno-0-162"></a>
-</span><span id="__span-0-163"><a id="__codelineno-0-163" name="__codelineno-0-163"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-164"><a id="__codelineno-0-164" name="__codelineno-0-164"></a><span class="sd">        sampler: The subset sampler to use for utility computations.</span>
-</span><span id="__span-0-165"><a id="__codelineno-0-165" name="__codelineno-0-165"></a><span class="sd">        u: Utility object with model, data, and scoring function.</span>
-</span><span id="__span-0-166"><a id="__codelineno-0-166" name="__codelineno-0-166"></a><span class="sd">        coefficient: The semi-value coefficient</span>
-</span><span id="__span-0-167"><a id="__codelineno-0-167" name="__codelineno-0-167"></a><span class="sd">        done: Stopping criterion.</span>
-</span><span id="__span-0-168"><a id="__codelineno-0-168" name="__codelineno-0-168"></a><span class="sd">        n_jobs: Number of parallel jobs to use.</span>
-</span><span id="__span-0-169"><a id="__codelineno-0-169" name="__codelineno-0-169"></a><span class="sd">        config: Object configuring parallel computation, with cluster</span>
-</span><span id="__span-0-170"><a id="__codelineno-0-170" name="__codelineno-0-170"></a><span class="sd">            address, number of cpus, etc.</span>
-</span><span id="__span-0-171"><a id="__codelineno-0-171" name="__codelineno-0-171"></a><span class="sd">        progress: Whether to display a progress bar.</span>
-</span><span id="__span-0-172"><a id="__codelineno-0-172" name="__codelineno-0-172"></a>
-</span><span id="__span-0-173"><a id="__codelineno-0-173" name="__codelineno-0-173"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-174"><a id="__codelineno-0-174" name="__codelineno-0-174"></a><span class="sd">        Object with the results.</span>
-</span><span id="__span-0-175"><a id="__codelineno-0-175" name="__codelineno-0-175"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-176"><a id="__codelineno-0-176" name="__codelineno-0-176"></a>    <span class="kn">from</span> <span class="nn">concurrent.futures</span> <span class="kn">import</span> <span class="n">FIRST_COMPLETED</span><span class="p">,</span> <span class="n">Future</span><span class="p">,</span> <span class="n">wait</span>
-</span><span id="__span-0-177"><a id="__codelineno-0-177" name="__codelineno-0-177"></a>
-</span><span id="__span-0-178"><a id="__codelineno-0-178" name="__codelineno-0-178"></a>    <span class="kn">from</span> <span class="nn">pydvl.utils</span> <span class="kn">import</span> <span class="n">effective_n_jobs</span><span class="p">,</span> <span class="n">init_executor</span><span class="p">,</span> <span class="n">init_parallel_backend</span>
-</span><span id="__span-0-179"><a id="__codelineno-0-179" name="__codelineno-0-179"></a>
-</span><span id="__span-0-180"><a id="__codelineno-0-180" name="__codelineno-0-180"></a>    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">sampler</span><span class="p">,</span> <span class="n">PermutationSampler</span><span class="p">)</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">u</span><span class="o">.</span><span class="n">enable_cache</span><span class="p">:</span>
-</span><span id="__span-0-181"><a id="__codelineno-0-181" name="__codelineno-0-181"></a>        <span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
-</span><span id="__span-0-182"><a id="__codelineno-0-182" name="__codelineno-0-182"></a>            <span class="s2">&quot;PermutationSampler requires caching to be enabled or computation &quot;</span>
-</span><span id="__span-0-183"><a id="__codelineno-0-183" name="__codelineno-0-183"></a>            <span class="s2">&quot;will be doubled wrt. a &#39;direct&#39; implementation of permutation MC&quot;</span>
-</span><span id="__span-0-184"><a id="__codelineno-0-184" name="__codelineno-0-184"></a>        <span class="p">)</span>
-</span><span id="__span-0-185"><a id="__codelineno-0-185" name="__codelineno-0-185"></a>
-</span><span id="__span-0-186"><a id="__codelineno-0-186" name="__codelineno-0-186"></a>    <span class="n">result</span> <span class="o">=</span> <span class="n">ValuationResult</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span>
-</span><span id="__span-0-187"><a id="__codelineno-0-187" name="__codelineno-0-187"></a>        <span class="n">algorithm</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;semivalue-</span><span class="si">{</span><span class="nb">str</span><span class="p">(</span><span class="n">sampler</span><span class="p">)</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">coefficient</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>  <span class="c1"># type: ignore</span>
-</span><span id="__span-0-188"><a id="__codelineno-0-188" name="__codelineno-0-188"></a>        <span class="n">indices</span><span class="o">=</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span>
-</span><span id="__span-0-189"><a id="__codelineno-0-189" name="__codelineno-0-189"></a>        <span class="n">data_names</span><span class="o">=</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">data_names</span><span class="p">,</span>
-</span><span id="__span-0-190"><a id="__codelineno-0-190" name="__codelineno-0-190"></a>    <span class="p">)</span>
-</span><span id="__span-0-191"><a id="__codelineno-0-191" name="__codelineno-0-191"></a>
-</span><span id="__span-0-192"><a id="__codelineno-0-192" name="__codelineno-0-192"></a>    <span class="n">parallel_backend</span> <span class="o">=</span> <span class="n">init_parallel_backend</span><span class="p">(</span><span class="n">config</span><span class="p">)</span>
-</span><span id="__span-0-193"><a id="__codelineno-0-193" name="__codelineno-0-193"></a>    <span class="n">u</span> <span class="o">=</span> <span class="n">parallel_backend</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">u</span><span class="p">)</span>
-</span><span id="__span-0-194"><a id="__codelineno-0-194" name="__codelineno-0-194"></a>    <span class="n">correction</span> <span class="o">=</span> <span class="n">parallel_backend</span><span class="o">.</span><span class="n">put</span><span class="p">(</span>
-</span><span id="__span-0-195"><a id="__codelineno-0-195" name="__codelineno-0-195"></a>        <span class="k">lambda</span> <span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">:</span> <span class="n">coefficient</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">)</span> <span class="o">*</span> <span class="n">sampler</span><span class="o">.</span><span class="n">weight</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">)</span>
-</span><span id="__span-0-196"><a id="__codelineno-0-196" name="__codelineno-0-196"></a>    <span class="p">)</span>
-</span><span id="__span-0-197"><a id="__codelineno-0-197" name="__codelineno-0-197"></a>
-</span><span id="__span-0-198"><a id="__codelineno-0-198" name="__codelineno-0-198"></a>    <span class="n">max_workers</span> <span class="o">=</span> <span class="n">effective_n_jobs</span><span class="p">(</span><span class="n">n_jobs</span><span class="p">,</span> <span class="n">config</span><span class="p">)</span>
-</span><span id="__span-0-199"><a id="__codelineno-0-199" name="__codelineno-0-199"></a>    <span class="n">n_submitted_jobs</span> <span class="o">=</span> <span class="mi">2</span> <span class="o">*</span> <span class="n">max_workers</span>  <span class="c1"># number of jobs in the queue</span>
-</span><span id="__span-0-200"><a id="__codelineno-0-200" name="__codelineno-0-200"></a>
-</span><span id="__span-0-201"><a id="__codelineno-0-201" name="__codelineno-0-201"></a>    <span class="n">sampler_it</span> <span class="o">=</span> <span class="nb">iter</span><span class="p">(</span><span class="n">sampler</span><span class="p">)</span>
-</span><span id="__span-0-202"><a id="__codelineno-0-202" name="__codelineno-0-202"></a>    <span class="n">pbar</span> <span class="o">=</span> <span class="n">tqdm</span><span class="p">(</span><span class="n">disable</span><span class="o">=</span><span class="ow">not</span> <span class="n">progress</span><span class="p">,</span> <span class="n">total</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">unit</span><span class="o">=</span><span class="s2">&quot;%&quot;</span><span class="p">)</span>
-</span><span id="__span-0-203"><a id="__codelineno-0-203" name="__codelineno-0-203"></a>
-</span><span id="__span-0-204"><a id="__codelineno-0-204" name="__codelineno-0-204"></a>    <span class="k">with</span> <span class="n">init_executor</span><span class="p">(</span>
-</span><span id="__span-0-205"><a id="__codelineno-0-205" name="__codelineno-0-205"></a>        <span class="n">max_workers</span><span class="o">=</span><span class="n">max_workers</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span> <span class="n">cancel_futures</span><span class="o">=</span><span class="kc">True</span>
-</span><span id="__span-0-206"><a id="__codelineno-0-206" name="__codelineno-0-206"></a>    <span class="p">)</span> <span class="k">as</span> <span class="n">executor</span><span class="p">:</span>
-</span><span id="__span-0-207"><a id="__codelineno-0-207" name="__codelineno-0-207"></a>        <span class="n">pending</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="n">Future</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
-</span><span id="__span-0-208"><a id="__codelineno-0-208" name="__codelineno-0-208"></a>        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
-</span><span id="__span-0-209"><a id="__codelineno-0-209" name="__codelineno-0-209"></a>            <span class="n">pbar</span><span class="o">.</span><span class="n">n</span> <span class="o">=</span> <span class="mi">100</span> <span class="o">*</span> <span class="n">done</span><span class="o">.</span><span class="n">completion</span><span class="p">()</span>
-</span><span id="__span-0-210"><a id="__codelineno-0-210" name="__codelineno-0-210"></a>            <span class="n">pbar</span><span class="o">.</span><span class="n">refresh</span><span class="p">()</span>
-</span><span id="__span-0-211"><a id="__codelineno-0-211" name="__codelineno-0-211"></a>
-</span><span id="__span-0-212"><a id="__codelineno-0-212" name="__codelineno-0-212"></a>            <span class="n">completed</span><span class="p">,</span> <span class="n">pending</span> <span class="o">=</span> <span class="n">wait</span><span class="p">(</span><span class="n">pending</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">return_when</span><span class="o">=</span><span class="n">FIRST_COMPLETED</span><span class="p">)</span>
-</span><span id="__span-0-213"><a id="__codelineno-0-213" name="__codelineno-0-213"></a>            <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">completed</span><span class="p">:</span>
-</span><span id="__span-0-214"><a id="__codelineno-0-214" name="__codelineno-0-214"></a>                <span class="n">idx</span><span class="p">,</span> <span class="n">marginal</span> <span class="o">=</span> <span class="n">future</span><span class="o">.</span><span class="n">result</span><span class="p">()</span>
-</span><span id="__span-0-215"><a id="__codelineno-0-215" name="__codelineno-0-215"></a>                <span class="n">result</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">idx</span><span class="p">,</span> <span class="n">marginal</span><span class="p">)</span>
-</span><span id="__span-0-216"><a id="__codelineno-0-216" name="__codelineno-0-216"></a>                <span class="k">if</span> <span class="n">done</span><span class="p">(</span><span class="n">result</span><span class="p">):</span>
-</span><span id="__span-0-217"><a id="__codelineno-0-217" name="__codelineno-0-217"></a>                    <span class="k">return</span> <span class="n">result</span>
-</span><span id="__span-0-218"><a id="__codelineno-0-218" name="__codelineno-0-218"></a>
-</span><span id="__span-0-219"><a id="__codelineno-0-219" name="__codelineno-0-219"></a>            <span class="c1"># Ensure that we always have n_submitted_jobs running</span>
-</span><span id="__span-0-220"><a id="__codelineno-0-220" name="__codelineno-0-220"></a>            <span class="k">try</span><span class="p">:</span>
-</span><span id="__span-0-221"><a id="__codelineno-0-221" name="__codelineno-0-221"></a>                <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_submitted_jobs</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">pending</span><span class="p">)):</span>
-</span><span id="__span-0-222"><a id="__codelineno-0-222" name="__codelineno-0-222"></a>                    <span class="n">pending</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
-</span><span id="__span-0-223"><a id="__codelineno-0-223" name="__codelineno-0-223"></a>                        <span class="n">executor</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span>
-</span><span id="__span-0-224"><a id="__codelineno-0-224" name="__codelineno-0-224"></a>                            <span class="n">_marginal</span><span class="p">,</span>
-</span><span id="__span-0-225"><a id="__codelineno-0-225" name="__codelineno-0-225"></a>                            <span class="n">u</span><span class="o">=</span><span class="n">u</span><span class="p">,</span>
-</span><span id="__span-0-226"><a id="__codelineno-0-226" name="__codelineno-0-226"></a>                            <span class="n">coefficient</span><span class="o">=</span><span class="n">correction</span><span class="p">,</span>
-</span><span id="__span-0-227"><a id="__codelineno-0-227" name="__codelineno-0-227"></a>                            <span class="n">sample</span><span class="o">=</span><span class="nb">next</span><span class="p">(</span><span class="n">sampler_it</span><span class="p">),</span>
-</span><span id="__span-0-228"><a id="__codelineno-0-228" name="__codelineno-0-228"></a>                        <span class="p">)</span>
-</span><span id="__span-0-229"><a id="__codelineno-0-229" name="__codelineno-0-229"></a>                    <span class="p">)</span>
-</span><span id="__span-0-230"><a id="__codelineno-0-230" name="__codelineno-0-230"></a>            <span class="k">except</span> <span class="ne">StopIteration</span><span class="p">:</span>
-</span><span id="__span-0-231"><a id="__codelineno-0-231" name="__codelineno-0-231"></a>                <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">pending</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
-</span><span id="__span-0-232"><a id="__codelineno-0-232" name="__codelineno-0-232"></a>                    <span class="k">return</span> <span class="n">result</span>
+<span class="normal"><a href="#__codelineno-0-232">232</a></span>
+<span class="normal"><a href="#__codelineno-0-233">233</a></span>
+<span class="normal"><a href="#__codelineno-0-234">234</a></span>
+<span class="normal"><a href="#__codelineno-0-235">235</a></span>
+<span class="normal"><a href="#__codelineno-0-236">236</a></span>
+<span class="normal"><a href="#__codelineno-0-237">237</a></span>
+<span class="normal"><a href="#__codelineno-0-238">238</a></span>
+<span class="normal"><a href="#__codelineno-0-239">239</a></span>
+<span class="normal"><a href="#__codelineno-0-240">240</a></span>
+<span class="normal"><a href="#__codelineno-0-241">241</a></span>
+<span class="normal"><a href="#__codelineno-0-242">242</a></span>
+<span class="normal"><a href="#__codelineno-0-243">243</a></span>
+<span class="normal"><a href="#__codelineno-0-244">244</a></span>
+<span class="normal"><a href="#__codelineno-0-245">245</a></span>
+<span class="normal"><a href="#__codelineno-0-246">246</a></span>
+<span class="normal"><a href="#__codelineno-0-247">247</a></span>
+<span class="normal"><a href="#__codelineno-0-248">248</a></span>
+<span class="normal"><a href="#__codelineno-0-249">249</a></span>
+<span class="normal"><a href="#__codelineno-0-250">250</a></span>
+<span class="normal"><a href="#__codelineno-0-251">251</a></span>
+<span class="normal"><a href="#__codelineno-0-252">252</a></span>
+<span class="normal"><a href="#__codelineno-0-253">253</a></span>
+<span class="normal"><a href="#__codelineno-0-254">254</a></span>
+<span class="normal"><a href="#__codelineno-0-255">255</a></span>
+<span class="normal"><a href="#__codelineno-0-256">256</a></span>
+<span class="normal"><a href="#__codelineno-0-257">257</a></span>
+<span class="normal"><a href="#__codelineno-0-258">258</a></span>
+<span class="normal"><a href="#__codelineno-0-259">259</a></span>
+<span class="normal"><a href="#__codelineno-0-260">260</a></span>
+<span class="normal"><a href="#__codelineno-0-261">261</a></span>
+<span class="normal"><a href="#__codelineno-0-262">262</a></span>
+<span class="normal"><a href="#__codelineno-0-263">263</a></span>
+<span class="normal"><a href="#__codelineno-0-264">264</a></span>
+<span class="normal"><a href="#__codelineno-0-265">265</a></span>
+<span class="normal"><a href="#__codelineno-0-266">266</a></span>
+<span class="normal"><a href="#__codelineno-0-267">267</a></span>
+<span class="normal"><a href="#__codelineno-0-268">268</a></span>
+<span class="normal"><a href="#__codelineno-0-269">269</a></span>
+<span class="normal"><a href="#__codelineno-0-270">270</a></span>
+<span class="normal"><a href="#__codelineno-0-271">271</a></span>
+<span class="normal"><a href="#__codelineno-0-272">272</a></span>
+<span class="normal"><a href="#__codelineno-0-273">273</a></span>
+<span class="normal"><a href="#__codelineno-0-274">274</a></span>
+<span class="normal"><a href="#__codelineno-0-275">275</a></span>
+<span class="normal"><a href="#__codelineno-0-276">276</a></span>
+<span class="normal"><a href="#__codelineno-0-277">277</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-174"><a id="__codelineno-0-174" name="__codelineno-0-174"></a><span class="nd">@deprecated</span><span class="p">(</span>
+</span><span id="__span-0-175"><a id="__codelineno-0-175" name="__codelineno-0-175"></a>    <span class="n">target</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+</span><span id="__span-0-176"><a id="__codelineno-0-176" name="__codelineno-0-176"></a>    <span class="n">deprecated_in</span><span class="o">=</span><span class="s2">&quot;0.7.0&quot;</span><span class="p">,</span>
+</span><span id="__span-0-177"><a id="__codelineno-0-177" name="__codelineno-0-177"></a>    <span class="n">remove_in</span><span class="o">=</span><span class="s2">&quot;0.9.0&quot;</span><span class="p">,</span>
+</span><span id="__span-0-178"><a id="__codelineno-0-178" name="__codelineno-0-178"></a>    <span class="n">args_mapping</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;batch_size&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">},</span>
+</span><span id="__span-0-179"><a id="__codelineno-0-179" name="__codelineno-0-179"></a>    <span class="n">template_mgs</span><span class="o">=</span><span class="s2">&quot;batch_size is for experimental use and will be removed&quot;</span>
+</span><span id="__span-0-180"><a id="__codelineno-0-180" name="__codelineno-0-180"></a>    <span class="s2">&quot;in future versions.&quot;</span><span class="p">,</span>
+</span><span id="__span-0-181"><a id="__codelineno-0-181" name="__codelineno-0-181"></a><span class="p">)</span>
+</span><span id="__span-0-182"><a id="__codelineno-0-182" name="__codelineno-0-182"></a><span class="k">def</span> <span class="nf">compute_generic_semivalues</span><span class="p">(</span>
+</span><span id="__span-0-183"><a id="__codelineno-0-183" name="__codelineno-0-183"></a>    <span class="n">sampler</span><span class="p">:</span> <span class="n">PowersetSampler</span><span class="p">[</span><span class="n">IndexT</span><span class="p">],</span>
+</span><span id="__span-0-184"><a id="__codelineno-0-184" name="__codelineno-0-184"></a>    <span class="n">u</span><span class="p">:</span> <span class="n">Utility</span><span class="p">,</span>
+</span><span id="__span-0-185"><a id="__codelineno-0-185" name="__codelineno-0-185"></a>    <span class="n">coefficient</span><span class="p">:</span> <span class="n">SVCoefficient</span><span class="p">,</span>
+</span><span id="__span-0-186"><a id="__codelineno-0-186" name="__codelineno-0-186"></a>    <span class="n">done</span><span class="p">:</span> <span class="n">StoppingCriterion</span><span class="p">,</span>
+</span><span id="__span-0-187"><a id="__codelineno-0-187" name="__codelineno-0-187"></a>    <span class="o">*</span><span class="p">,</span>
+</span><span id="__span-0-188"><a id="__codelineno-0-188" name="__codelineno-0-188"></a>    <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+</span><span id="__span-0-189"><a id="__codelineno-0-189" name="__codelineno-0-189"></a>    <span class="n">n_jobs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+</span><span id="__span-0-190"><a id="__codelineno-0-190" name="__codelineno-0-190"></a>    <span class="n">config</span><span class="p">:</span> <span class="n">ParallelConfig</span> <span class="o">=</span> <span class="n">ParallelConfig</span><span class="p">(),</span>
+</span><span id="__span-0-191"><a id="__codelineno-0-191" name="__codelineno-0-191"></a>    <span class="n">progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+</span><span id="__span-0-192"><a id="__codelineno-0-192" name="__codelineno-0-192"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">:</span>
+</span><span id="__span-0-193"><a id="__codelineno-0-193" name="__codelineno-0-193"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Computes semi-values for a given utility function and subset sampler.</span>
+</span><span id="__span-0-194"><a id="__codelineno-0-194" name="__codelineno-0-194"></a>
+</span><span id="__span-0-195"><a id="__codelineno-0-195" name="__codelineno-0-195"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-196"><a id="__codelineno-0-196" name="__codelineno-0-196"></a><span class="sd">        sampler: The subset sampler to use for utility computations.</span>
+</span><span id="__span-0-197"><a id="__codelineno-0-197" name="__codelineno-0-197"></a><span class="sd">        u: Utility object with model, data, and scoring function.</span>
+</span><span id="__span-0-198"><a id="__codelineno-0-198" name="__codelineno-0-198"></a><span class="sd">        coefficient: The semi-value coefficient</span>
+</span><span id="__span-0-199"><a id="__codelineno-0-199" name="__codelineno-0-199"></a><span class="sd">        done: Stopping criterion.</span>
+</span><span id="__span-0-200"><a id="__codelineno-0-200" name="__codelineno-0-200"></a><span class="sd">        batch_size: Number of marginal evaluations per single parallel job.</span>
+</span><span id="__span-0-201"><a id="__codelineno-0-201" name="__codelineno-0-201"></a><span class="sd">        n_jobs: Number of parallel jobs to use.</span>
+</span><span id="__span-0-202"><a id="__codelineno-0-202" name="__codelineno-0-202"></a><span class="sd">        config: Object configuring parallel computation, with cluster</span>
+</span><span id="__span-0-203"><a id="__codelineno-0-203" name="__codelineno-0-203"></a><span class="sd">            address, number of cpus, etc.</span>
+</span><span id="__span-0-204"><a id="__codelineno-0-204" name="__codelineno-0-204"></a><span class="sd">        progress: Whether to display a progress bar.</span>
+</span><span id="__span-0-205"><a id="__codelineno-0-205" name="__codelineno-0-205"></a>
+</span><span id="__span-0-206"><a id="__codelineno-0-206" name="__codelineno-0-206"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-207"><a id="__codelineno-0-207" name="__codelineno-0-207"></a><span class="sd">        Object with the results.</span>
+</span><span id="__span-0-208"><a id="__codelineno-0-208" name="__codelineno-0-208"></a>
+</span><span id="__span-0-209"><a id="__codelineno-0-209" name="__codelineno-0-209"></a><span class="sd">    !!! warning &quot;Deprecation notice&quot;</span>
+</span><span id="__span-0-210"><a id="__codelineno-0-210" name="__codelineno-0-210"></a><span class="sd">        Parameter `batch_size` is for experimental use and will be removed in</span>
+</span><span id="__span-0-211"><a id="__codelineno-0-211" name="__codelineno-0-211"></a><span class="sd">        future versions.</span>
+</span><span id="__span-0-212"><a id="__codelineno-0-212" name="__codelineno-0-212"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-213"><a id="__codelineno-0-213" name="__codelineno-0-213"></a>    <span class="kn">from</span> <span class="nn">concurrent.futures</span> <span class="kn">import</span> <span class="n">FIRST_COMPLETED</span><span class="p">,</span> <span class="n">Future</span><span class="p">,</span> <span class="n">wait</span>
+</span><span id="__span-0-214"><a id="__codelineno-0-214" name="__codelineno-0-214"></a>
+</span><span id="__span-0-215"><a id="__codelineno-0-215" name="__codelineno-0-215"></a>    <span class="kn">from</span> <span class="nn">pydvl.utils</span> <span class="kn">import</span> <span class="n">effective_n_jobs</span><span class="p">,</span> <span class="n">init_executor</span><span class="p">,</span> <span class="n">init_parallel_backend</span>
+</span><span id="__span-0-216"><a id="__codelineno-0-216" name="__codelineno-0-216"></a>
+</span><span id="__span-0-217"><a id="__codelineno-0-217" name="__codelineno-0-217"></a>    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">sampler</span><span class="p">,</span> <span class="n">PermutationSampler</span><span class="p">)</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">u</span><span class="o">.</span><span class="n">enable_cache</span><span class="p">:</span>
+</span><span id="__span-0-218"><a id="__codelineno-0-218" name="__codelineno-0-218"></a>        <span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
+</span><span id="__span-0-219"><a id="__codelineno-0-219" name="__codelineno-0-219"></a>            <span class="s2">&quot;PermutationSampler requires caching to be enabled or computation &quot;</span>
+</span><span id="__span-0-220"><a id="__codelineno-0-220" name="__codelineno-0-220"></a>            <span class="s2">&quot;will be doubled wrt. a &#39;direct&#39; implementation of permutation MC&quot;</span>
+</span><span id="__span-0-221"><a id="__codelineno-0-221" name="__codelineno-0-221"></a>        <span class="p">)</span>
+</span><span id="__span-0-222"><a id="__codelineno-0-222" name="__codelineno-0-222"></a>
+</span><span id="__span-0-223"><a id="__codelineno-0-223" name="__codelineno-0-223"></a>    <span class="k">if</span> <span class="n">batch_size</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">:</span>
+</span><span id="__span-0-224"><a id="__codelineno-0-224" name="__codelineno-0-224"></a>        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+</span><span id="__span-0-225"><a id="__codelineno-0-225" name="__codelineno-0-225"></a>            <span class="s2">&quot;Parameter `batch_size` is for experimental use and will be&quot;</span>
+</span><span id="__span-0-226"><a id="__codelineno-0-226" name="__codelineno-0-226"></a>            <span class="s2">&quot; removed in future versions&quot;</span><span class="p">,</span>
+</span><span id="__span-0-227"><a id="__codelineno-0-227" name="__codelineno-0-227"></a>            <span class="ne">DeprecationWarning</span><span class="p">,</span>
+</span><span id="__span-0-228"><a id="__codelineno-0-228" name="__codelineno-0-228"></a>        <span class="p">)</span>
+</span><span id="__span-0-229"><a id="__codelineno-0-229" name="__codelineno-0-229"></a>
+</span><span id="__span-0-230"><a id="__codelineno-0-230" name="__codelineno-0-230"></a>    <span class="n">result</span> <span class="o">=</span> <span class="n">ValuationResult</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span>
+</span><span id="__span-0-231"><a id="__codelineno-0-231" name="__codelineno-0-231"></a>        <span class="n">algorithm</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;semivalue-</span><span class="si">{</span><span class="nb">str</span><span class="p">(</span><span class="n">sampler</span><span class="p">)</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">coefficient</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>  <span class="c1"># type: ignore</span>
+</span><span id="__span-0-232"><a id="__codelineno-0-232" name="__codelineno-0-232"></a>        <span class="n">indices</span><span class="o">=</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span>
+</span><span id="__span-0-233"><a id="__codelineno-0-233" name="__codelineno-0-233"></a>        <span class="n">data_names</span><span class="o">=</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">data_names</span><span class="p">,</span>
+</span><span id="__span-0-234"><a id="__codelineno-0-234" name="__codelineno-0-234"></a>    <span class="p">)</span>
+</span><span id="__span-0-235"><a id="__codelineno-0-235" name="__codelineno-0-235"></a>
+</span><span id="__span-0-236"><a id="__codelineno-0-236" name="__codelineno-0-236"></a>    <span class="n">parallel_backend</span> <span class="o">=</span> <span class="n">init_parallel_backend</span><span class="p">(</span><span class="n">config</span><span class="p">)</span>
+</span><span id="__span-0-237"><a id="__codelineno-0-237" name="__codelineno-0-237"></a>    <span class="n">u</span> <span class="o">=</span> <span class="n">parallel_backend</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">u</span><span class="p">)</span>
+</span><span id="__span-0-238"><a id="__codelineno-0-238" name="__codelineno-0-238"></a>    <span class="n">correction</span> <span class="o">=</span> <span class="n">parallel_backend</span><span class="o">.</span><span class="n">put</span><span class="p">(</span>
+</span><span id="__span-0-239"><a id="__codelineno-0-239" name="__codelineno-0-239"></a>        <span class="k">lambda</span> <span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">:</span> <span class="n">coefficient</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">)</span> <span class="o">*</span> <span class="n">sampler</span><span class="o">.</span><span class="n">weight</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">k</span><span class="p">)</span>
+</span><span id="__span-0-240"><a id="__codelineno-0-240" name="__codelineno-0-240"></a>    <span class="p">)</span>
+</span><span id="__span-0-241"><a id="__codelineno-0-241" name="__codelineno-0-241"></a>
+</span><span id="__span-0-242"><a id="__codelineno-0-242" name="__codelineno-0-242"></a>    <span class="n">max_workers</span> <span class="o">=</span> <span class="n">effective_n_jobs</span><span class="p">(</span><span class="n">n_jobs</span><span class="p">,</span> <span class="n">config</span><span class="p">)</span>
+</span><span id="__span-0-243"><a id="__codelineno-0-243" name="__codelineno-0-243"></a>    <span class="n">n_submitted_jobs</span> <span class="o">=</span> <span class="mi">2</span> <span class="o">*</span> <span class="n">max_workers</span>  <span class="c1"># number of jobs in the queue</span>
+</span><span id="__span-0-244"><a id="__codelineno-0-244" name="__codelineno-0-244"></a>
+</span><span id="__span-0-245"><a id="__codelineno-0-245" name="__codelineno-0-245"></a>    <span class="n">sampler_it</span> <span class="o">=</span> <span class="nb">iter</span><span class="p">(</span><span class="n">sampler</span><span class="p">)</span>
+</span><span id="__span-0-246"><a id="__codelineno-0-246" name="__codelineno-0-246"></a>    <span class="n">pbar</span> <span class="o">=</span> <span class="n">tqdm</span><span class="p">(</span><span class="n">disable</span><span class="o">=</span><span class="ow">not</span> <span class="n">progress</span><span class="p">,</span> <span class="n">total</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">unit</span><span class="o">=</span><span class="s2">&quot;%&quot;</span><span class="p">)</span>
+</span><span id="__span-0-247"><a id="__codelineno-0-247" name="__codelineno-0-247"></a>
+</span><span id="__span-0-248"><a id="__codelineno-0-248" name="__codelineno-0-248"></a>    <span class="k">with</span> <span class="n">init_executor</span><span class="p">(</span>
+</span><span id="__span-0-249"><a id="__codelineno-0-249" name="__codelineno-0-249"></a>        <span class="n">max_workers</span><span class="o">=</span><span class="n">max_workers</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span> <span class="n">cancel_futures</span><span class="o">=</span><span class="kc">True</span>
+</span><span id="__span-0-250"><a id="__codelineno-0-250" name="__codelineno-0-250"></a>    <span class="p">)</span> <span class="k">as</span> <span class="n">executor</span><span class="p">:</span>
+</span><span id="__span-0-251"><a id="__codelineno-0-251" name="__codelineno-0-251"></a>        <span class="n">pending</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="n">Future</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
+</span><span id="__span-0-252"><a id="__codelineno-0-252" name="__codelineno-0-252"></a>        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+</span><span id="__span-0-253"><a id="__codelineno-0-253" name="__codelineno-0-253"></a>            <span class="n">pbar</span><span class="o">.</span><span class="n">n</span> <span class="o">=</span> <span class="mi">100</span> <span class="o">*</span> <span class="n">done</span><span class="o">.</span><span class="n">completion</span><span class="p">()</span>
+</span><span id="__span-0-254"><a id="__codelineno-0-254" name="__codelineno-0-254"></a>            <span class="n">pbar</span><span class="o">.</span><span class="n">refresh</span><span class="p">()</span>
+</span><span id="__span-0-255"><a id="__codelineno-0-255" name="__codelineno-0-255"></a>
+</span><span id="__span-0-256"><a id="__codelineno-0-256" name="__codelineno-0-256"></a>            <span class="n">completed</span><span class="p">,</span> <span class="n">pending</span> <span class="o">=</span> <span class="n">wait</span><span class="p">(</span><span class="n">pending</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">return_when</span><span class="o">=</span><span class="n">FIRST_COMPLETED</span><span class="p">)</span>
+</span><span id="__span-0-257"><a id="__codelineno-0-257" name="__codelineno-0-257"></a>            <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">completed</span><span class="p">:</span>
+</span><span id="__span-0-258"><a id="__codelineno-0-258" name="__codelineno-0-258"></a>                <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">marginal</span> <span class="ow">in</span> <span class="n">future</span><span class="o">.</span><span class="n">result</span><span class="p">():</span>
+</span><span id="__span-0-259"><a id="__codelineno-0-259" name="__codelineno-0-259"></a>                    <span class="n">result</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">idx</span><span class="p">,</span> <span class="n">marginal</span><span class="p">)</span>
+</span><span id="__span-0-260"><a id="__codelineno-0-260" name="__codelineno-0-260"></a>                    <span class="k">if</span> <span class="n">done</span><span class="p">(</span><span class="n">result</span><span class="p">):</span>
+</span><span id="__span-0-261"><a id="__codelineno-0-261" name="__codelineno-0-261"></a>                        <span class="k">return</span> <span class="n">result</span>
+</span><span id="__span-0-262"><a id="__codelineno-0-262" name="__codelineno-0-262"></a>
+</span><span id="__span-0-263"><a id="__codelineno-0-263" name="__codelineno-0-263"></a>            <span class="c1"># Ensure that we always have n_submitted_jobs running</span>
+</span><span id="__span-0-264"><a id="__codelineno-0-264" name="__codelineno-0-264"></a>            <span class="k">try</span><span class="p">:</span>
+</span><span id="__span-0-265"><a id="__codelineno-0-265" name="__codelineno-0-265"></a>                <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_submitted_jobs</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">pending</span><span class="p">)):</span>
+</span><span id="__span-0-266"><a id="__codelineno-0-266" name="__codelineno-0-266"></a>                    <span class="n">samples</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">islice</span><span class="p">(</span><span class="n">sampler_it</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">))</span>
+</span><span id="__span-0-267"><a id="__codelineno-0-267" name="__codelineno-0-267"></a>                    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">samples</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+</span><span id="__span-0-268"><a id="__codelineno-0-268" name="__codelineno-0-268"></a>                        <span class="k">raise</span> <span class="ne">StopIteration</span>
+</span><span id="__span-0-269"><a id="__codelineno-0-269" name="__codelineno-0-269"></a>
+</span><span id="__span-0-270"><a id="__codelineno-0-270" name="__codelineno-0-270"></a>                    <span class="n">pending</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
+</span><span id="__span-0-271"><a id="__codelineno-0-271" name="__codelineno-0-271"></a>                        <span class="n">executor</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span>
+</span><span id="__span-0-272"><a id="__codelineno-0-272" name="__codelineno-0-272"></a>                            <span class="n">_marginal</span><span class="p">,</span> <span class="n">u</span><span class="o">=</span><span class="n">u</span><span class="p">,</span> <span class="n">coefficient</span><span class="o">=</span><span class="n">correction</span><span class="p">,</span> <span class="n">samples</span><span class="o">=</span><span class="n">samples</span>
+</span><span id="__span-0-273"><a id="__codelineno-0-273" name="__codelineno-0-273"></a>                        <span class="p">)</span>
+</span><span id="__span-0-274"><a id="__codelineno-0-274" name="__codelineno-0-274"></a>                    <span class="p">)</span>
+</span><span id="__span-0-275"><a id="__codelineno-0-275" name="__codelineno-0-275"></a>            <span class="k">except</span> <span class="ne">StopIteration</span><span class="p">:</span>
+</span><span id="__span-0-276"><a id="__codelineno-0-276" name="__codelineno-0-276"></a>                <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">pending</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+</span><span id="__span-0-277"><a id="__codelineno-0-277" name="__codelineno-0-277"></a>                    <span class="k">return</span> <span class="n">result</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3247,7 +3359,7 @@ <h2 id="pydvl.value.semivalues.compute_generic_semivalues" class="doc doc-headin
 
 
 <h2 id="pydvl.value.semivalues.compute_shapley_semivalues" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">compute_shapley_semivalues</span><span class="p">(</span><span class="n">u</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">done</span><span class="o">=</span><span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span> <span class="n">sampler_t</span><span class="o">=</span><span class="n">PermutationSampler</span><span class="p">,</span> <span class="n">n_jobs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">ParallelConfig</span><span class="p">(),</span> <span class="n">progress</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span></code>
+          <code class="highlight language-python"><span class="n">compute_shapley_semivalues</span><span class="p">(</span><span class="n">u</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">done</span><span class="o">=</span><span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span> <span class="n">sampler_t</span><span class="o">=</span><span class="n">PermutationSampler</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">n_jobs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">ParallelConfig</span><span class="p">(),</span> <span class="n">progress</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span></code>
 
 <a href="#pydvl.value.semivalues.compute_shapley_semivalues" class="headerlink" title="Permanent link">&para;</a></h2>
 
@@ -3307,8 +3419,8 @@ <h2 id="pydvl.value.semivalues.compute_shapley_semivalues" class="doc doc-headin
           <td><code>sampler_t</code></td>
           <td class="doc-param-details">
             <div class="doc-md-description">
-              <p>The sampler type to use. See :mod:<code>pydvl.value.sampler</code>
-for a list.</p>
+              <p>The sampler type to use. See the
+<a class="autorefs autorefs-internal" href="../sampler/#pydvl.value.sampler">sampler</a> module for a list.</p>
             </div>
             <p>
                 <span class="doc-param-annotation">
@@ -3322,6 +3434,24 @@ <h2 id="pydvl.value.semivalues.compute_shapley_semivalues" class="doc doc-headin
             </p>
           </td>
         </tr>
+        <tr>
+          <td><code>batch_size</code></td>
+          <td class="doc-param-details">
+            <div class="doc-md-description">
+              <p>Number of marginal evaluations per single parallel job.</p>
+            </div>
+            <p>
+                <span class="doc-param-annotation">
+                  <b>TYPE:</b>
+                    <code><a class="autorefs autorefs-external" href="https://docs.python.org/3/library/functions.html#int">int</a></code>
+                </span>
+                <span class="doc-param-default">
+                  <b>DEFAULT:</b> 
+                    <code>1</code>
+                </span>
+            </p>
+          </td>
+        </tr>
         <tr>
           <td><code>n_jobs</code></td>
           <td class="doc-param-details">
@@ -3363,7 +3493,8 @@ <h2 id="pydvl.value.semivalues.compute_shapley_semivalues" class="doc doc-headin
           <td><code>seed</code></td>
           <td class="doc-param-details">
             <div class="doc-md-description">
-              <p>Either an instance of a numpy random number generator or a seed for it.</p>
+              <p>Either an instance of a numpy random number generator or a seed
+for it.</p>
             </div>
             <p>
                 <span class="doc-param-annotation">
@@ -3422,90 +3553,129 @@ <h2 id="pydvl.value.semivalues.compute_shapley_semivalues" class="doc doc-headin
         </tr>
     </tbody>
   </table>
+      <div class="admonition warning">
+<p class="admonition-title">Deprecation notice</p>
+<p>Parameter <code>batch_size</code> is for experimental use and will be removed in
+future versions.</p>
+</div>
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/semivalues.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-265">265</a></span>
-<span class="normal"><a href="#__codelineno-0-266">266</a></span>
-<span class="normal"><a href="#__codelineno-0-267">267</a></span>
-<span class="normal"><a href="#__codelineno-0-268">268</a></span>
-<span class="normal"><a href="#__codelineno-0-269">269</a></span>
-<span class="normal"><a href="#__codelineno-0-270">270</a></span>
-<span class="normal"><a href="#__codelineno-0-271">271</a></span>
-<span class="normal"><a href="#__codelineno-0-272">272</a></span>
-<span class="normal"><a href="#__codelineno-0-273">273</a></span>
-<span class="normal"><a href="#__codelineno-0-274">274</a></span>
-<span class="normal"><a href="#__codelineno-0-275">275</a></span>
-<span class="normal"><a href="#__codelineno-0-276">276</a></span>
-<span class="normal"><a href="#__codelineno-0-277">277</a></span>
-<span class="normal"><a href="#__codelineno-0-278">278</a></span>
-<span class="normal"><a href="#__codelineno-0-279">279</a></span>
-<span class="normal"><a href="#__codelineno-0-280">280</a></span>
-<span class="normal"><a href="#__codelineno-0-281">281</a></span>
-<span class="normal"><a href="#__codelineno-0-282">282</a></span>
-<span class="normal"><a href="#__codelineno-0-283">283</a></span>
-<span class="normal"><a href="#__codelineno-0-284">284</a></span>
-<span class="normal"><a href="#__codelineno-0-285">285</a></span>
-<span class="normal"><a href="#__codelineno-0-286">286</a></span>
-<span class="normal"><a href="#__codelineno-0-287">287</a></span>
-<span class="normal"><a href="#__codelineno-0-288">288</a></span>
-<span class="normal"><a href="#__codelineno-0-289">289</a></span>
-<span class="normal"><a href="#__codelineno-0-290">290</a></span>
-<span class="normal"><a href="#__codelineno-0-291">291</a></span>
-<span class="normal"><a href="#__codelineno-0-292">292</a></span>
-<span class="normal"><a href="#__codelineno-0-293">293</a></span>
-<span class="normal"><a href="#__codelineno-0-294">294</a></span>
-<span class="normal"><a href="#__codelineno-0-295">295</a></span>
-<span class="normal"><a href="#__codelineno-0-296">296</a></span>
-<span class="normal"><a href="#__codelineno-0-297">297</a></span>
-<span class="normal"><a href="#__codelineno-0-298">298</a></span>
-<span class="normal"><a href="#__codelineno-0-299">299</a></span>
-<span class="normal"><a href="#__codelineno-0-300">300</a></span>
-<span class="normal"><a href="#__codelineno-0-301">301</a></span>
-<span class="normal"><a href="#__codelineno-0-302">302</a></span>
-<span class="normal"><a href="#__codelineno-0-303">303</a></span>
-<span class="normal"><a href="#__codelineno-0-304">304</a></span>
-<span class="normal"><a href="#__codelineno-0-305">305</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-265"><a id="__codelineno-0-265" name="__codelineno-0-265"></a><span class="k">def</span> <span class="nf">compute_shapley_semivalues</span><span class="p">(</span>
-</span><span id="__span-0-266"><a id="__codelineno-0-266" name="__codelineno-0-266"></a>    <span class="n">u</span><span class="p">:</span> <span class="n">Utility</span><span class="p">,</span>
-</span><span id="__span-0-267"><a id="__codelineno-0-267" name="__codelineno-0-267"></a>    <span class="o">*</span><span class="p">,</span>
-</span><span id="__span-0-268"><a id="__codelineno-0-268" name="__codelineno-0-268"></a>    <span class="n">done</span><span class="p">:</span> <span class="n">StoppingCriterion</span> <span class="o">=</span> <span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span>
-</span><span id="__span-0-269"><a id="__codelineno-0-269" name="__codelineno-0-269"></a>    <span class="n">sampler_t</span><span class="p">:</span> <span class="n">Type</span><span class="p">[</span><span class="n">StochasticSampler</span><span class="p">]</span> <span class="o">=</span> <span class="n">PermutationSampler</span><span class="p">,</span>
-</span><span id="__span-0-270"><a id="__codelineno-0-270" name="__codelineno-0-270"></a>    <span class="n">n_jobs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-</span><span id="__span-0-271"><a id="__codelineno-0-271" name="__codelineno-0-271"></a>    <span class="n">config</span><span class="p">:</span> <span class="n">ParallelConfig</span> <span class="o">=</span> <span class="n">ParallelConfig</span><span class="p">(),</span>
-</span><span id="__span-0-272"><a id="__codelineno-0-272" name="__codelineno-0-272"></a>    <span class="n">progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-</span><span id="__span-0-273"><a id="__codelineno-0-273" name="__codelineno-0-273"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-274"><a id="__codelineno-0-274" name="__codelineno-0-274"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">:</span>
-</span><span id="__span-0-275"><a id="__codelineno-0-275" name="__codelineno-0-275"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Computes Shapley values for a given utility function.</span>
-</span><span id="__span-0-276"><a id="__codelineno-0-276" name="__codelineno-0-276"></a>
-</span><span id="__span-0-277"><a id="__codelineno-0-277" name="__codelineno-0-277"></a><span class="sd">    This is a convenience wrapper for</span>
-</span><span id="__span-0-278"><a id="__codelineno-0-278" name="__codelineno-0-278"></a><span class="sd">    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]</span>
-</span><span id="__span-0-279"><a id="__codelineno-0-279" name="__codelineno-0-279"></a><span class="sd">    with the Shapley coefficient. Use</span>
-</span><span id="__span-0-280"><a id="__codelineno-0-280" name="__codelineno-0-280"></a><span class="sd">    [compute_shapley_values][pydvl.value.shapley.common.compute_shapley_values]</span>
-</span><span id="__span-0-281"><a id="__codelineno-0-281" name="__codelineno-0-281"></a><span class="sd">    for a more flexible interface and additional methods, including TMCS.</span>
-</span><span id="__span-0-282"><a id="__codelineno-0-282" name="__codelineno-0-282"></a>
-</span><span id="__span-0-283"><a id="__codelineno-0-283" name="__codelineno-0-283"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-284"><a id="__codelineno-0-284" name="__codelineno-0-284"></a><span class="sd">        u: Utility object with model, data, and scoring function.</span>
-</span><span id="__span-0-285"><a id="__codelineno-0-285" name="__codelineno-0-285"></a><span class="sd">        done: Stopping criterion.</span>
-</span><span id="__span-0-286"><a id="__codelineno-0-286" name="__codelineno-0-286"></a><span class="sd">        sampler_t: The sampler type to use. See :mod:`pydvl.value.sampler`</span>
-</span><span id="__span-0-287"><a id="__codelineno-0-287" name="__codelineno-0-287"></a><span class="sd">            for a list.</span>
-</span><span id="__span-0-288"><a id="__codelineno-0-288" name="__codelineno-0-288"></a><span class="sd">        n_jobs: Number of parallel jobs to use.</span>
-</span><span id="__span-0-289"><a id="__codelineno-0-289" name="__codelineno-0-289"></a><span class="sd">        config: Object configuring parallel computation, with cluster</span>
-</span><span id="__span-0-290"><a id="__codelineno-0-290" name="__codelineno-0-290"></a><span class="sd">            address, number of cpus, etc.</span>
-</span><span id="__span-0-291"><a id="__codelineno-0-291" name="__codelineno-0-291"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
-</span><span id="__span-0-292"><a id="__codelineno-0-292" name="__codelineno-0-292"></a><span class="sd">        progress: Whether to display a progress bar.</span>
-</span><span id="__span-0-293"><a id="__codelineno-0-293" name="__codelineno-0-293"></a>
-</span><span id="__span-0-294"><a id="__codelineno-0-294" name="__codelineno-0-294"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-295"><a id="__codelineno-0-295" name="__codelineno-0-295"></a><span class="sd">        Object with the results.</span>
-</span><span id="__span-0-296"><a id="__codelineno-0-296" name="__codelineno-0-296"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-297"><a id="__codelineno-0-297" name="__codelineno-0-297"></a>    <span class="k">return</span> <span class="n">compute_generic_semivalues</span><span class="p">(</span>
-</span><span id="__span-0-298"><a id="__codelineno-0-298" name="__codelineno-0-298"></a>        <span class="n">sampler_t</span><span class="p">(</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">),</span>
-</span><span id="__span-0-299"><a id="__codelineno-0-299" name="__codelineno-0-299"></a>        <span class="n">u</span><span class="p">,</span>
-</span><span id="__span-0-300"><a id="__codelineno-0-300" name="__codelineno-0-300"></a>        <span class="n">shapley_coefficient</span><span class="p">,</span>
-</span><span id="__span-0-301"><a id="__codelineno-0-301" name="__codelineno-0-301"></a>        <span class="n">done</span><span class="p">,</span>
-</span><span id="__span-0-302"><a id="__codelineno-0-302" name="__codelineno-0-302"></a>        <span class="n">n_jobs</span><span class="o">=</span><span class="n">n_jobs</span><span class="p">,</span>
-</span><span id="__span-0-303"><a id="__codelineno-0-303" name="__codelineno-0-303"></a>        <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span>
-</span><span id="__span-0-304"><a id="__codelineno-0-304" name="__codelineno-0-304"></a>        <span class="n">progress</span><span class="o">=</span><span class="n">progress</span><span class="p">,</span>
-</span><span id="__span-0-305"><a id="__codelineno-0-305" name="__codelineno-0-305"></a>    <span class="p">)</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-310">310</a></span>
+<span class="normal"><a href="#__codelineno-0-311">311</a></span>
+<span class="normal"><a href="#__codelineno-0-312">312</a></span>
+<span class="normal"><a href="#__codelineno-0-313">313</a></span>
+<span class="normal"><a href="#__codelineno-0-314">314</a></span>
+<span class="normal"><a href="#__codelineno-0-315">315</a></span>
+<span class="normal"><a href="#__codelineno-0-316">316</a></span>
+<span class="normal"><a href="#__codelineno-0-317">317</a></span>
+<span class="normal"><a href="#__codelineno-0-318">318</a></span>
+<span class="normal"><a href="#__codelineno-0-319">319</a></span>
+<span class="normal"><a href="#__codelineno-0-320">320</a></span>
+<span class="normal"><a href="#__codelineno-0-321">321</a></span>
+<span class="normal"><a href="#__codelineno-0-322">322</a></span>
+<span class="normal"><a href="#__codelineno-0-323">323</a></span>
+<span class="normal"><a href="#__codelineno-0-324">324</a></span>
+<span class="normal"><a href="#__codelineno-0-325">325</a></span>
+<span class="normal"><a href="#__codelineno-0-326">326</a></span>
+<span class="normal"><a href="#__codelineno-0-327">327</a></span>
+<span class="normal"><a href="#__codelineno-0-328">328</a></span>
+<span class="normal"><a href="#__codelineno-0-329">329</a></span>
+<span class="normal"><a href="#__codelineno-0-330">330</a></span>
+<span class="normal"><a href="#__codelineno-0-331">331</a></span>
+<span class="normal"><a href="#__codelineno-0-332">332</a></span>
+<span class="normal"><a href="#__codelineno-0-333">333</a></span>
+<span class="normal"><a href="#__codelineno-0-334">334</a></span>
+<span class="normal"><a href="#__codelineno-0-335">335</a></span>
+<span class="normal"><a href="#__codelineno-0-336">336</a></span>
+<span class="normal"><a href="#__codelineno-0-337">337</a></span>
+<span class="normal"><a href="#__codelineno-0-338">338</a></span>
+<span class="normal"><a href="#__codelineno-0-339">339</a></span>
+<span class="normal"><a href="#__codelineno-0-340">340</a></span>
+<span class="normal"><a href="#__codelineno-0-341">341</a></span>
+<span class="normal"><a href="#__codelineno-0-342">342</a></span>
+<span class="normal"><a href="#__codelineno-0-343">343</a></span>
+<span class="normal"><a href="#__codelineno-0-344">344</a></span>
+<span class="normal"><a href="#__codelineno-0-345">345</a></span>
+<span class="normal"><a href="#__codelineno-0-346">346</a></span>
+<span class="normal"><a href="#__codelineno-0-347">347</a></span>
+<span class="normal"><a href="#__codelineno-0-348">348</a></span>
+<span class="normal"><a href="#__codelineno-0-349">349</a></span>
+<span class="normal"><a href="#__codelineno-0-350">350</a></span>
+<span class="normal"><a href="#__codelineno-0-351">351</a></span>
+<span class="normal"><a href="#__codelineno-0-352">352</a></span>
+<span class="normal"><a href="#__codelineno-0-353">353</a></span>
+<span class="normal"><a href="#__codelineno-0-354">354</a></span>
+<span class="normal"><a href="#__codelineno-0-355">355</a></span>
+<span class="normal"><a href="#__codelineno-0-356">356</a></span>
+<span class="normal"><a href="#__codelineno-0-357">357</a></span>
+<span class="normal"><a href="#__codelineno-0-358">358</a></span>
+<span class="normal"><a href="#__codelineno-0-359">359</a></span>
+<span class="normal"><a href="#__codelineno-0-360">360</a></span>
+<span class="normal"><a href="#__codelineno-0-361">361</a></span>
+<span class="normal"><a href="#__codelineno-0-362">362</a></span>
+<span class="normal"><a href="#__codelineno-0-363">363</a></span>
+<span class="normal"><a href="#__codelineno-0-364">364</a></span>
+<span class="normal"><a href="#__codelineno-0-365">365</a></span>
+<span class="normal"><a href="#__codelineno-0-366">366</a></span>
+<span class="normal"><a href="#__codelineno-0-367">367</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-310"><a id="__codelineno-0-310" name="__codelineno-0-310"></a><span class="nd">@deprecated</span><span class="p">(</span>
+</span><span id="__span-0-311"><a id="__codelineno-0-311" name="__codelineno-0-311"></a>    <span class="n">target</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+</span><span id="__span-0-312"><a id="__codelineno-0-312" name="__codelineno-0-312"></a>    <span class="n">deprecated_in</span><span class="o">=</span><span class="s2">&quot;0.7.0&quot;</span><span class="p">,</span>
+</span><span id="__span-0-313"><a id="__codelineno-0-313" name="__codelineno-0-313"></a>    <span class="n">remove_in</span><span class="o">=</span><span class="s2">&quot;0.9.0&quot;</span><span class="p">,</span>
+</span><span id="__span-0-314"><a id="__codelineno-0-314" name="__codelineno-0-314"></a>    <span class="n">args_mapping</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;batch_size&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">},</span>
+</span><span id="__span-0-315"><a id="__codelineno-0-315" name="__codelineno-0-315"></a>    <span class="n">template_mgs</span><span class="o">=</span><span class="s2">&quot;batch_size is for experimental use and will be removed&quot;</span>
+</span><span id="__span-0-316"><a id="__codelineno-0-316" name="__codelineno-0-316"></a>    <span class="s2">&quot;in future versions.&quot;</span><span class="p">,</span>
+</span><span id="__span-0-317"><a id="__codelineno-0-317" name="__codelineno-0-317"></a><span class="p">)</span>
+</span><span id="__span-0-318"><a id="__codelineno-0-318" name="__codelineno-0-318"></a><span class="k">def</span> <span class="nf">compute_shapley_semivalues</span><span class="p">(</span>
+</span><span id="__span-0-319"><a id="__codelineno-0-319" name="__codelineno-0-319"></a>    <span class="n">u</span><span class="p">:</span> <span class="n">Utility</span><span class="p">,</span>
+</span><span id="__span-0-320"><a id="__codelineno-0-320" name="__codelineno-0-320"></a>    <span class="o">*</span><span class="p">,</span>
+</span><span id="__span-0-321"><a id="__codelineno-0-321" name="__codelineno-0-321"></a>    <span class="n">done</span><span class="p">:</span> <span class="n">StoppingCriterion</span> <span class="o">=</span> <span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span>
+</span><span id="__span-0-322"><a id="__codelineno-0-322" name="__codelineno-0-322"></a>    <span class="n">sampler_t</span><span class="p">:</span> <span class="n">Type</span><span class="p">[</span><span class="n">StochasticSampler</span><span class="p">]</span> <span class="o">=</span> <span class="n">PermutationSampler</span><span class="p">,</span>
+</span><span id="__span-0-323"><a id="__codelineno-0-323" name="__codelineno-0-323"></a>    <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+</span><span id="__span-0-324"><a id="__codelineno-0-324" name="__codelineno-0-324"></a>    <span class="n">n_jobs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+</span><span id="__span-0-325"><a id="__codelineno-0-325" name="__codelineno-0-325"></a>    <span class="n">config</span><span class="p">:</span> <span class="n">ParallelConfig</span> <span class="o">=</span> <span class="n">ParallelConfig</span><span class="p">(),</span>
+</span><span id="__span-0-326"><a id="__codelineno-0-326" name="__codelineno-0-326"></a>    <span class="n">progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+</span><span id="__span-0-327"><a id="__codelineno-0-327" name="__codelineno-0-327"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-328"><a id="__codelineno-0-328" name="__codelineno-0-328"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">:</span>
+</span><span id="__span-0-329"><a id="__codelineno-0-329" name="__codelineno-0-329"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Computes Shapley values for a given utility function.</span>
+</span><span id="__span-0-330"><a id="__codelineno-0-330" name="__codelineno-0-330"></a>
+</span><span id="__span-0-331"><a id="__codelineno-0-331" name="__codelineno-0-331"></a><span class="sd">    This is a convenience wrapper for</span>
+</span><span id="__span-0-332"><a id="__codelineno-0-332" name="__codelineno-0-332"></a><span class="sd">    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]</span>
+</span><span id="__span-0-333"><a id="__codelineno-0-333" name="__codelineno-0-333"></a><span class="sd">    with the Shapley coefficient. Use</span>
+</span><span id="__span-0-334"><a id="__codelineno-0-334" name="__codelineno-0-334"></a><span class="sd">    [compute_shapley_values][pydvl.value.shapley.common.compute_shapley_values]</span>
+</span><span id="__span-0-335"><a id="__codelineno-0-335" name="__codelineno-0-335"></a><span class="sd">    for a more flexible interface and additional methods, including TMCS.</span>
+</span><span id="__span-0-336"><a id="__codelineno-0-336" name="__codelineno-0-336"></a>
+</span><span id="__span-0-337"><a id="__codelineno-0-337" name="__codelineno-0-337"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-338"><a id="__codelineno-0-338" name="__codelineno-0-338"></a><span class="sd">        u: Utility object with model, data, and scoring function.</span>
+</span><span id="__span-0-339"><a id="__codelineno-0-339" name="__codelineno-0-339"></a><span class="sd">        done: Stopping criterion.</span>
+</span><span id="__span-0-340"><a id="__codelineno-0-340" name="__codelineno-0-340"></a><span class="sd">        sampler_t: The sampler type to use. See the</span>
+</span><span id="__span-0-341"><a id="__codelineno-0-341" name="__codelineno-0-341"></a><span class="sd">            [sampler][pydvl.value.sampler] module for a list.</span>
+</span><span id="__span-0-342"><a id="__codelineno-0-342" name="__codelineno-0-342"></a><span class="sd">        batch_size: Number of marginal evaluations per single parallel job.</span>
+</span><span id="__span-0-343"><a id="__codelineno-0-343" name="__codelineno-0-343"></a><span class="sd">        n_jobs: Number of parallel jobs to use.</span>
+</span><span id="__span-0-344"><a id="__codelineno-0-344" name="__codelineno-0-344"></a><span class="sd">        config: Object configuring parallel computation, with cluster</span>
+</span><span id="__span-0-345"><a id="__codelineno-0-345" name="__codelineno-0-345"></a><span class="sd">            address, number of cpus, etc.</span>
+</span><span id="__span-0-346"><a id="__codelineno-0-346" name="__codelineno-0-346"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed</span>
+</span><span id="__span-0-347"><a id="__codelineno-0-347" name="__codelineno-0-347"></a><span class="sd">            for it.</span>
+</span><span id="__span-0-348"><a id="__codelineno-0-348" name="__codelineno-0-348"></a><span class="sd">        progress: Whether to display a progress bar.</span>
+</span><span id="__span-0-349"><a id="__codelineno-0-349" name="__codelineno-0-349"></a>
+</span><span id="__span-0-350"><a id="__codelineno-0-350" name="__codelineno-0-350"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-351"><a id="__codelineno-0-351" name="__codelineno-0-351"></a><span class="sd">        Object with the results.</span>
+</span><span id="__span-0-352"><a id="__codelineno-0-352" name="__codelineno-0-352"></a>
+</span><span id="__span-0-353"><a id="__codelineno-0-353" name="__codelineno-0-353"></a><span class="sd">    !!! warning &quot;Deprecation notice&quot;</span>
+</span><span id="__span-0-354"><a id="__codelineno-0-354" name="__codelineno-0-354"></a><span class="sd">        Parameter `batch_size` is for experimental use and will be removed in</span>
+</span><span id="__span-0-355"><a id="__codelineno-0-355" name="__codelineno-0-355"></a><span class="sd">        future versions.</span>
+</span><span id="__span-0-356"><a id="__codelineno-0-356" name="__codelineno-0-356"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-357"><a id="__codelineno-0-357" name="__codelineno-0-357"></a>    <span class="c1"># HACK: cannot infer return type because of useless IndexT, NameT</span>
+</span><span id="__span-0-358"><a id="__codelineno-0-358" name="__codelineno-0-358"></a>    <span class="k">return</span> <span class="n">compute_generic_semivalues</span><span class="p">(</span>  <span class="c1"># type: ignore</span>
+</span><span id="__span-0-359"><a id="__codelineno-0-359" name="__codelineno-0-359"></a>        <span class="n">sampler_t</span><span class="p">(</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">),</span>
+</span><span id="__span-0-360"><a id="__codelineno-0-360" name="__codelineno-0-360"></a>        <span class="n">u</span><span class="p">,</span>
+</span><span id="__span-0-361"><a id="__codelineno-0-361" name="__codelineno-0-361"></a>        <span class="n">shapley_coefficient</span><span class="p">,</span>
+</span><span id="__span-0-362"><a id="__codelineno-0-362" name="__codelineno-0-362"></a>        <span class="n">done</span><span class="p">,</span>
+</span><span id="__span-0-363"><a id="__codelineno-0-363" name="__codelineno-0-363"></a>        <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
+</span><span id="__span-0-364"><a id="__codelineno-0-364" name="__codelineno-0-364"></a>        <span class="n">n_jobs</span><span class="o">=</span><span class="n">n_jobs</span><span class="p">,</span>
+</span><span id="__span-0-365"><a id="__codelineno-0-365" name="__codelineno-0-365"></a>        <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span>
+</span><span id="__span-0-366"><a id="__codelineno-0-366" name="__codelineno-0-366"></a>        <span class="n">progress</span><span class="o">=</span><span class="n">progress</span><span class="p">,</span>
+</span><span id="__span-0-367"><a id="__codelineno-0-367" name="__codelineno-0-367"></a>    <span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3519,7 +3689,7 @@ <h2 id="pydvl.value.semivalues.compute_shapley_semivalues" class="doc doc-headin
 
 
 <h2 id="pydvl.value.semivalues.compute_banzhaf_semivalues" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">compute_banzhaf_semivalues</span><span class="p">(</span><span class="n">u</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">done</span><span class="o">=</span><span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span> <span class="n">sampler_t</span><span class="o">=</span><span class="n">PermutationSampler</span><span class="p">,</span> <span class="n">n_jobs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">ParallelConfig</span><span class="p">(),</span> <span class="n">progress</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span></code>
+          <code class="highlight language-python"><span class="n">compute_banzhaf_semivalues</span><span class="p">(</span><span class="n">u</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">done</span><span class="o">=</span><span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span> <span class="n">sampler_t</span><span class="o">=</span><span class="n">PermutationSampler</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">n_jobs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">ParallelConfig</span><span class="p">(),</span> <span class="n">progress</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span></code>
 
 <a href="#pydvl.value.semivalues.compute_banzhaf_semivalues" class="headerlink" title="Permanent link">&para;</a></h2>
 
@@ -3577,8 +3747,8 @@ <h2 id="pydvl.value.semivalues.compute_banzhaf_semivalues" class="doc doc-headin
           <td><code>sampler_t</code></td>
           <td class="doc-param-details">
             <div class="doc-md-description">
-              <p>The sampler type to use. See :mod:<code>pydvl.value.sampler</code> for a
-list.</p>
+              <p>The sampler type to use. See the
+<a class="autorefs autorefs-internal" href="../sampler/#pydvl.value.sampler">sampler</a> module for a list.</p>
             </div>
             <p>
                 <span class="doc-param-annotation">
@@ -3592,6 +3762,24 @@ <h2 id="pydvl.value.semivalues.compute_banzhaf_semivalues" class="doc doc-headin
             </p>
           </td>
         </tr>
+        <tr>
+          <td><code>batch_size</code></td>
+          <td class="doc-param-details">
+            <div class="doc-md-description">
+              <p>Number of marginal evaluations per single parallel job.</p>
+            </div>
+            <p>
+                <span class="doc-param-annotation">
+                  <b>TYPE:</b>
+                    <code><a class="autorefs autorefs-external" href="https://docs.python.org/3/library/functions.html#int">int</a></code>
+                </span>
+                <span class="doc-param-default">
+                  <b>DEFAULT:</b> 
+                    <code>1</code>
+                </span>
+            </p>
+          </td>
+        </tr>
         <tr>
           <td><code>n_jobs</code></td>
           <td class="doc-param-details">
@@ -3614,7 +3802,8 @@ <h2 id="pydvl.value.semivalues.compute_banzhaf_semivalues" class="doc doc-headin
           <td><code>seed</code></td>
           <td class="doc-param-details">
             <div class="doc-md-description">
-              <p>Either an instance of a numpy random number generator or a seed for it.</p>
+              <p>Either an instance of a numpy random number generator or a seed
+for it.</p>
             </div>
             <p>
                 <span class="doc-param-annotation">
@@ -3692,86 +3881,125 @@ <h2 id="pydvl.value.semivalues.compute_banzhaf_semivalues" class="doc doc-headin
         </tr>
     </tbody>
   </table>
+      <div class="admonition warning">
+<p class="admonition-title">Deprecation notice</p>
+<p>Parameter <code>batch_size</code> is for experimental use and will be removed in
+future versions.</p>
+</div>
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/semivalues.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-308">308</a></span>
-<span class="normal"><a href="#__codelineno-0-309">309</a></span>
-<span class="normal"><a href="#__codelineno-0-310">310</a></span>
-<span class="normal"><a href="#__codelineno-0-311">311</a></span>
-<span class="normal"><a href="#__codelineno-0-312">312</a></span>
-<span class="normal"><a href="#__codelineno-0-313">313</a></span>
-<span class="normal"><a href="#__codelineno-0-314">314</a></span>
-<span class="normal"><a href="#__codelineno-0-315">315</a></span>
-<span class="normal"><a href="#__codelineno-0-316">316</a></span>
-<span class="normal"><a href="#__codelineno-0-317">317</a></span>
-<span class="normal"><a href="#__codelineno-0-318">318</a></span>
-<span class="normal"><a href="#__codelineno-0-319">319</a></span>
-<span class="normal"><a href="#__codelineno-0-320">320</a></span>
-<span class="normal"><a href="#__codelineno-0-321">321</a></span>
-<span class="normal"><a href="#__codelineno-0-322">322</a></span>
-<span class="normal"><a href="#__codelineno-0-323">323</a></span>
-<span class="normal"><a href="#__codelineno-0-324">324</a></span>
-<span class="normal"><a href="#__codelineno-0-325">325</a></span>
-<span class="normal"><a href="#__codelineno-0-326">326</a></span>
-<span class="normal"><a href="#__codelineno-0-327">327</a></span>
-<span class="normal"><a href="#__codelineno-0-328">328</a></span>
-<span class="normal"><a href="#__codelineno-0-329">329</a></span>
-<span class="normal"><a href="#__codelineno-0-330">330</a></span>
-<span class="normal"><a href="#__codelineno-0-331">331</a></span>
-<span class="normal"><a href="#__codelineno-0-332">332</a></span>
-<span class="normal"><a href="#__codelineno-0-333">333</a></span>
-<span class="normal"><a href="#__codelineno-0-334">334</a></span>
-<span class="normal"><a href="#__codelineno-0-335">335</a></span>
-<span class="normal"><a href="#__codelineno-0-336">336</a></span>
-<span class="normal"><a href="#__codelineno-0-337">337</a></span>
-<span class="normal"><a href="#__codelineno-0-338">338</a></span>
-<span class="normal"><a href="#__codelineno-0-339">339</a></span>
-<span class="normal"><a href="#__codelineno-0-340">340</a></span>
-<span class="normal"><a href="#__codelineno-0-341">341</a></span>
-<span class="normal"><a href="#__codelineno-0-342">342</a></span>
-<span class="normal"><a href="#__codelineno-0-343">343</a></span>
-<span class="normal"><a href="#__codelineno-0-344">344</a></span>
-<span class="normal"><a href="#__codelineno-0-345">345</a></span>
-<span class="normal"><a href="#__codelineno-0-346">346</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-308"><a id="__codelineno-0-308" name="__codelineno-0-308"></a><span class="k">def</span> <span class="nf">compute_banzhaf_semivalues</span><span class="p">(</span>
-</span><span id="__span-0-309"><a id="__codelineno-0-309" name="__codelineno-0-309"></a>    <span class="n">u</span><span class="p">:</span> <span class="n">Utility</span><span class="p">,</span>
-</span><span id="__span-0-310"><a id="__codelineno-0-310" name="__codelineno-0-310"></a>    <span class="o">*</span><span class="p">,</span>
-</span><span id="__span-0-311"><a id="__codelineno-0-311" name="__codelineno-0-311"></a>    <span class="n">done</span><span class="p">:</span> <span class="n">StoppingCriterion</span> <span class="o">=</span> <span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span>
-</span><span id="__span-0-312"><a id="__codelineno-0-312" name="__codelineno-0-312"></a>    <span class="n">sampler_t</span><span class="p">:</span> <span class="n">Type</span><span class="p">[</span><span class="n">StochasticSampler</span><span class="p">]</span> <span class="o">=</span> <span class="n">PermutationSampler</span><span class="p">,</span>
-</span><span id="__span-0-313"><a id="__codelineno-0-313" name="__codelineno-0-313"></a>    <span class="n">n_jobs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-</span><span id="__span-0-314"><a id="__codelineno-0-314" name="__codelineno-0-314"></a>    <span class="n">config</span><span class="p">:</span> <span class="n">ParallelConfig</span> <span class="o">=</span> <span class="n">ParallelConfig</span><span class="p">(),</span>
-</span><span id="__span-0-315"><a id="__codelineno-0-315" name="__codelineno-0-315"></a>    <span class="n">progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-</span><span id="__span-0-316"><a id="__codelineno-0-316" name="__codelineno-0-316"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-317"><a id="__codelineno-0-317" name="__codelineno-0-317"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">:</span>
-</span><span id="__span-0-318"><a id="__codelineno-0-318" name="__codelineno-0-318"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Computes Banzhaf values for a given utility function.</span>
-</span><span id="__span-0-319"><a id="__codelineno-0-319" name="__codelineno-0-319"></a>
-</span><span id="__span-0-320"><a id="__codelineno-0-320" name="__codelineno-0-320"></a><span class="sd">    This is a convenience wrapper for</span>
-</span><span id="__span-0-321"><a id="__codelineno-0-321" name="__codelineno-0-321"></a><span class="sd">    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]</span>
-</span><span id="__span-0-322"><a id="__codelineno-0-322" name="__codelineno-0-322"></a><span class="sd">    with the Banzhaf coefficient.</span>
-</span><span id="__span-0-323"><a id="__codelineno-0-323" name="__codelineno-0-323"></a>
-</span><span id="__span-0-324"><a id="__codelineno-0-324" name="__codelineno-0-324"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-325"><a id="__codelineno-0-325" name="__codelineno-0-325"></a><span class="sd">        u: Utility object with model, data, and scoring function.</span>
-</span><span id="__span-0-326"><a id="__codelineno-0-326" name="__codelineno-0-326"></a><span class="sd">        done: Stopping criterion.</span>
-</span><span id="__span-0-327"><a id="__codelineno-0-327" name="__codelineno-0-327"></a><span class="sd">        sampler_t: The sampler type to use. See :mod:`pydvl.value.sampler` for a</span>
-</span><span id="__span-0-328"><a id="__codelineno-0-328" name="__codelineno-0-328"></a><span class="sd">            list.</span>
-</span><span id="__span-0-329"><a id="__codelineno-0-329" name="__codelineno-0-329"></a><span class="sd">        n_jobs: Number of parallel jobs to use.</span>
-</span><span id="__span-0-330"><a id="__codelineno-0-330" name="__codelineno-0-330"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
-</span><span id="__span-0-331"><a id="__codelineno-0-331" name="__codelineno-0-331"></a><span class="sd">        config: Object configuring parallel computation, with cluster address,</span>
-</span><span id="__span-0-332"><a id="__codelineno-0-332" name="__codelineno-0-332"></a><span class="sd">            number of cpus, etc.</span>
-</span><span id="__span-0-333"><a id="__codelineno-0-333" name="__codelineno-0-333"></a><span class="sd">        progress: Whether to display a progress bar.</span>
-</span><span id="__span-0-334"><a id="__codelineno-0-334" name="__codelineno-0-334"></a>
-</span><span id="__span-0-335"><a id="__codelineno-0-335" name="__codelineno-0-335"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-336"><a id="__codelineno-0-336" name="__codelineno-0-336"></a><span class="sd">        Object with the results.</span>
-</span><span id="__span-0-337"><a id="__codelineno-0-337" name="__codelineno-0-337"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-338"><a id="__codelineno-0-338" name="__codelineno-0-338"></a>    <span class="k">return</span> <span class="n">compute_generic_semivalues</span><span class="p">(</span>
-</span><span id="__span-0-339"><a id="__codelineno-0-339" name="__codelineno-0-339"></a>        <span class="n">sampler_t</span><span class="p">(</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">),</span>
-</span><span id="__span-0-340"><a id="__codelineno-0-340" name="__codelineno-0-340"></a>        <span class="n">u</span><span class="p">,</span>
-</span><span id="__span-0-341"><a id="__codelineno-0-341" name="__codelineno-0-341"></a>        <span class="n">banzhaf_coefficient</span><span class="p">,</span>
-</span><span id="__span-0-342"><a id="__codelineno-0-342" name="__codelineno-0-342"></a>        <span class="n">done</span><span class="p">,</span>
-</span><span id="__span-0-343"><a id="__codelineno-0-343" name="__codelineno-0-343"></a>        <span class="n">n_jobs</span><span class="o">=</span><span class="n">n_jobs</span><span class="p">,</span>
-</span><span id="__span-0-344"><a id="__codelineno-0-344" name="__codelineno-0-344"></a>        <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span>
-</span><span id="__span-0-345"><a id="__codelineno-0-345" name="__codelineno-0-345"></a>        <span class="n">progress</span><span class="o">=</span><span class="n">progress</span><span class="p">,</span>
-</span><span id="__span-0-346"><a id="__codelineno-0-346" name="__codelineno-0-346"></a>    <span class="p">)</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-370">370</a></span>
+<span class="normal"><a href="#__codelineno-0-371">371</a></span>
+<span class="normal"><a href="#__codelineno-0-372">372</a></span>
+<span class="normal"><a href="#__codelineno-0-373">373</a></span>
+<span class="normal"><a href="#__codelineno-0-374">374</a></span>
+<span class="normal"><a href="#__codelineno-0-375">375</a></span>
+<span class="normal"><a href="#__codelineno-0-376">376</a></span>
+<span class="normal"><a href="#__codelineno-0-377">377</a></span>
+<span class="normal"><a href="#__codelineno-0-378">378</a></span>
+<span class="normal"><a href="#__codelineno-0-379">379</a></span>
+<span class="normal"><a href="#__codelineno-0-380">380</a></span>
+<span class="normal"><a href="#__codelineno-0-381">381</a></span>
+<span class="normal"><a href="#__codelineno-0-382">382</a></span>
+<span class="normal"><a href="#__codelineno-0-383">383</a></span>
+<span class="normal"><a href="#__codelineno-0-384">384</a></span>
+<span class="normal"><a href="#__codelineno-0-385">385</a></span>
+<span class="normal"><a href="#__codelineno-0-386">386</a></span>
+<span class="normal"><a href="#__codelineno-0-387">387</a></span>
+<span class="normal"><a href="#__codelineno-0-388">388</a></span>
+<span class="normal"><a href="#__codelineno-0-389">389</a></span>
+<span class="normal"><a href="#__codelineno-0-390">390</a></span>
+<span class="normal"><a href="#__codelineno-0-391">391</a></span>
+<span class="normal"><a href="#__codelineno-0-392">392</a></span>
+<span class="normal"><a href="#__codelineno-0-393">393</a></span>
+<span class="normal"><a href="#__codelineno-0-394">394</a></span>
+<span class="normal"><a href="#__codelineno-0-395">395</a></span>
+<span class="normal"><a href="#__codelineno-0-396">396</a></span>
+<span class="normal"><a href="#__codelineno-0-397">397</a></span>
+<span class="normal"><a href="#__codelineno-0-398">398</a></span>
+<span class="normal"><a href="#__codelineno-0-399">399</a></span>
+<span class="normal"><a href="#__codelineno-0-400">400</a></span>
+<span class="normal"><a href="#__codelineno-0-401">401</a></span>
+<span class="normal"><a href="#__codelineno-0-402">402</a></span>
+<span class="normal"><a href="#__codelineno-0-403">403</a></span>
+<span class="normal"><a href="#__codelineno-0-404">404</a></span>
+<span class="normal"><a href="#__codelineno-0-405">405</a></span>
+<span class="normal"><a href="#__codelineno-0-406">406</a></span>
+<span class="normal"><a href="#__codelineno-0-407">407</a></span>
+<span class="normal"><a href="#__codelineno-0-408">408</a></span>
+<span class="normal"><a href="#__codelineno-0-409">409</a></span>
+<span class="normal"><a href="#__codelineno-0-410">410</a></span>
+<span class="normal"><a href="#__codelineno-0-411">411</a></span>
+<span class="normal"><a href="#__codelineno-0-412">412</a></span>
+<span class="normal"><a href="#__codelineno-0-413">413</a></span>
+<span class="normal"><a href="#__codelineno-0-414">414</a></span>
+<span class="normal"><a href="#__codelineno-0-415">415</a></span>
+<span class="normal"><a href="#__codelineno-0-416">416</a></span>
+<span class="normal"><a href="#__codelineno-0-417">417</a></span>
+<span class="normal"><a href="#__codelineno-0-418">418</a></span>
+<span class="normal"><a href="#__codelineno-0-419">419</a></span>
+<span class="normal"><a href="#__codelineno-0-420">420</a></span>
+<span class="normal"><a href="#__codelineno-0-421">421</a></span>
+<span class="normal"><a href="#__codelineno-0-422">422</a></span>
+<span class="normal"><a href="#__codelineno-0-423">423</a></span>
+<span class="normal"><a href="#__codelineno-0-424">424</a></span>
+<span class="normal"><a href="#__codelineno-0-425">425</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-370"><a id="__codelineno-0-370" name="__codelineno-0-370"></a><span class="nd">@deprecated</span><span class="p">(</span>
+</span><span id="__span-0-371"><a id="__codelineno-0-371" name="__codelineno-0-371"></a>    <span class="n">target</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+</span><span id="__span-0-372"><a id="__codelineno-0-372" name="__codelineno-0-372"></a>    <span class="n">deprecated_in</span><span class="o">=</span><span class="s2">&quot;0.7.0&quot;</span><span class="p">,</span>
+</span><span id="__span-0-373"><a id="__codelineno-0-373" name="__codelineno-0-373"></a>    <span class="n">remove_in</span><span class="o">=</span><span class="s2">&quot;0.9.0&quot;</span><span class="p">,</span>
+</span><span id="__span-0-374"><a id="__codelineno-0-374" name="__codelineno-0-374"></a>    <span class="n">args_mapping</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;batch_size&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">},</span>
+</span><span id="__span-0-375"><a id="__codelineno-0-375" name="__codelineno-0-375"></a>    <span class="n">template_mgs</span><span class="o">=</span><span class="s2">&quot;batch_size is for experimental use and will be removed&quot;</span>
+</span><span id="__span-0-376"><a id="__codelineno-0-376" name="__codelineno-0-376"></a>    <span class="s2">&quot;in future versions.&quot;</span><span class="p">,</span>
+</span><span id="__span-0-377"><a id="__codelineno-0-377" name="__codelineno-0-377"></a><span class="p">)</span>
+</span><span id="__span-0-378"><a id="__codelineno-0-378" name="__codelineno-0-378"></a><span class="k">def</span> <span class="nf">compute_banzhaf_semivalues</span><span class="p">(</span>
+</span><span id="__span-0-379"><a id="__codelineno-0-379" name="__codelineno-0-379"></a>    <span class="n">u</span><span class="p">:</span> <span class="n">Utility</span><span class="p">,</span>
+</span><span id="__span-0-380"><a id="__codelineno-0-380" name="__codelineno-0-380"></a>    <span class="o">*</span><span class="p">,</span>
+</span><span id="__span-0-381"><a id="__codelineno-0-381" name="__codelineno-0-381"></a>    <span class="n">done</span><span class="p">:</span> <span class="n">StoppingCriterion</span> <span class="o">=</span> <span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span>
+</span><span id="__span-0-382"><a id="__codelineno-0-382" name="__codelineno-0-382"></a>    <span class="n">sampler_t</span><span class="p">:</span> <span class="n">Type</span><span class="p">[</span><span class="n">StochasticSampler</span><span class="p">]</span> <span class="o">=</span> <span class="n">PermutationSampler</span><span class="p">,</span>
+</span><span id="__span-0-383"><a id="__codelineno-0-383" name="__codelineno-0-383"></a>    <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+</span><span id="__span-0-384"><a id="__codelineno-0-384" name="__codelineno-0-384"></a>    <span class="n">n_jobs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+</span><span id="__span-0-385"><a id="__codelineno-0-385" name="__codelineno-0-385"></a>    <span class="n">config</span><span class="p">:</span> <span class="n">ParallelConfig</span> <span class="o">=</span> <span class="n">ParallelConfig</span><span class="p">(),</span>
+</span><span id="__span-0-386"><a id="__codelineno-0-386" name="__codelineno-0-386"></a>    <span class="n">progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+</span><span id="__span-0-387"><a id="__codelineno-0-387" name="__codelineno-0-387"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-388"><a id="__codelineno-0-388" name="__codelineno-0-388"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">:</span>
+</span><span id="__span-0-389"><a id="__codelineno-0-389" name="__codelineno-0-389"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Computes Banzhaf values for a given utility function.</span>
+</span><span id="__span-0-390"><a id="__codelineno-0-390" name="__codelineno-0-390"></a>
+</span><span id="__span-0-391"><a id="__codelineno-0-391" name="__codelineno-0-391"></a><span class="sd">    This is a convenience wrapper for</span>
+</span><span id="__span-0-392"><a id="__codelineno-0-392" name="__codelineno-0-392"></a><span class="sd">    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]</span>
+</span><span id="__span-0-393"><a id="__codelineno-0-393" name="__codelineno-0-393"></a><span class="sd">    with the Banzhaf coefficient.</span>
+</span><span id="__span-0-394"><a id="__codelineno-0-394" name="__codelineno-0-394"></a>
+</span><span id="__span-0-395"><a id="__codelineno-0-395" name="__codelineno-0-395"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-396"><a id="__codelineno-0-396" name="__codelineno-0-396"></a><span class="sd">        u: Utility object with model, data, and scoring function.</span>
+</span><span id="__span-0-397"><a id="__codelineno-0-397" name="__codelineno-0-397"></a><span class="sd">        done: Stopping criterion.</span>
+</span><span id="__span-0-398"><a id="__codelineno-0-398" name="__codelineno-0-398"></a><span class="sd">        sampler_t: The sampler type to use. See the</span>
+</span><span id="__span-0-399"><a id="__codelineno-0-399" name="__codelineno-0-399"></a><span class="sd">            [sampler][pydvl.value.sampler] module for a list.</span>
+</span><span id="__span-0-400"><a id="__codelineno-0-400" name="__codelineno-0-400"></a><span class="sd">        batch_size: Number of marginal evaluations per single parallel job.</span>
+</span><span id="__span-0-401"><a id="__codelineno-0-401" name="__codelineno-0-401"></a><span class="sd">        n_jobs: Number of parallel jobs to use.</span>
+</span><span id="__span-0-402"><a id="__codelineno-0-402" name="__codelineno-0-402"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed</span>
+</span><span id="__span-0-403"><a id="__codelineno-0-403" name="__codelineno-0-403"></a><span class="sd">            for it.</span>
+</span><span id="__span-0-404"><a id="__codelineno-0-404" name="__codelineno-0-404"></a><span class="sd">        config: Object configuring parallel computation, with cluster address,</span>
+</span><span id="__span-0-405"><a id="__codelineno-0-405" name="__codelineno-0-405"></a><span class="sd">            number of cpus, etc.</span>
+</span><span id="__span-0-406"><a id="__codelineno-0-406" name="__codelineno-0-406"></a><span class="sd">        progress: Whether to display a progress bar.</span>
+</span><span id="__span-0-407"><a id="__codelineno-0-407" name="__codelineno-0-407"></a>
+</span><span id="__span-0-408"><a id="__codelineno-0-408" name="__codelineno-0-408"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-409"><a id="__codelineno-0-409" name="__codelineno-0-409"></a><span class="sd">        Object with the results.</span>
+</span><span id="__span-0-410"><a id="__codelineno-0-410" name="__codelineno-0-410"></a>
+</span><span id="__span-0-411"><a id="__codelineno-0-411" name="__codelineno-0-411"></a><span class="sd">    !!! warning &quot;Deprecation notice&quot;</span>
+</span><span id="__span-0-412"><a id="__codelineno-0-412" name="__codelineno-0-412"></a><span class="sd">        Parameter `batch_size` is for experimental use and will be removed in</span>
+</span><span id="__span-0-413"><a id="__codelineno-0-413" name="__codelineno-0-413"></a><span class="sd">        future versions.</span>
+</span><span id="__span-0-414"><a id="__codelineno-0-414" name="__codelineno-0-414"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-415"><a id="__codelineno-0-415" name="__codelineno-0-415"></a>    <span class="c1"># HACK: cannot infer return type because of useless IndexT, NameT</span>
+</span><span id="__span-0-416"><a id="__codelineno-0-416" name="__codelineno-0-416"></a>    <span class="k">return</span> <span class="n">compute_generic_semivalues</span><span class="p">(</span>  <span class="c1"># type: ignore</span>
+</span><span id="__span-0-417"><a id="__codelineno-0-417" name="__codelineno-0-417"></a>        <span class="n">sampler_t</span><span class="p">(</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">),</span>
+</span><span id="__span-0-418"><a id="__codelineno-0-418" name="__codelineno-0-418"></a>        <span class="n">u</span><span class="p">,</span>
+</span><span id="__span-0-419"><a id="__codelineno-0-419" name="__codelineno-0-419"></a>        <span class="n">banzhaf_coefficient</span><span class="p">,</span>
+</span><span id="__span-0-420"><a id="__codelineno-0-420" name="__codelineno-0-420"></a>        <span class="n">done</span><span class="p">,</span>
+</span><span id="__span-0-421"><a id="__codelineno-0-421" name="__codelineno-0-421"></a>        <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
+</span><span id="__span-0-422"><a id="__codelineno-0-422" name="__codelineno-0-422"></a>        <span class="n">n_jobs</span><span class="o">=</span><span class="n">n_jobs</span><span class="p">,</span>
+</span><span id="__span-0-423"><a id="__codelineno-0-423" name="__codelineno-0-423"></a>        <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span>
+</span><span id="__span-0-424"><a id="__codelineno-0-424" name="__codelineno-0-424"></a>        <span class="n">progress</span><span class="o">=</span><span class="n">progress</span><span class="p">,</span>
+</span><span id="__span-0-425"><a id="__codelineno-0-425" name="__codelineno-0-425"></a>    <span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3785,7 +4013,7 @@ <h2 id="pydvl.value.semivalues.compute_banzhaf_semivalues" class="doc doc-headin
 
 
 <h2 id="pydvl.value.semivalues.compute_beta_shapley_semivalues" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">compute_beta_shapley_semivalues</span><span class="p">(</span><span class="n">u</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">beta</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">done</span><span class="o">=</span><span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span> <span class="n">sampler_t</span><span class="o">=</span><span class="n">PermutationSampler</span><span class="p">,</span> <span class="n">n_jobs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">ParallelConfig</span><span class="p">(),</span> <span class="n">progress</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span></code>
+          <code class="highlight language-python"><span class="n">compute_beta_shapley_semivalues</span><span class="p">(</span><span class="n">u</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">beta</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">done</span><span class="o">=</span><span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span> <span class="n">sampler_t</span><span class="o">=</span><span class="n">PermutationSampler</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">n_jobs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">ParallelConfig</span><span class="p">(),</span> <span class="n">progress</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span></code>
 
 <a href="#pydvl.value.semivalues.compute_beta_shapley_semivalues" class="headerlink" title="Permanent link">&para;</a></h2>
 
@@ -3879,7 +4107,8 @@ <h2 id="pydvl.value.semivalues.compute_beta_shapley_semivalues" class="doc doc-h
           <td><code>sampler_t</code></td>
           <td class="doc-param-details">
             <div class="doc-md-description">
-              <p>The sampler type to use. See :mod:<code>pydvl.value.sampler</code> for a list.</p>
+              <p>The sampler type to use. See the
+<a class="autorefs autorefs-internal" href="../sampler/#pydvl.value.sampler">sampler</a> module for a list.</p>
             </div>
             <p>
                 <span class="doc-param-annotation">
@@ -3893,6 +4122,24 @@ <h2 id="pydvl.value.semivalues.compute_beta_shapley_semivalues" class="doc doc-h
             </p>
           </td>
         </tr>
+        <tr>
+          <td><code>batch_size</code></td>
+          <td class="doc-param-details">
+            <div class="doc-md-description">
+              <p>Number of marginal evaluations per (parallelized) task.</p>
+            </div>
+            <p>
+                <span class="doc-param-annotation">
+                  <b>TYPE:</b>
+                    <code><a class="autorefs autorefs-external" href="https://docs.python.org/3/library/functions.html#int">int</a></code>
+                </span>
+                <span class="doc-param-default">
+                  <b>DEFAULT:</b> 
+                    <code>1</code>
+                </span>
+            </p>
+          </td>
+        </tr>
         <tr>
           <td><code>n_jobs</code></td>
           <td class="doc-param-details">
@@ -3993,92 +4240,131 @@ <h2 id="pydvl.value.semivalues.compute_beta_shapley_semivalues" class="doc doc-h
         </tr>
     </tbody>
   </table>
+      <div class="admonition warning">
+<p class="admonition-title">Deprecation notice</p>
+<p>Parameter <code>batch_size</code> is for experimental use and will be removed in
+future versions.</p>
+</div>
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/semivalues.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-349">349</a></span>
-<span class="normal"><a href="#__codelineno-0-350">350</a></span>
-<span class="normal"><a href="#__codelineno-0-351">351</a></span>
-<span class="normal"><a href="#__codelineno-0-352">352</a></span>
-<span class="normal"><a href="#__codelineno-0-353">353</a></span>
-<span class="normal"><a href="#__codelineno-0-354">354</a></span>
-<span class="normal"><a href="#__codelineno-0-355">355</a></span>
-<span class="normal"><a href="#__codelineno-0-356">356</a></span>
-<span class="normal"><a href="#__codelineno-0-357">357</a></span>
-<span class="normal"><a href="#__codelineno-0-358">358</a></span>
-<span class="normal"><a href="#__codelineno-0-359">359</a></span>
-<span class="normal"><a href="#__codelineno-0-360">360</a></span>
-<span class="normal"><a href="#__codelineno-0-361">361</a></span>
-<span class="normal"><a href="#__codelineno-0-362">362</a></span>
-<span class="normal"><a href="#__codelineno-0-363">363</a></span>
-<span class="normal"><a href="#__codelineno-0-364">364</a></span>
-<span class="normal"><a href="#__codelineno-0-365">365</a></span>
-<span class="normal"><a href="#__codelineno-0-366">366</a></span>
-<span class="normal"><a href="#__codelineno-0-367">367</a></span>
-<span class="normal"><a href="#__codelineno-0-368">368</a></span>
-<span class="normal"><a href="#__codelineno-0-369">369</a></span>
-<span class="normal"><a href="#__codelineno-0-370">370</a></span>
-<span class="normal"><a href="#__codelineno-0-371">371</a></span>
-<span class="normal"><a href="#__codelineno-0-372">372</a></span>
-<span class="normal"><a href="#__codelineno-0-373">373</a></span>
-<span class="normal"><a href="#__codelineno-0-374">374</a></span>
-<span class="normal"><a href="#__codelineno-0-375">375</a></span>
-<span class="normal"><a href="#__codelineno-0-376">376</a></span>
-<span class="normal"><a href="#__codelineno-0-377">377</a></span>
-<span class="normal"><a href="#__codelineno-0-378">378</a></span>
-<span class="normal"><a href="#__codelineno-0-379">379</a></span>
-<span class="normal"><a href="#__codelineno-0-380">380</a></span>
-<span class="normal"><a href="#__codelineno-0-381">381</a></span>
-<span class="normal"><a href="#__codelineno-0-382">382</a></span>
-<span class="normal"><a href="#__codelineno-0-383">383</a></span>
-<span class="normal"><a href="#__codelineno-0-384">384</a></span>
-<span class="normal"><a href="#__codelineno-0-385">385</a></span>
-<span class="normal"><a href="#__codelineno-0-386">386</a></span>
-<span class="normal"><a href="#__codelineno-0-387">387</a></span>
-<span class="normal"><a href="#__codelineno-0-388">388</a></span>
-<span class="normal"><a href="#__codelineno-0-389">389</a></span>
-<span class="normal"><a href="#__codelineno-0-390">390</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-349"><a id="__codelineno-0-349" name="__codelineno-0-349"></a><span class="k">def</span> <span class="nf">compute_beta_shapley_semivalues</span><span class="p">(</span>
-</span><span id="__span-0-350"><a id="__codelineno-0-350" name="__codelineno-0-350"></a>    <span class="n">u</span><span class="p">:</span> <span class="n">Utility</span><span class="p">,</span>
-</span><span id="__span-0-351"><a id="__codelineno-0-351" name="__codelineno-0-351"></a>    <span class="o">*</span><span class="p">,</span>
-</span><span id="__span-0-352"><a id="__codelineno-0-352" name="__codelineno-0-352"></a>    <span class="n">alpha</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-</span><span id="__span-0-353"><a id="__codelineno-0-353" name="__codelineno-0-353"></a>    <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-</span><span id="__span-0-354"><a id="__codelineno-0-354" name="__codelineno-0-354"></a>    <span class="n">done</span><span class="p">:</span> <span class="n">StoppingCriterion</span> <span class="o">=</span> <span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span>
-</span><span id="__span-0-355"><a id="__codelineno-0-355" name="__codelineno-0-355"></a>    <span class="n">sampler_t</span><span class="p">:</span> <span class="n">Type</span><span class="p">[</span><span class="n">StochasticSampler</span><span class="p">]</span> <span class="o">=</span> <span class="n">PermutationSampler</span><span class="p">,</span>
-</span><span id="__span-0-356"><a id="__codelineno-0-356" name="__codelineno-0-356"></a>    <span class="n">n_jobs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-</span><span id="__span-0-357"><a id="__codelineno-0-357" name="__codelineno-0-357"></a>    <span class="n">config</span><span class="p">:</span> <span class="n">ParallelConfig</span> <span class="o">=</span> <span class="n">ParallelConfig</span><span class="p">(),</span>
-</span><span id="__span-0-358"><a id="__codelineno-0-358" name="__codelineno-0-358"></a>    <span class="n">progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-</span><span id="__span-0-359"><a id="__codelineno-0-359" name="__codelineno-0-359"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-360"><a id="__codelineno-0-360" name="__codelineno-0-360"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">:</span>
-</span><span id="__span-0-361"><a id="__codelineno-0-361" name="__codelineno-0-361"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Computes Beta Shapley values for a given utility function.</span>
-</span><span id="__span-0-362"><a id="__codelineno-0-362" name="__codelineno-0-362"></a>
-</span><span id="__span-0-363"><a id="__codelineno-0-363" name="__codelineno-0-363"></a><span class="sd">    This is a convenience wrapper for</span>
-</span><span id="__span-0-364"><a id="__codelineno-0-364" name="__codelineno-0-364"></a><span class="sd">    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]</span>
-</span><span id="__span-0-365"><a id="__codelineno-0-365" name="__codelineno-0-365"></a><span class="sd">    with the Beta Shapley coefficient.</span>
-</span><span id="__span-0-366"><a id="__codelineno-0-366" name="__codelineno-0-366"></a>
-</span><span id="__span-0-367"><a id="__codelineno-0-367" name="__codelineno-0-367"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-368"><a id="__codelineno-0-368" name="__codelineno-0-368"></a><span class="sd">        u: Utility object with model, data, and scoring function.</span>
-</span><span id="__span-0-369"><a id="__codelineno-0-369" name="__codelineno-0-369"></a><span class="sd">        alpha: Alpha parameter of the Beta distribution.</span>
-</span><span id="__span-0-370"><a id="__codelineno-0-370" name="__codelineno-0-370"></a><span class="sd">        beta: Beta parameter of the Beta distribution.</span>
-</span><span id="__span-0-371"><a id="__codelineno-0-371" name="__codelineno-0-371"></a><span class="sd">        done: Stopping criterion.</span>
-</span><span id="__span-0-372"><a id="__codelineno-0-372" name="__codelineno-0-372"></a><span class="sd">        sampler_t: The sampler type to use. See :mod:`pydvl.value.sampler` for a list.</span>
-</span><span id="__span-0-373"><a id="__codelineno-0-373" name="__codelineno-0-373"></a><span class="sd">        n_jobs: Number of parallel jobs to use.</span>
-</span><span id="__span-0-374"><a id="__codelineno-0-374" name="__codelineno-0-374"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
-</span><span id="__span-0-375"><a id="__codelineno-0-375" name="__codelineno-0-375"></a><span class="sd">        config: Object configuring parallel computation, with cluster address, number of</span>
-</span><span id="__span-0-376"><a id="__codelineno-0-376" name="__codelineno-0-376"></a><span class="sd">            cpus, etc.</span>
-</span><span id="__span-0-377"><a id="__codelineno-0-377" name="__codelineno-0-377"></a><span class="sd">        progress: Whether to display a progress bar.</span>
-</span><span id="__span-0-378"><a id="__codelineno-0-378" name="__codelineno-0-378"></a>
-</span><span id="__span-0-379"><a id="__codelineno-0-379" name="__codelineno-0-379"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-380"><a id="__codelineno-0-380" name="__codelineno-0-380"></a><span class="sd">        Object with the results.</span>
-</span><span id="__span-0-381"><a id="__codelineno-0-381" name="__codelineno-0-381"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-382"><a id="__codelineno-0-382" name="__codelineno-0-382"></a>    <span class="k">return</span> <span class="n">compute_generic_semivalues</span><span class="p">(</span>
-</span><span id="__span-0-383"><a id="__codelineno-0-383" name="__codelineno-0-383"></a>        <span class="n">sampler_t</span><span class="p">(</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">),</span>
-</span><span id="__span-0-384"><a id="__codelineno-0-384" name="__codelineno-0-384"></a>        <span class="n">u</span><span class="p">,</span>
-</span><span id="__span-0-385"><a id="__codelineno-0-385" name="__codelineno-0-385"></a>        <span class="n">beta_coefficient</span><span class="p">(</span><span class="n">alpha</span><span class="p">,</span> <span class="n">beta</span><span class="p">),</span>
-</span><span id="__span-0-386"><a id="__codelineno-0-386" name="__codelineno-0-386"></a>        <span class="n">done</span><span class="p">,</span>
-</span><span id="__span-0-387"><a id="__codelineno-0-387" name="__codelineno-0-387"></a>        <span class="n">n_jobs</span><span class="o">=</span><span class="n">n_jobs</span><span class="p">,</span>
-</span><span id="__span-0-388"><a id="__codelineno-0-388" name="__codelineno-0-388"></a>        <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span>
-</span><span id="__span-0-389"><a id="__codelineno-0-389" name="__codelineno-0-389"></a>        <span class="n">progress</span><span class="o">=</span><span class="n">progress</span><span class="p">,</span>
-</span><span id="__span-0-390"><a id="__codelineno-0-390" name="__codelineno-0-390"></a>    <span class="p">)</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-428">428</a></span>
+<span class="normal"><a href="#__codelineno-0-429">429</a></span>
+<span class="normal"><a href="#__codelineno-0-430">430</a></span>
+<span class="normal"><a href="#__codelineno-0-431">431</a></span>
+<span class="normal"><a href="#__codelineno-0-432">432</a></span>
+<span class="normal"><a href="#__codelineno-0-433">433</a></span>
+<span class="normal"><a href="#__codelineno-0-434">434</a></span>
+<span class="normal"><a href="#__codelineno-0-435">435</a></span>
+<span class="normal"><a href="#__codelineno-0-436">436</a></span>
+<span class="normal"><a href="#__codelineno-0-437">437</a></span>
+<span class="normal"><a href="#__codelineno-0-438">438</a></span>
+<span class="normal"><a href="#__codelineno-0-439">439</a></span>
+<span class="normal"><a href="#__codelineno-0-440">440</a></span>
+<span class="normal"><a href="#__codelineno-0-441">441</a></span>
+<span class="normal"><a href="#__codelineno-0-442">442</a></span>
+<span class="normal"><a href="#__codelineno-0-443">443</a></span>
+<span class="normal"><a href="#__codelineno-0-444">444</a></span>
+<span class="normal"><a href="#__codelineno-0-445">445</a></span>
+<span class="normal"><a href="#__codelineno-0-446">446</a></span>
+<span class="normal"><a href="#__codelineno-0-447">447</a></span>
+<span class="normal"><a href="#__codelineno-0-448">448</a></span>
+<span class="normal"><a href="#__codelineno-0-449">449</a></span>
+<span class="normal"><a href="#__codelineno-0-450">450</a></span>
+<span class="normal"><a href="#__codelineno-0-451">451</a></span>
+<span class="normal"><a href="#__codelineno-0-452">452</a></span>
+<span class="normal"><a href="#__codelineno-0-453">453</a></span>
+<span class="normal"><a href="#__codelineno-0-454">454</a></span>
+<span class="normal"><a href="#__codelineno-0-455">455</a></span>
+<span class="normal"><a href="#__codelineno-0-456">456</a></span>
+<span class="normal"><a href="#__codelineno-0-457">457</a></span>
+<span class="normal"><a href="#__codelineno-0-458">458</a></span>
+<span class="normal"><a href="#__codelineno-0-459">459</a></span>
+<span class="normal"><a href="#__codelineno-0-460">460</a></span>
+<span class="normal"><a href="#__codelineno-0-461">461</a></span>
+<span class="normal"><a href="#__codelineno-0-462">462</a></span>
+<span class="normal"><a href="#__codelineno-0-463">463</a></span>
+<span class="normal"><a href="#__codelineno-0-464">464</a></span>
+<span class="normal"><a href="#__codelineno-0-465">465</a></span>
+<span class="normal"><a href="#__codelineno-0-466">466</a></span>
+<span class="normal"><a href="#__codelineno-0-467">467</a></span>
+<span class="normal"><a href="#__codelineno-0-468">468</a></span>
+<span class="normal"><a href="#__codelineno-0-469">469</a></span>
+<span class="normal"><a href="#__codelineno-0-470">470</a></span>
+<span class="normal"><a href="#__codelineno-0-471">471</a></span>
+<span class="normal"><a href="#__codelineno-0-472">472</a></span>
+<span class="normal"><a href="#__codelineno-0-473">473</a></span>
+<span class="normal"><a href="#__codelineno-0-474">474</a></span>
+<span class="normal"><a href="#__codelineno-0-475">475</a></span>
+<span class="normal"><a href="#__codelineno-0-476">476</a></span>
+<span class="normal"><a href="#__codelineno-0-477">477</a></span>
+<span class="normal"><a href="#__codelineno-0-478">478</a></span>
+<span class="normal"><a href="#__codelineno-0-479">479</a></span>
+<span class="normal"><a href="#__codelineno-0-480">480</a></span>
+<span class="normal"><a href="#__codelineno-0-481">481</a></span>
+<span class="normal"><a href="#__codelineno-0-482">482</a></span>
+<span class="normal"><a href="#__codelineno-0-483">483</a></span>
+<span class="normal"><a href="#__codelineno-0-484">484</a></span>
+<span class="normal"><a href="#__codelineno-0-485">485</a></span>
+<span class="normal"><a href="#__codelineno-0-486">486</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-428"><a id="__codelineno-0-428" name="__codelineno-0-428"></a><span class="nd">@deprecated</span><span class="p">(</span>
+</span><span id="__span-0-429"><a id="__codelineno-0-429" name="__codelineno-0-429"></a>    <span class="n">target</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+</span><span id="__span-0-430"><a id="__codelineno-0-430" name="__codelineno-0-430"></a>    <span class="n">deprecated_in</span><span class="o">=</span><span class="s2">&quot;0.7.0&quot;</span><span class="p">,</span>
+</span><span id="__span-0-431"><a id="__codelineno-0-431" name="__codelineno-0-431"></a>    <span class="n">remove_in</span><span class="o">=</span><span class="s2">&quot;0.9.0&quot;</span><span class="p">,</span>
+</span><span id="__span-0-432"><a id="__codelineno-0-432" name="__codelineno-0-432"></a>    <span class="n">args_mapping</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;batch_size&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">},</span>
+</span><span id="__span-0-433"><a id="__codelineno-0-433" name="__codelineno-0-433"></a>    <span class="n">template_mgs</span><span class="o">=</span><span class="s2">&quot;batch_size is for experimental use and will be removed&quot;</span>
+</span><span id="__span-0-434"><a id="__codelineno-0-434" name="__codelineno-0-434"></a>    <span class="s2">&quot;in future versions.&quot;</span><span class="p">,</span>
+</span><span id="__span-0-435"><a id="__codelineno-0-435" name="__codelineno-0-435"></a><span class="p">)</span>
+</span><span id="__span-0-436"><a id="__codelineno-0-436" name="__codelineno-0-436"></a><span class="k">def</span> <span class="nf">compute_beta_shapley_semivalues</span><span class="p">(</span>
+</span><span id="__span-0-437"><a id="__codelineno-0-437" name="__codelineno-0-437"></a>    <span class="n">u</span><span class="p">:</span> <span class="n">Utility</span><span class="p">,</span>
+</span><span id="__span-0-438"><a id="__codelineno-0-438" name="__codelineno-0-438"></a>    <span class="o">*</span><span class="p">,</span>
+</span><span id="__span-0-439"><a id="__codelineno-0-439" name="__codelineno-0-439"></a>    <span class="n">alpha</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+</span><span id="__span-0-440"><a id="__codelineno-0-440" name="__codelineno-0-440"></a>    <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+</span><span id="__span-0-441"><a id="__codelineno-0-441" name="__codelineno-0-441"></a>    <span class="n">done</span><span class="p">:</span> <span class="n">StoppingCriterion</span> <span class="o">=</span> <span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span>
+</span><span id="__span-0-442"><a id="__codelineno-0-442" name="__codelineno-0-442"></a>    <span class="n">sampler_t</span><span class="p">:</span> <span class="n">Type</span><span class="p">[</span><span class="n">StochasticSampler</span><span class="p">]</span> <span class="o">=</span> <span class="n">PermutationSampler</span><span class="p">,</span>
+</span><span id="__span-0-443"><a id="__codelineno-0-443" name="__codelineno-0-443"></a>    <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+</span><span id="__span-0-444"><a id="__codelineno-0-444" name="__codelineno-0-444"></a>    <span class="n">n_jobs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+</span><span id="__span-0-445"><a id="__codelineno-0-445" name="__codelineno-0-445"></a>    <span class="n">config</span><span class="p">:</span> <span class="n">ParallelConfig</span> <span class="o">=</span> <span class="n">ParallelConfig</span><span class="p">(),</span>
+</span><span id="__span-0-446"><a id="__codelineno-0-446" name="__codelineno-0-446"></a>    <span class="n">progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+</span><span id="__span-0-447"><a id="__codelineno-0-447" name="__codelineno-0-447"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-448"><a id="__codelineno-0-448" name="__codelineno-0-448"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">:</span>
+</span><span id="__span-0-449"><a id="__codelineno-0-449" name="__codelineno-0-449"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Computes Beta Shapley values for a given utility function.</span>
+</span><span id="__span-0-450"><a id="__codelineno-0-450" name="__codelineno-0-450"></a>
+</span><span id="__span-0-451"><a id="__codelineno-0-451" name="__codelineno-0-451"></a><span class="sd">    This is a convenience wrapper for</span>
+</span><span id="__span-0-452"><a id="__codelineno-0-452" name="__codelineno-0-452"></a><span class="sd">    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]</span>
+</span><span id="__span-0-453"><a id="__codelineno-0-453" name="__codelineno-0-453"></a><span class="sd">    with the Beta Shapley coefficient.</span>
+</span><span id="__span-0-454"><a id="__codelineno-0-454" name="__codelineno-0-454"></a>
+</span><span id="__span-0-455"><a id="__codelineno-0-455" name="__codelineno-0-455"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-456"><a id="__codelineno-0-456" name="__codelineno-0-456"></a><span class="sd">        u: Utility object with model, data, and scoring function.</span>
+</span><span id="__span-0-457"><a id="__codelineno-0-457" name="__codelineno-0-457"></a><span class="sd">        alpha: Alpha parameter of the Beta distribution.</span>
+</span><span id="__span-0-458"><a id="__codelineno-0-458" name="__codelineno-0-458"></a><span class="sd">        beta: Beta parameter of the Beta distribution.</span>
+</span><span id="__span-0-459"><a id="__codelineno-0-459" name="__codelineno-0-459"></a><span class="sd">        done: Stopping criterion.</span>
+</span><span id="__span-0-460"><a id="__codelineno-0-460" name="__codelineno-0-460"></a><span class="sd">        sampler_t: The sampler type to use. See the</span>
+</span><span id="__span-0-461"><a id="__codelineno-0-461" name="__codelineno-0-461"></a><span class="sd">            [sampler][pydvl.value.sampler] module for a list.</span>
+</span><span id="__span-0-462"><a id="__codelineno-0-462" name="__codelineno-0-462"></a><span class="sd">        batch_size: Number of marginal evaluations per (parallelized) task.</span>
+</span><span id="__span-0-463"><a id="__codelineno-0-463" name="__codelineno-0-463"></a><span class="sd">        n_jobs: Number of parallel jobs to use.</span>
+</span><span id="__span-0-464"><a id="__codelineno-0-464" name="__codelineno-0-464"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
+</span><span id="__span-0-465"><a id="__codelineno-0-465" name="__codelineno-0-465"></a><span class="sd">        config: Object configuring parallel computation, with cluster address, number of</span>
+</span><span id="__span-0-466"><a id="__codelineno-0-466" name="__codelineno-0-466"></a><span class="sd">            cpus, etc.</span>
+</span><span id="__span-0-467"><a id="__codelineno-0-467" name="__codelineno-0-467"></a><span class="sd">        progress: Whether to display a progress bar.</span>
+</span><span id="__span-0-468"><a id="__codelineno-0-468" name="__codelineno-0-468"></a>
+</span><span id="__span-0-469"><a id="__codelineno-0-469" name="__codelineno-0-469"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-470"><a id="__codelineno-0-470" name="__codelineno-0-470"></a><span class="sd">        Object with the results.</span>
+</span><span id="__span-0-471"><a id="__codelineno-0-471" name="__codelineno-0-471"></a>
+</span><span id="__span-0-472"><a id="__codelineno-0-472" name="__codelineno-0-472"></a><span class="sd">    !!! warning &quot;Deprecation notice&quot;</span>
+</span><span id="__span-0-473"><a id="__codelineno-0-473" name="__codelineno-0-473"></a><span class="sd">        Parameter `batch_size` is for experimental use and will be removed in</span>
+</span><span id="__span-0-474"><a id="__codelineno-0-474" name="__codelineno-0-474"></a><span class="sd">        future versions.</span>
+</span><span id="__span-0-475"><a id="__codelineno-0-475" name="__codelineno-0-475"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-476"><a id="__codelineno-0-476" name="__codelineno-0-476"></a>    <span class="c1"># HACK: cannot infer return type because of useless IndexT, NameT</span>
+</span><span id="__span-0-477"><a id="__codelineno-0-477" name="__codelineno-0-477"></a>    <span class="k">return</span> <span class="n">compute_generic_semivalues</span><span class="p">(</span>  <span class="c1"># type: ignore</span>
+</span><span id="__span-0-478"><a id="__codelineno-0-478" name="__codelineno-0-478"></a>        <span class="n">sampler_t</span><span class="p">(</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">),</span>
+</span><span id="__span-0-479"><a id="__codelineno-0-479" name="__codelineno-0-479"></a>        <span class="n">u</span><span class="p">,</span>
+</span><span id="__span-0-480"><a id="__codelineno-0-480" name="__codelineno-0-480"></a>        <span class="n">beta_coefficient</span><span class="p">(</span><span class="n">alpha</span><span class="p">,</span> <span class="n">beta</span><span class="p">),</span>
+</span><span id="__span-0-481"><a id="__codelineno-0-481" name="__codelineno-0-481"></a>        <span class="n">done</span><span class="p">,</span>
+</span><span id="__span-0-482"><a id="__codelineno-0-482" name="__codelineno-0-482"></a>        <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
+</span><span id="__span-0-483"><a id="__codelineno-0-483" name="__codelineno-0-483"></a>        <span class="n">n_jobs</span><span class="o">=</span><span class="n">n_jobs</span><span class="p">,</span>
+</span><span id="__span-0-484"><a id="__codelineno-0-484" name="__codelineno-0-484"></a>        <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span>
+</span><span id="__span-0-485"><a id="__codelineno-0-485" name="__codelineno-0-485"></a>        <span class="n">progress</span><span class="o">=</span><span class="n">progress</span><span class="p">,</span>
+</span><span id="__span-0-486"><a id="__codelineno-0-486" name="__codelineno-0-486"></a>    <span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -4092,7 +4378,7 @@ <h2 id="pydvl.value.semivalues.compute_beta_shapley_semivalues" class="doc doc-h
 
 
 <h2 id="pydvl.value.semivalues.compute_semivalues" class="doc doc-heading">
-          <code class="highlight language-python"><span class="n">compute_semivalues</span><span class="p">(</span><span class="n">u</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">done</span><span class="o">=</span><span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span> <span class="n">mode</span><span class="o">=</span><span class="n">SemiValueMode</span><span class="o">.</span><span class="n">Shapley</span><span class="p">,</span> <span class="n">sampler_t</span><span class="o">=</span><span class="n">PermutationSampler</span><span class="p">,</span> <span class="n">n_jobs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></code>
+          <code class="highlight language-python"><span class="n">compute_semivalues</span><span class="p">(</span><span class="n">u</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">done</span><span class="o">=</span><span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span> <span class="n">mode</span><span class="o">=</span><span class="n">SemiValueMode</span><span class="o">.</span><span class="n">Shapley</span><span class="p">,</span> <span class="n">sampler_t</span><span class="o">=</span><span class="n">PermutationSampler</span><span class="p">[</span><span class="n">IndexT</span><span class="p">],</span> <span class="n">batch_size</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">n_jobs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></code>
 
 <a href="#pydvl.value.semivalues.compute_semivalues" class="headerlink" title="Permanent link">&para;</a></h2>
 
@@ -4202,11 +4488,29 @@ <h2 id="pydvl.value.semivalues.compute_semivalues" class="doc doc-heading">
             <p>
                 <span class="doc-param-annotation">
                   <b>TYPE:</b>
-                    <code><a class="autorefs autorefs-external" title="typing.Type" href="https://docs.python.org/3/library/typing.html#typing.Type">Type</a>[<span title="pydvl.value.sampler.StochasticSampler">StochasticSampler</span>]</code>
+                    <code><a class="autorefs autorefs-external" title="typing.Type" href="https://docs.python.org/3/library/typing.html#typing.Type">Type</a>[<span title="pydvl.value.sampler.StochasticSampler">StochasticSampler</span>[<span title="pydvl.utils.types.IndexT">IndexT</span>]]</code>
                 </span>
                 <span class="doc-param-default">
                   <b>DEFAULT:</b> 
-                    <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PermutationSampler" href="../sampler/#pydvl.value.sampler.PermutationSampler">PermutationSampler</a></code>
+                    <code><a class="autorefs autorefs-internal" title="pydvl.value.sampler.PermutationSampler" href="../sampler/#pydvl.value.sampler.PermutationSampler">PermutationSampler</a>[<span title="pydvl.utils.types.IndexT">IndexT</span>]</code>
+                </span>
+            </p>
+          </td>
+        </tr>
+        <tr>
+          <td><code>batch_size</code></td>
+          <td class="doc-param-details">
+            <div class="doc-md-description">
+              <p>Number of marginal evaluations per (parallelized) task.</p>
+            </div>
+            <p>
+                <span class="doc-param-annotation">
+                  <b>TYPE:</b>
+                    <code><a class="autorefs autorefs-external" href="https://docs.python.org/3/library/functions.html#int">int</a></code>
+                </span>
+                <span class="doc-param-default">
+                  <b>DEFAULT:</b> 
+                    <code>1</code>
                 </span>
             </p>
           </td>
@@ -4289,162 +4593,185 @@ <h2 id="pydvl.value.semivalues.compute_semivalues" class="doc doc-heading">
         </tr>
     </tbody>
   </table>
+      <div class="admonition warning">
+<p class="admonition-title">Deprecation notice</p>
+<p>Parameter <code>batch_size</code> is for experimental use and will be removed in
+future versions.</p>
+</div>
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/semivalues.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-411">411</a></span>
-<span class="normal"><a href="#__codelineno-0-412">412</a></span>
-<span class="normal"><a href="#__codelineno-0-413">413</a></span>
-<span class="normal"><a href="#__codelineno-0-414">414</a></span>
-<span class="normal"><a href="#__codelineno-0-415">415</a></span>
-<span class="normal"><a href="#__codelineno-0-416">416</a></span>
-<span class="normal"><a href="#__codelineno-0-417">417</a></span>
-<span class="normal"><a href="#__codelineno-0-418">418</a></span>
-<span class="normal"><a href="#__codelineno-0-419">419</a></span>
-<span class="normal"><a href="#__codelineno-0-420">420</a></span>
-<span class="normal"><a href="#__codelineno-0-421">421</a></span>
-<span class="normal"><a href="#__codelineno-0-422">422</a></span>
-<span class="normal"><a href="#__codelineno-0-423">423</a></span>
-<span class="normal"><a href="#__codelineno-0-424">424</a></span>
-<span class="normal"><a href="#__codelineno-0-425">425</a></span>
-<span class="normal"><a href="#__codelineno-0-426">426</a></span>
-<span class="normal"><a href="#__codelineno-0-427">427</a></span>
-<span class="normal"><a href="#__codelineno-0-428">428</a></span>
-<span class="normal"><a href="#__codelineno-0-429">429</a></span>
-<span class="normal"><a href="#__codelineno-0-430">430</a></span>
-<span class="normal"><a href="#__codelineno-0-431">431</a></span>
-<span class="normal"><a href="#__codelineno-0-432">432</a></span>
-<span class="normal"><a href="#__codelineno-0-433">433</a></span>
-<span class="normal"><a href="#__codelineno-0-434">434</a></span>
-<span class="normal"><a href="#__codelineno-0-435">435</a></span>
-<span class="normal"><a href="#__codelineno-0-436">436</a></span>
-<span class="normal"><a href="#__codelineno-0-437">437</a></span>
-<span class="normal"><a href="#__codelineno-0-438">438</a></span>
-<span class="normal"><a href="#__codelineno-0-439">439</a></span>
-<span class="normal"><a href="#__codelineno-0-440">440</a></span>
-<span class="normal"><a href="#__codelineno-0-441">441</a></span>
-<span class="normal"><a href="#__codelineno-0-442">442</a></span>
-<span class="normal"><a href="#__codelineno-0-443">443</a></span>
-<span class="normal"><a href="#__codelineno-0-444">444</a></span>
-<span class="normal"><a href="#__codelineno-0-445">445</a></span>
-<span class="normal"><a href="#__codelineno-0-446">446</a></span>
-<span class="normal"><a href="#__codelineno-0-447">447</a></span>
-<span class="normal"><a href="#__codelineno-0-448">448</a></span>
-<span class="normal"><a href="#__codelineno-0-449">449</a></span>
-<span class="normal"><a href="#__codelineno-0-450">450</a></span>
-<span class="normal"><a href="#__codelineno-0-451">451</a></span>
-<span class="normal"><a href="#__codelineno-0-452">452</a></span>
-<span class="normal"><a href="#__codelineno-0-453">453</a></span>
-<span class="normal"><a href="#__codelineno-0-454">454</a></span>
-<span class="normal"><a href="#__codelineno-0-455">455</a></span>
-<span class="normal"><a href="#__codelineno-0-456">456</a></span>
-<span class="normal"><a href="#__codelineno-0-457">457</a></span>
-<span class="normal"><a href="#__codelineno-0-458">458</a></span>
-<span class="normal"><a href="#__codelineno-0-459">459</a></span>
-<span class="normal"><a href="#__codelineno-0-460">460</a></span>
-<span class="normal"><a href="#__codelineno-0-461">461</a></span>
-<span class="normal"><a href="#__codelineno-0-462">462</a></span>
-<span class="normal"><a href="#__codelineno-0-463">463</a></span>
-<span class="normal"><a href="#__codelineno-0-464">464</a></span>
-<span class="normal"><a href="#__codelineno-0-465">465</a></span>
-<span class="normal"><a href="#__codelineno-0-466">466</a></span>
-<span class="normal"><a href="#__codelineno-0-467">467</a></span>
-<span class="normal"><a href="#__codelineno-0-468">468</a></span>
-<span class="normal"><a href="#__codelineno-0-469">469</a></span>
-<span class="normal"><a href="#__codelineno-0-470">470</a></span>
-<span class="normal"><a href="#__codelineno-0-471">471</a></span>
-<span class="normal"><a href="#__codelineno-0-472">472</a></span>
-<span class="normal"><a href="#__codelineno-0-473">473</a></span>
-<span class="normal"><a href="#__codelineno-0-474">474</a></span>
-<span class="normal"><a href="#__codelineno-0-475">475</a></span>
-<span class="normal"><a href="#__codelineno-0-476">476</a></span>
-<span class="normal"><a href="#__codelineno-0-477">477</a></span>
-<span class="normal"><a href="#__codelineno-0-478">478</a></span>
-<span class="normal"><a href="#__codelineno-0-479">479</a></span>
-<span class="normal"><a href="#__codelineno-0-480">480</a></span>
-<span class="normal"><a href="#__codelineno-0-481">481</a></span>
-<span class="normal"><a href="#__codelineno-0-482">482</a></span>
-<span class="normal"><a href="#__codelineno-0-483">483</a></span>
-<span class="normal"><a href="#__codelineno-0-484">484</a></span>
-<span class="normal"><a href="#__codelineno-0-485">485</a></span>
-<span class="normal"><a href="#__codelineno-0-486">486</a></span>
-<span class="normal"><a href="#__codelineno-0-487">487</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-411"><a id="__codelineno-0-411" name="__codelineno-0-411"></a><span class="nd">@deprecated</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">deprecated_in</span><span class="o">=</span><span class="s2">&quot;0.7.0&quot;</span><span class="p">,</span> <span class="n">remove_in</span><span class="o">=</span><span class="s2">&quot;0.8.0&quot;</span><span class="p">)</span>
-</span><span id="__span-0-412"><a id="__codelineno-0-412" name="__codelineno-0-412"></a><span class="k">def</span> <span class="nf">compute_semivalues</span><span class="p">(</span>
-</span><span id="__span-0-413"><a id="__codelineno-0-413" name="__codelineno-0-413"></a>    <span class="n">u</span><span class="p">:</span> <span class="n">Utility</span><span class="p">,</span>
-</span><span id="__span-0-414"><a id="__codelineno-0-414" name="__codelineno-0-414"></a>    <span class="o">*</span><span class="p">,</span>
-</span><span id="__span-0-415"><a id="__codelineno-0-415" name="__codelineno-0-415"></a>    <span class="n">done</span><span class="p">:</span> <span class="n">StoppingCriterion</span> <span class="o">=</span> <span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span>
-</span><span id="__span-0-416"><a id="__codelineno-0-416" name="__codelineno-0-416"></a>    <span class="n">mode</span><span class="p">:</span> <span class="n">SemiValueMode</span> <span class="o">=</span> <span class="n">SemiValueMode</span><span class="o">.</span><span class="n">Shapley</span><span class="p">,</span>
-</span><span id="__span-0-417"><a id="__codelineno-0-417" name="__codelineno-0-417"></a>    <span class="n">sampler_t</span><span class="p">:</span> <span class="n">Type</span><span class="p">[</span><span class="n">StochasticSampler</span><span class="p">]</span> <span class="o">=</span> <span class="n">PermutationSampler</span><span class="p">,</span>
-</span><span id="__span-0-418"><a id="__codelineno-0-418" name="__codelineno-0-418"></a>    <span class="n">n_jobs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-</span><span id="__span-0-419"><a id="__codelineno-0-419" name="__codelineno-0-419"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-420"><a id="__codelineno-0-420" name="__codelineno-0-420"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-</span><span id="__span-0-421"><a id="__codelineno-0-421" name="__codelineno-0-421"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">:</span>
-</span><span id="__span-0-422"><a id="__codelineno-0-422" name="__codelineno-0-422"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Convenience entry point for most common semi-value computations.</span>
-</span><span id="__span-0-423"><a id="__codelineno-0-423" name="__codelineno-0-423"></a>
-</span><span id="__span-0-424"><a id="__codelineno-0-424" name="__codelineno-0-424"></a><span class="sd">    !!! warning &quot;Deprecation warning&quot;</span>
-</span><span id="__span-0-425"><a id="__codelineno-0-425" name="__codelineno-0-425"></a><span class="sd">        This method is deprecated and will be replaced in 0.8.0 by the more</span>
-</span><span id="__span-0-426"><a id="__codelineno-0-426" name="__codelineno-0-426"></a><span class="sd">        general implementation of</span>
-</span><span id="__span-0-427"><a id="__codelineno-0-427" name="__codelineno-0-427"></a><span class="sd">        [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].</span>
-</span><span id="__span-0-428"><a id="__codelineno-0-428" name="__codelineno-0-428"></a><span class="sd">        Use</span>
-</span><span id="__span-0-429"><a id="__codelineno-0-429" name="__codelineno-0-429"></a><span class="sd">        [compute_shapley_semivalues][pydvl.value.semivalues.compute_shapley_semivalues],</span>
-</span><span id="__span-0-430"><a id="__codelineno-0-430" name="__codelineno-0-430"></a><span class="sd">        [compute_banzhaf_semivalues][pydvl.value.semivalues.compute_banzhaf_semivalues],</span>
-</span><span id="__span-0-431"><a id="__codelineno-0-431" name="__codelineno-0-431"></a><span class="sd">        or</span>
-</span><span id="__span-0-432"><a id="__codelineno-0-432" name="__codelineno-0-432"></a><span class="sd">        [compute_beta_shapley_semivalues][pydvl.value.semivalues.compute_beta_shapley_semivalues]</span>
-</span><span id="__span-0-433"><a id="__codelineno-0-433" name="__codelineno-0-433"></a><span class="sd">        instead.</span>
-</span><span id="__span-0-434"><a id="__codelineno-0-434" name="__codelineno-0-434"></a>
-</span><span id="__span-0-435"><a id="__codelineno-0-435" name="__codelineno-0-435"></a><span class="sd">    The modes supported with this interface are the following. For greater</span>
-</span><span id="__span-0-436"><a id="__codelineno-0-436" name="__codelineno-0-436"></a><span class="sd">    flexibility use</span>
-</span><span id="__span-0-437"><a id="__codelineno-0-437" name="__codelineno-0-437"></a><span class="sd">    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]</span>
-</span><span id="__span-0-438"><a id="__codelineno-0-438" name="__codelineno-0-438"></a><span class="sd">    directly.</span>
-</span><span id="__span-0-439"><a id="__codelineno-0-439" name="__codelineno-0-439"></a>
-</span><span id="__span-0-440"><a id="__codelineno-0-440" name="__codelineno-0-440"></a><span class="sd">    - [SemiValueMode.Shapley][pydvl.value.semivalues.SemiValueMode]:</span>
-</span><span id="__span-0-441"><a id="__codelineno-0-441" name="__codelineno-0-441"></a><span class="sd">      Shapley values.</span>
-</span><span id="__span-0-442"><a id="__codelineno-0-442" name="__codelineno-0-442"></a><span class="sd">    - [SemiValueMode.BetaShapley][pydvl.value.semivalues.SemiValueMode.BetaShapley]:</span>
-</span><span id="__span-0-443"><a id="__codelineno-0-443" name="__codelineno-0-443"></a><span class="sd">      Implements the Beta Shapley semi-value as introduced in</span>
-</span><span id="__span-0-444"><a id="__codelineno-0-444" name="__codelineno-0-444"></a><span class="sd">      (Kwon and Zou, 2022)&lt;sup&gt;&lt;a href=&quot;#kwon_beta_2022&quot;&gt;1&lt;/a&gt;&lt;/sup&gt;.</span>
-</span><span id="__span-0-445"><a id="__codelineno-0-445" name="__codelineno-0-445"></a><span class="sd">      Pass additional keyword arguments `alpha` and `beta` to set the</span>
-</span><span id="__span-0-446"><a id="__codelineno-0-446" name="__codelineno-0-446"></a><span class="sd">      parameters of the Beta distribution (both default to 1).</span>
-</span><span id="__span-0-447"><a id="__codelineno-0-447" name="__codelineno-0-447"></a><span class="sd">    - [SemiValueMode.Banzhaf][SemiValueMode.Banzhaf]: Implements the Banzhaf</span>
-</span><span id="__span-0-448"><a id="__codelineno-0-448" name="__codelineno-0-448"></a><span class="sd">      semi-value as introduced in (Wang and Jia, 2022)&lt;sup&gt;&lt;a href=&quot;#wang_data_2022&quot;&gt;1&lt;/a&gt;&lt;/sup&gt;.</span>
-</span><span id="__span-0-449"><a id="__codelineno-0-449" name="__codelineno-0-449"></a>
-</span><span id="__span-0-450"><a id="__codelineno-0-450" name="__codelineno-0-450"></a><span class="sd">    See [[data-valuation]] for an overview of valuation.</span>
-</span><span id="__span-0-451"><a id="__codelineno-0-451" name="__codelineno-0-451"></a><span class="sd">    - [SemiValueMode.Banzhaf][pydvl.value.semivalues.SemiValueMode]: Implements</span>
-</span><span id="__span-0-452"><a id="__codelineno-0-452" name="__codelineno-0-452"></a><span class="sd">      the Banzhaf semi-value as introduced in [@wang_data_2022].</span>
-</span><span id="__span-0-453"><a id="__codelineno-0-453" name="__codelineno-0-453"></a>
-</span><span id="__span-0-454"><a id="__codelineno-0-454" name="__codelineno-0-454"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-455"><a id="__codelineno-0-455" name="__codelineno-0-455"></a><span class="sd">        u: Utility object with model, data, and scoring function.</span>
-</span><span id="__span-0-456"><a id="__codelineno-0-456" name="__codelineno-0-456"></a><span class="sd">        done: Stopping criterion.</span>
-</span><span id="__span-0-457"><a id="__codelineno-0-457" name="__codelineno-0-457"></a><span class="sd">        mode: The semi-value mode to use. See</span>
-</span><span id="__span-0-458"><a id="__codelineno-0-458" name="__codelineno-0-458"></a><span class="sd">            [SemiValueMode][pydvl.value.semivalues.SemiValueMode] for a list.</span>
-</span><span id="__span-0-459"><a id="__codelineno-0-459" name="__codelineno-0-459"></a><span class="sd">        sampler_t: The sampler type to use. See [sampler][pydvl.value.sampler]</span>
-</span><span id="__span-0-460"><a id="__codelineno-0-460" name="__codelineno-0-460"></a><span class="sd">            for a list.</span>
-</span><span id="__span-0-461"><a id="__codelineno-0-461" name="__codelineno-0-461"></a><span class="sd">        n_jobs: Number of parallel jobs to use.</span>
-</span><span id="__span-0-462"><a id="__codelineno-0-462" name="__codelineno-0-462"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
-</span><span id="__span-0-463"><a id="__codelineno-0-463" name="__codelineno-0-463"></a><span class="sd">        kwargs: Additional keyword arguments passed to</span>
-</span><span id="__span-0-464"><a id="__codelineno-0-464" name="__codelineno-0-464"></a><span class="sd">            [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].</span>
-</span><span id="__span-0-465"><a id="__codelineno-0-465" name="__codelineno-0-465"></a>
-</span><span id="__span-0-466"><a id="__codelineno-0-466" name="__codelineno-0-466"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-467"><a id="__codelineno-0-467" name="__codelineno-0-467"></a><span class="sd">        Object with the results.</span>
-</span><span id="__span-0-468"><a id="__codelineno-0-468" name="__codelineno-0-468"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-469"><a id="__codelineno-0-469" name="__codelineno-0-469"></a>    <span class="k">if</span> <span class="n">mode</span> <span class="o">==</span> <span class="n">SemiValueMode</span><span class="o">.</span><span class="n">Shapley</span><span class="p">:</span>
-</span><span id="__span-0-470"><a id="__codelineno-0-470" name="__codelineno-0-470"></a>        <span class="n">coefficient</span> <span class="o">=</span> <span class="n">shapley_coefficient</span>
-</span><span id="__span-0-471"><a id="__codelineno-0-471" name="__codelineno-0-471"></a>    <span class="k">elif</span> <span class="n">mode</span> <span class="o">==</span> <span class="n">SemiValueMode</span><span class="o">.</span><span class="n">BetaShapley</span><span class="p">:</span>
-</span><span id="__span-0-472"><a id="__codelineno-0-472" name="__codelineno-0-472"></a>        <span class="n">alpha</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">&quot;alpha&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-</span><span id="__span-0-473"><a id="__codelineno-0-473" name="__codelineno-0-473"></a>        <span class="n">beta</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-</span><span id="__span-0-474"><a id="__codelineno-0-474" name="__codelineno-0-474"></a>        <span class="n">coefficient</span> <span class="o">=</span> <span class="n">beta_coefficient</span><span class="p">(</span><span class="n">alpha</span><span class="p">,</span> <span class="n">beta</span><span class="p">)</span>
-</span><span id="__span-0-475"><a id="__codelineno-0-475" name="__codelineno-0-475"></a>    <span class="k">elif</span> <span class="n">mode</span> <span class="o">==</span> <span class="n">SemiValueMode</span><span class="o">.</span><span class="n">Banzhaf</span><span class="p">:</span>
-</span><span id="__span-0-476"><a id="__codelineno-0-476" name="__codelineno-0-476"></a>        <span class="n">coefficient</span> <span class="o">=</span> <span class="n">banzhaf_coefficient</span>
-</span><span id="__span-0-477"><a id="__codelineno-0-477" name="__codelineno-0-477"></a>    <span class="k">else</span><span class="p">:</span>
-</span><span id="__span-0-478"><a id="__codelineno-0-478" name="__codelineno-0-478"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Unknown mode </span><span class="si">{</span><span class="n">mode</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
-</span><span id="__span-0-479"><a id="__codelineno-0-479" name="__codelineno-0-479"></a>    <span class="n">coefficient</span> <span class="o">=</span> <span class="n">cast</span><span class="p">(</span><span class="n">SVCoefficient</span><span class="p">,</span> <span class="n">coefficient</span><span class="p">)</span>
-</span><span id="__span-0-480"><a id="__codelineno-0-480" name="__codelineno-0-480"></a>    <span class="k">return</span> <span class="n">compute_generic_semivalues</span><span class="p">(</span>
-</span><span id="__span-0-481"><a id="__codelineno-0-481" name="__codelineno-0-481"></a>        <span class="n">sampler_t</span><span class="p">(</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">),</span>
-</span><span id="__span-0-482"><a id="__codelineno-0-482" name="__codelineno-0-482"></a>        <span class="n">u</span><span class="p">,</span>
-</span><span id="__span-0-483"><a id="__codelineno-0-483" name="__codelineno-0-483"></a>        <span class="n">coefficient</span><span class="p">,</span>
-</span><span id="__span-0-484"><a id="__codelineno-0-484" name="__codelineno-0-484"></a>        <span class="n">done</span><span class="p">,</span>
-</span><span id="__span-0-485"><a id="__codelineno-0-485" name="__codelineno-0-485"></a>        <span class="n">n_jobs</span><span class="o">=</span><span class="n">n_jobs</span><span class="p">,</span>
-</span><span id="__span-0-486"><a id="__codelineno-0-486" name="__codelineno-0-486"></a>        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-</span><span id="__span-0-487"><a id="__codelineno-0-487" name="__codelineno-0-487"></a>    <span class="p">)</span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-503">503</a></span>
+<span class="normal"><a href="#__codelineno-0-504">504</a></span>
+<span class="normal"><a href="#__codelineno-0-505">505</a></span>
+<span class="normal"><a href="#__codelineno-0-506">506</a></span>
+<span class="normal"><a href="#__codelineno-0-507">507</a></span>
+<span class="normal"><a href="#__codelineno-0-508">508</a></span>
+<span class="normal"><a href="#__codelineno-0-509">509</a></span>
+<span class="normal"><a href="#__codelineno-0-510">510</a></span>
+<span class="normal"><a href="#__codelineno-0-511">511</a></span>
+<span class="normal"><a href="#__codelineno-0-512">512</a></span>
+<span class="normal"><a href="#__codelineno-0-513">513</a></span>
+<span class="normal"><a href="#__codelineno-0-514">514</a></span>
+<span class="normal"><a href="#__codelineno-0-515">515</a></span>
+<span class="normal"><a href="#__codelineno-0-516">516</a></span>
+<span class="normal"><a href="#__codelineno-0-517">517</a></span>
+<span class="normal"><a href="#__codelineno-0-518">518</a></span>
+<span class="normal"><a href="#__codelineno-0-519">519</a></span>
+<span class="normal"><a href="#__codelineno-0-520">520</a></span>
+<span class="normal"><a href="#__codelineno-0-521">521</a></span>
+<span class="normal"><a href="#__codelineno-0-522">522</a></span>
+<span class="normal"><a href="#__codelineno-0-523">523</a></span>
+<span class="normal"><a href="#__codelineno-0-524">524</a></span>
+<span class="normal"><a href="#__codelineno-0-525">525</a></span>
+<span class="normal"><a href="#__codelineno-0-526">526</a></span>
+<span class="normal"><a href="#__codelineno-0-527">527</a></span>
+<span class="normal"><a href="#__codelineno-0-528">528</a></span>
+<span class="normal"><a href="#__codelineno-0-529">529</a></span>
+<span class="normal"><a href="#__codelineno-0-530">530</a></span>
+<span class="normal"><a href="#__codelineno-0-531">531</a></span>
+<span class="normal"><a href="#__codelineno-0-532">532</a></span>
+<span class="normal"><a href="#__codelineno-0-533">533</a></span>
+<span class="normal"><a href="#__codelineno-0-534">534</a></span>
+<span class="normal"><a href="#__codelineno-0-535">535</a></span>
+<span class="normal"><a href="#__codelineno-0-536">536</a></span>
+<span class="normal"><a href="#__codelineno-0-537">537</a></span>
+<span class="normal"><a href="#__codelineno-0-538">538</a></span>
+<span class="normal"><a href="#__codelineno-0-539">539</a></span>
+<span class="normal"><a href="#__codelineno-0-540">540</a></span>
+<span class="normal"><a href="#__codelineno-0-541">541</a></span>
+<span class="normal"><a href="#__codelineno-0-542">542</a></span>
+<span class="normal"><a href="#__codelineno-0-543">543</a></span>
+<span class="normal"><a href="#__codelineno-0-544">544</a></span>
+<span class="normal"><a href="#__codelineno-0-545">545</a></span>
+<span class="normal"><a href="#__codelineno-0-546">546</a></span>
+<span class="normal"><a href="#__codelineno-0-547">547</a></span>
+<span class="normal"><a href="#__codelineno-0-548">548</a></span>
+<span class="normal"><a href="#__codelineno-0-549">549</a></span>
+<span class="normal"><a href="#__codelineno-0-550">550</a></span>
+<span class="normal"><a href="#__codelineno-0-551">551</a></span>
+<span class="normal"><a href="#__codelineno-0-552">552</a></span>
+<span class="normal"><a href="#__codelineno-0-553">553</a></span>
+<span class="normal"><a href="#__codelineno-0-554">554</a></span>
+<span class="normal"><a href="#__codelineno-0-555">555</a></span>
+<span class="normal"><a href="#__codelineno-0-556">556</a></span>
+<span class="normal"><a href="#__codelineno-0-557">557</a></span>
+<span class="normal"><a href="#__codelineno-0-558">558</a></span>
+<span class="normal"><a href="#__codelineno-0-559">559</a></span>
+<span class="normal"><a href="#__codelineno-0-560">560</a></span>
+<span class="normal"><a href="#__codelineno-0-561">561</a></span>
+<span class="normal"><a href="#__codelineno-0-562">562</a></span>
+<span class="normal"><a href="#__codelineno-0-563">563</a></span>
+<span class="normal"><a href="#__codelineno-0-564">564</a></span>
+<span class="normal"><a href="#__codelineno-0-565">565</a></span>
+<span class="normal"><a href="#__codelineno-0-566">566</a></span>
+<span class="normal"><a href="#__codelineno-0-567">567</a></span>
+<span class="normal"><a href="#__codelineno-0-568">568</a></span>
+<span class="normal"><a href="#__codelineno-0-569">569</a></span>
+<span class="normal"><a href="#__codelineno-0-570">570</a></span>
+<span class="normal"><a href="#__codelineno-0-571">571</a></span>
+<span class="normal"><a href="#__codelineno-0-572">572</a></span>
+<span class="normal"><a href="#__codelineno-0-573">573</a></span>
+<span class="normal"><a href="#__codelineno-0-574">574</a></span>
+<span class="normal"><a href="#__codelineno-0-575">575</a></span>
+<span class="normal"><a href="#__codelineno-0-576">576</a></span>
+<span class="normal"><a href="#__codelineno-0-577">577</a></span>
+<span class="normal"><a href="#__codelineno-0-578">578</a></span>
+<span class="normal"><a href="#__codelineno-0-579">579</a></span>
+<span class="normal"><a href="#__codelineno-0-580">580</a></span>
+<span class="normal"><a href="#__codelineno-0-581">581</a></span>
+<span class="normal"><a href="#__codelineno-0-582">582</a></span>
+<span class="normal"><a href="#__codelineno-0-583">583</a></span>
+<span class="normal"><a href="#__codelineno-0-584">584</a></span>
+<span class="normal"><a href="#__codelineno-0-585">585</a></span>
+<span class="normal"><a href="#__codelineno-0-586">586</a></span>
+<span class="normal"><a href="#__codelineno-0-587">587</a></span>
+<span class="normal"><a href="#__codelineno-0-588">588</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-503"><a id="__codelineno-0-503" name="__codelineno-0-503"></a><span class="nd">@deprecated</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">deprecated_in</span><span class="o">=</span><span class="s2">&quot;0.7.0&quot;</span><span class="p">,</span> <span class="n">remove_in</span><span class="o">=</span><span class="s2">&quot;0.8.0&quot;</span><span class="p">)</span>
+</span><span id="__span-0-504"><a id="__codelineno-0-504" name="__codelineno-0-504"></a><span class="k">def</span> <span class="nf">compute_semivalues</span><span class="p">(</span>
+</span><span id="__span-0-505"><a id="__codelineno-0-505" name="__codelineno-0-505"></a>    <span class="n">u</span><span class="p">:</span> <span class="n">Utility</span><span class="p">,</span>
+</span><span id="__span-0-506"><a id="__codelineno-0-506" name="__codelineno-0-506"></a>    <span class="o">*</span><span class="p">,</span>
+</span><span id="__span-0-507"><a id="__codelineno-0-507" name="__codelineno-0-507"></a>    <span class="n">done</span><span class="p">:</span> <span class="n">StoppingCriterion</span> <span class="o">=</span> <span class="n">MaxUpdates</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span>
+</span><span id="__span-0-508"><a id="__codelineno-0-508" name="__codelineno-0-508"></a>    <span class="n">mode</span><span class="p">:</span> <span class="n">SemiValueMode</span> <span class="o">=</span> <span class="n">SemiValueMode</span><span class="o">.</span><span class="n">Shapley</span><span class="p">,</span>
+</span><span id="__span-0-509"><a id="__codelineno-0-509" name="__codelineno-0-509"></a>    <span class="n">sampler_t</span><span class="p">:</span> <span class="n">Type</span><span class="p">[</span><span class="n">StochasticSampler</span><span class="p">[</span><span class="n">IndexT</span><span class="p">]]</span> <span class="o">=</span> <span class="n">PermutationSampler</span><span class="p">[</span><span class="n">IndexT</span><span class="p">],</span>
+</span><span id="__span-0-510"><a id="__codelineno-0-510" name="__codelineno-0-510"></a>    <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+</span><span id="__span-0-511"><a id="__codelineno-0-511" name="__codelineno-0-511"></a>    <span class="n">n_jobs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+</span><span id="__span-0-512"><a id="__codelineno-0-512" name="__codelineno-0-512"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-513"><a id="__codelineno-0-513" name="__codelineno-0-513"></a>    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+</span><span id="__span-0-514"><a id="__codelineno-0-514" name="__codelineno-0-514"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">:</span>
+</span><span id="__span-0-515"><a id="__codelineno-0-515" name="__codelineno-0-515"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Convenience entry point for most common semi-value computations.</span>
+</span><span id="__span-0-516"><a id="__codelineno-0-516" name="__codelineno-0-516"></a>
+</span><span id="__span-0-517"><a id="__codelineno-0-517" name="__codelineno-0-517"></a><span class="sd">    !!! warning &quot;Deprecation warning&quot;</span>
+</span><span id="__span-0-518"><a id="__codelineno-0-518" name="__codelineno-0-518"></a><span class="sd">        This method is deprecated and will be replaced in 0.8.0 by the more</span>
+</span><span id="__span-0-519"><a id="__codelineno-0-519" name="__codelineno-0-519"></a><span class="sd">        general implementation of</span>
+</span><span id="__span-0-520"><a id="__codelineno-0-520" name="__codelineno-0-520"></a><span class="sd">        [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].</span>
+</span><span id="__span-0-521"><a id="__codelineno-0-521" name="__codelineno-0-521"></a><span class="sd">        Use</span>
+</span><span id="__span-0-522"><a id="__codelineno-0-522" name="__codelineno-0-522"></a><span class="sd">        [compute_shapley_semivalues][pydvl.value.semivalues.compute_shapley_semivalues],</span>
+</span><span id="__span-0-523"><a id="__codelineno-0-523" name="__codelineno-0-523"></a><span class="sd">        [compute_banzhaf_semivalues][pydvl.value.semivalues.compute_banzhaf_semivalues],</span>
+</span><span id="__span-0-524"><a id="__codelineno-0-524" name="__codelineno-0-524"></a><span class="sd">        or</span>
+</span><span id="__span-0-525"><a id="__codelineno-0-525" name="__codelineno-0-525"></a><span class="sd">        [compute_beta_shapley_semivalues][pydvl.value.semivalues.compute_beta_shapley_semivalues]</span>
+</span><span id="__span-0-526"><a id="__codelineno-0-526" name="__codelineno-0-526"></a><span class="sd">        instead.</span>
+</span><span id="__span-0-527"><a id="__codelineno-0-527" name="__codelineno-0-527"></a>
+</span><span id="__span-0-528"><a id="__codelineno-0-528" name="__codelineno-0-528"></a><span class="sd">    The modes supported with this interface are the following. For greater</span>
+</span><span id="__span-0-529"><a id="__codelineno-0-529" name="__codelineno-0-529"></a><span class="sd">    flexibility use</span>
+</span><span id="__span-0-530"><a id="__codelineno-0-530" name="__codelineno-0-530"></a><span class="sd">    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]</span>
+</span><span id="__span-0-531"><a id="__codelineno-0-531" name="__codelineno-0-531"></a><span class="sd">    directly.</span>
+</span><span id="__span-0-532"><a id="__codelineno-0-532" name="__codelineno-0-532"></a>
+</span><span id="__span-0-533"><a id="__codelineno-0-533" name="__codelineno-0-533"></a><span class="sd">    - [SemiValueMode.Shapley][pydvl.value.semivalues.SemiValueMode]:</span>
+</span><span id="__span-0-534"><a id="__codelineno-0-534" name="__codelineno-0-534"></a><span class="sd">      Shapley values.</span>
+</span><span id="__span-0-535"><a id="__codelineno-0-535" name="__codelineno-0-535"></a><span class="sd">    - [SemiValueMode.BetaShapley][pydvl.value.semivalues.SemiValueMode.BetaShapley]:</span>
+</span><span id="__span-0-536"><a id="__codelineno-0-536" name="__codelineno-0-536"></a><span class="sd">      Implements the Beta Shapley semi-value as introduced in</span>
+</span><span id="__span-0-537"><a id="__codelineno-0-537" name="__codelineno-0-537"></a><span class="sd">      (Kwon and Zou, 2022)&lt;sup&gt;&lt;a href=&quot;#kwon_beta_2022&quot;&gt;1&lt;/a&gt;&lt;/sup&gt;.</span>
+</span><span id="__span-0-538"><a id="__codelineno-0-538" name="__codelineno-0-538"></a><span class="sd">      Pass additional keyword arguments `alpha` and `beta` to set the</span>
+</span><span id="__span-0-539"><a id="__codelineno-0-539" name="__codelineno-0-539"></a><span class="sd">      parameters of the Beta distribution (both default to 1).</span>
+</span><span id="__span-0-540"><a id="__codelineno-0-540" name="__codelineno-0-540"></a><span class="sd">    - [SemiValueMode.Banzhaf][SemiValueMode.Banzhaf]: Implements the Banzhaf</span>
+</span><span id="__span-0-541"><a id="__codelineno-0-541" name="__codelineno-0-541"></a><span class="sd">      semi-value as introduced in (Wang and Jia, 2022)&lt;sup&gt;&lt;a href=&quot;#wang_data_2022&quot;&gt;1&lt;/a&gt;&lt;/sup&gt;.</span>
+</span><span id="__span-0-542"><a id="__codelineno-0-542" name="__codelineno-0-542"></a>
+</span><span id="__span-0-543"><a id="__codelineno-0-543" name="__codelineno-0-543"></a><span class="sd">    See [[data-valuation]] for an overview of valuation.</span>
+</span><span id="__span-0-544"><a id="__codelineno-0-544" name="__codelineno-0-544"></a><span class="sd">    - [SemiValueMode.Banzhaf][pydvl.value.semivalues.SemiValueMode]: Implements</span>
+</span><span id="__span-0-545"><a id="__codelineno-0-545" name="__codelineno-0-545"></a><span class="sd">      the Banzhaf semi-value as introduced in [@wang_data_2022].</span>
+</span><span id="__span-0-546"><a id="__codelineno-0-546" name="__codelineno-0-546"></a>
+</span><span id="__span-0-547"><a id="__codelineno-0-547" name="__codelineno-0-547"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-548"><a id="__codelineno-0-548" name="__codelineno-0-548"></a><span class="sd">        u: Utility object with model, data, and scoring function.</span>
+</span><span id="__span-0-549"><a id="__codelineno-0-549" name="__codelineno-0-549"></a><span class="sd">        done: Stopping criterion.</span>
+</span><span id="__span-0-550"><a id="__codelineno-0-550" name="__codelineno-0-550"></a><span class="sd">        mode: The semi-value mode to use. See</span>
+</span><span id="__span-0-551"><a id="__codelineno-0-551" name="__codelineno-0-551"></a><span class="sd">            [SemiValueMode][pydvl.value.semivalues.SemiValueMode] for a list.</span>
+</span><span id="__span-0-552"><a id="__codelineno-0-552" name="__codelineno-0-552"></a><span class="sd">        sampler_t: The sampler type to use. See [sampler][pydvl.value.sampler]</span>
+</span><span id="__span-0-553"><a id="__codelineno-0-553" name="__codelineno-0-553"></a><span class="sd">            for a list.</span>
+</span><span id="__span-0-554"><a id="__codelineno-0-554" name="__codelineno-0-554"></a><span class="sd">        batch_size: Number of marginal evaluations per (parallelized) task.</span>
+</span><span id="__span-0-555"><a id="__codelineno-0-555" name="__codelineno-0-555"></a><span class="sd">        n_jobs: Number of parallel jobs to use.</span>
+</span><span id="__span-0-556"><a id="__codelineno-0-556" name="__codelineno-0-556"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
+</span><span id="__span-0-557"><a id="__codelineno-0-557" name="__codelineno-0-557"></a><span class="sd">        kwargs: Additional keyword arguments passed to</span>
+</span><span id="__span-0-558"><a id="__codelineno-0-558" name="__codelineno-0-558"></a><span class="sd">            [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].</span>
+</span><span id="__span-0-559"><a id="__codelineno-0-559" name="__codelineno-0-559"></a>
+</span><span id="__span-0-560"><a id="__codelineno-0-560" name="__codelineno-0-560"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-561"><a id="__codelineno-0-561" name="__codelineno-0-561"></a><span class="sd">        Object with the results.</span>
+</span><span id="__span-0-562"><a id="__codelineno-0-562" name="__codelineno-0-562"></a>
+</span><span id="__span-0-563"><a id="__codelineno-0-563" name="__codelineno-0-563"></a><span class="sd">    !!! warning &quot;Deprecation notice&quot;</span>
+</span><span id="__span-0-564"><a id="__codelineno-0-564" name="__codelineno-0-564"></a><span class="sd">        Parameter `batch_size` is for experimental use and will be removed in</span>
+</span><span id="__span-0-565"><a id="__codelineno-0-565" name="__codelineno-0-565"></a><span class="sd">        future versions.</span>
+</span><span id="__span-0-566"><a id="__codelineno-0-566" name="__codelineno-0-566"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-567"><a id="__codelineno-0-567" name="__codelineno-0-567"></a>    <span class="k">if</span> <span class="n">mode</span> <span class="o">==</span> <span class="n">SemiValueMode</span><span class="o">.</span><span class="n">Shapley</span><span class="p">:</span>
+</span><span id="__span-0-568"><a id="__codelineno-0-568" name="__codelineno-0-568"></a>        <span class="n">coefficient</span> <span class="o">=</span> <span class="n">shapley_coefficient</span>
+</span><span id="__span-0-569"><a id="__codelineno-0-569" name="__codelineno-0-569"></a>    <span class="k">elif</span> <span class="n">mode</span> <span class="o">==</span> <span class="n">SemiValueMode</span><span class="o">.</span><span class="n">BetaShapley</span><span class="p">:</span>
+</span><span id="__span-0-570"><a id="__codelineno-0-570" name="__codelineno-0-570"></a>        <span class="n">alpha</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">&quot;alpha&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+</span><span id="__span-0-571"><a id="__codelineno-0-571" name="__codelineno-0-571"></a>        <span class="n">beta</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">&quot;beta&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+</span><span id="__span-0-572"><a id="__codelineno-0-572" name="__codelineno-0-572"></a>        <span class="n">coefficient</span> <span class="o">=</span> <span class="n">beta_coefficient</span><span class="p">(</span><span class="n">alpha</span><span class="p">,</span> <span class="n">beta</span><span class="p">)</span>
+</span><span id="__span-0-573"><a id="__codelineno-0-573" name="__codelineno-0-573"></a>    <span class="k">elif</span> <span class="n">mode</span> <span class="o">==</span> <span class="n">SemiValueMode</span><span class="o">.</span><span class="n">Banzhaf</span><span class="p">:</span>
+</span><span id="__span-0-574"><a id="__codelineno-0-574" name="__codelineno-0-574"></a>        <span class="n">coefficient</span> <span class="o">=</span> <span class="n">banzhaf_coefficient</span>
+</span><span id="__span-0-575"><a id="__codelineno-0-575" name="__codelineno-0-575"></a>    <span class="k">else</span><span class="p">:</span>
+</span><span id="__span-0-576"><a id="__codelineno-0-576" name="__codelineno-0-576"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Unknown mode </span><span class="si">{</span><span class="n">mode</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+</span><span id="__span-0-577"><a id="__codelineno-0-577" name="__codelineno-0-577"></a>    <span class="n">coefficient</span> <span class="o">=</span> <span class="n">cast</span><span class="p">(</span><span class="n">SVCoefficient</span><span class="p">,</span> <span class="n">coefficient</span><span class="p">)</span>
+</span><span id="__span-0-578"><a id="__codelineno-0-578" name="__codelineno-0-578"></a>
+</span><span id="__span-0-579"><a id="__codelineno-0-579" name="__codelineno-0-579"></a>    <span class="c1"># HACK: cannot infer return type because of useless IndexT, NameT</span>
+</span><span id="__span-0-580"><a id="__codelineno-0-580" name="__codelineno-0-580"></a>    <span class="k">return</span> <span class="n">compute_generic_semivalues</span><span class="p">(</span>  <span class="c1"># type: ignore</span>
+</span><span id="__span-0-581"><a id="__codelineno-0-581" name="__codelineno-0-581"></a>        <span class="n">sampler_t</span><span class="p">(</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">),</span>
+</span><span id="__span-0-582"><a id="__codelineno-0-582" name="__codelineno-0-582"></a>        <span class="n">u</span><span class="p">,</span>
+</span><span id="__span-0-583"><a id="__codelineno-0-583" name="__codelineno-0-583"></a>        <span class="n">coefficient</span><span class="p">,</span>
+</span><span id="__span-0-584"><a id="__codelineno-0-584" name="__codelineno-0-584"></a>        <span class="n">done</span><span class="p">,</span>
+</span><span id="__span-0-585"><a id="__codelineno-0-585" name="__codelineno-0-585"></a>        <span class="n">n_jobs</span><span class="o">=</span><span class="n">n_jobs</span><span class="p">,</span>
+</span><span id="__span-0-586"><a id="__codelineno-0-586" name="__codelineno-0-586"></a>        <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
+</span><span id="__span-0-587"><a id="__codelineno-0-587" name="__codelineno-0-587"></a>        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+</span><span id="__span-0-588"><a id="__codelineno-0-588" name="__codelineno-0-588"></a>    <span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -4464,11 +4791,11 @@ <h2 id="pydvl.value.semivalues.compute_semivalues" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/shapley/common/index.html b/devel/api/pydvl/value/shapley/common/index.html
index 05acd69e4..05389805b 100644
--- a/devel/api/pydvl/value/shapley/common/index.html
+++ b/devel/api/pydvl/value/shapley/common/index.html
@@ -3031,11 +3031,11 @@ <h2 id="pydvl.value.shapley.common.compute_shapley_values" class="doc doc-headin
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/shapley/gt/index.html b/devel/api/pydvl/value/shapley/gt/index.html
index 5a999e294..67513714d 100644
--- a/devel/api/pydvl/value/shapley/gt/index.html
+++ b/devel/api/pydvl/value/shapley/gt/index.html
@@ -3262,11 +3262,11 @@ <h2 id="pydvl.value.shapley.gt.group_testing_shapley" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/shapley/index.html b/devel/api/pydvl/value/shapley/index.html
index 6326db966..284d0e2c6 100644
--- a/devel/api/pydvl/value/shapley/index.html
+++ b/devel/api/pydvl/value/shapley/index.html
@@ -2501,11 +2501,11 @@ <h1>Shapley</h1>
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/shapley/knn/index.html b/devel/api/pydvl/value/shapley/knn/index.html
index 88f6ccdc1..d16a152f5 100644
--- a/devel/api/pydvl/value/shapley/knn/index.html
+++ b/devel/api/pydvl/value/shapley/knn/index.html
@@ -2861,11 +2861,11 @@ <h2 id="pydvl.value.shapley.knn.knn_shapley" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/shapley/montecarlo/index.html b/devel/api/pydvl/value/shapley/montecarlo/index.html
index ceff84a3c..d32b4a68d 100644
--- a/devel/api/pydvl/value/shapley/montecarlo/index.html
+++ b/devel/api/pydvl/value/shapley/montecarlo/index.html
@@ -2605,7 +2605,7 @@ <h3 id="pydvl.value.shapley.montecarlo--references">References<a class="headerli
 <ol>
 <li id="pydvl.value.shapley.montecarlo--fn:1">
 <p><a name="ghorbani_data_2019"></a>Ghorbani, A., Zou, J., 2019.
-<a href="http://proceedings.mlr.press/v97/ghorbani19c.html">Data Shapley: Equitable Valuation of Data for Machine Learning</a>.
+<a href="https://proceedings.mlr.press/v97/ghorbani19c.html">Data Shapley: Equitable Valuation of Data for Machine Learning</a>.
 In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242–2251.&#160;<a class="footnote-backref" href="#pydvl.value.shapley.montecarlo--fnref:1" title="Jump back to footnote 1 in the text">&#8617;</a></p>
 </li>
 </ol>
@@ -2936,7 +2936,9 @@ <h2 id="pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley" class="do
 <span class="normal"><a href="#__codelineno-0-234">234</a></span>
 <span class="normal"><a href="#__codelineno-0-235">235</a></span>
 <span class="normal"><a href="#__codelineno-0-236">236</a></span>
-<span class="normal"><a href="#__codelineno-0-237">237</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-128"><a id="__codelineno-0-128" name="__codelineno-0-128"></a><span class="nd">@deprecated</span><span class="p">(</span>
+<span class="normal"><a href="#__codelineno-0-237">237</a></span>
+<span class="normal"><a href="#__codelineno-0-238">238</a></span>
+<span class="normal"><a href="#__codelineno-0-239">239</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-128"><a id="__codelineno-0-128" name="__codelineno-0-128"></a><span class="nd">@deprecated</span><span class="p">(</span>
 </span><span id="__span-0-129"><a id="__codelineno-0-129" name="__codelineno-0-129"></a>    <span class="n">target</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
 </span><span id="__span-0-130"><a id="__codelineno-0-130" name="__codelineno-0-130"></a>    <span class="n">deprecated_in</span><span class="o">=</span><span class="s2">&quot;0.7.0&quot;</span><span class="p">,</span>
 </span><span id="__span-0-131"><a id="__codelineno-0-131" name="__codelineno-0-131"></a>    <span class="n">remove_in</span><span class="o">=</span><span class="s2">&quot;0.8.0&quot;</span><span class="p">,</span>
@@ -3012,40 +3014,42 @@ <h2 id="pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley" class="do
 </span><span id="__span-0-201"><a id="__codelineno-0-201" name="__codelineno-0-201"></a>    <span class="n">n_submitted_jobs</span> <span class="o">=</span> <span class="mi">2</span> <span class="o">*</span> <span class="n">max_workers</span>  <span class="c1"># number of jobs in the executor&#39;s queue</span>
 </span><span id="__span-0-202"><a id="__codelineno-0-202" name="__codelineno-0-202"></a>
 </span><span id="__span-0-203"><a id="__codelineno-0-203" name="__codelineno-0-203"></a>    <span class="n">seed_sequence</span> <span class="o">=</span> <span class="n">ensure_seed_sequence</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>
-</span><span id="__span-0-204"><a id="__codelineno-0-204" name="__codelineno-0-204"></a>    <span class="n">result</span> <span class="o">=</span> <span class="n">ValuationResult</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">algorithm</span><span class="o">=</span><span class="n">algorithm</span><span class="p">)</span>
-</span><span id="__span-0-205"><a id="__codelineno-0-205" name="__codelineno-0-205"></a>
-</span><span id="__span-0-206"><a id="__codelineno-0-206" name="__codelineno-0-206"></a>    <span class="n">pbar</span> <span class="o">=</span> <span class="n">tqdm</span><span class="p">(</span><span class="n">disable</span><span class="o">=</span><span class="ow">not</span> <span class="n">progress</span><span class="p">,</span> <span class="n">total</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">unit</span><span class="o">=</span><span class="s2">&quot;%&quot;</span><span class="p">)</span>
+</span><span id="__span-0-204"><a id="__codelineno-0-204" name="__codelineno-0-204"></a>    <span class="n">result</span> <span class="o">=</span> <span class="n">ValuationResult</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span>
+</span><span id="__span-0-205"><a id="__codelineno-0-205" name="__codelineno-0-205"></a>        <span class="n">algorithm</span><span class="o">=</span><span class="n">algorithm</span><span class="p">,</span> <span class="n">indices</span><span class="o">=</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="n">data_names</span><span class="o">=</span><span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">data_names</span>
+</span><span id="__span-0-206"><a id="__codelineno-0-206" name="__codelineno-0-206"></a>    <span class="p">)</span>
 </span><span id="__span-0-207"><a id="__codelineno-0-207" name="__codelineno-0-207"></a>
-</span><span id="__span-0-208"><a id="__codelineno-0-208" name="__codelineno-0-208"></a>    <span class="k">with</span> <span class="n">init_executor</span><span class="p">(</span>
-</span><span id="__span-0-209"><a id="__codelineno-0-209" name="__codelineno-0-209"></a>        <span class="n">max_workers</span><span class="o">=</span><span class="n">max_workers</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span> <span class="n">cancel_futures</span><span class="o">=</span><span class="n">CancellationPolicy</span><span class="o">.</span><span class="n">ALL</span>
-</span><span id="__span-0-210"><a id="__codelineno-0-210" name="__codelineno-0-210"></a>    <span class="p">)</span> <span class="k">as</span> <span class="n">executor</span><span class="p">:</span>
-</span><span id="__span-0-211"><a id="__codelineno-0-211" name="__codelineno-0-211"></a>        <span class="n">pending</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="n">Future</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
-</span><span id="__span-0-212"><a id="__codelineno-0-212" name="__codelineno-0-212"></a>        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
-</span><span id="__span-0-213"><a id="__codelineno-0-213" name="__codelineno-0-213"></a>            <span class="n">pbar</span><span class="o">.</span><span class="n">n</span> <span class="o">=</span> <span class="mi">100</span> <span class="o">*</span> <span class="n">done</span><span class="o">.</span><span class="n">completion</span><span class="p">()</span>
-</span><span id="__span-0-214"><a id="__codelineno-0-214" name="__codelineno-0-214"></a>            <span class="n">pbar</span><span class="o">.</span><span class="n">refresh</span><span class="p">()</span>
-</span><span id="__span-0-215"><a id="__codelineno-0-215" name="__codelineno-0-215"></a>
-</span><span id="__span-0-216"><a id="__codelineno-0-216" name="__codelineno-0-216"></a>            <span class="n">completed</span><span class="p">,</span> <span class="n">pending</span> <span class="o">=</span> <span class="n">wait</span><span class="p">(</span>
-</span><span id="__span-0-217"><a id="__codelineno-0-217" name="__codelineno-0-217"></a>                <span class="n">pending</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">wait_timeout</span><span class="p">,</span> <span class="n">return_when</span><span class="o">=</span><span class="n">FIRST_COMPLETED</span>
-</span><span id="__span-0-218"><a id="__codelineno-0-218" name="__codelineno-0-218"></a>            <span class="p">)</span>
-</span><span id="__span-0-219"><a id="__codelineno-0-219" name="__codelineno-0-219"></a>            <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">completed</span><span class="p">:</span>
-</span><span id="__span-0-220"><a id="__codelineno-0-220" name="__codelineno-0-220"></a>                <span class="n">result</span> <span class="o">+=</span> <span class="n">future</span><span class="o">.</span><span class="n">result</span><span class="p">()</span>
-</span><span id="__span-0-221"><a id="__codelineno-0-221" name="__codelineno-0-221"></a>                <span class="c1"># we could check outside the loop, but that means more</span>
-</span><span id="__span-0-222"><a id="__codelineno-0-222" name="__codelineno-0-222"></a>                <span class="c1"># submissions if the stopping criterion is unstable</span>
-</span><span id="__span-0-223"><a id="__codelineno-0-223" name="__codelineno-0-223"></a>                <span class="k">if</span> <span class="n">done</span><span class="p">(</span><span class="n">result</span><span class="p">):</span>
-</span><span id="__span-0-224"><a id="__codelineno-0-224" name="__codelineno-0-224"></a>                    <span class="k">return</span> <span class="n">result</span>
-</span><span id="__span-0-225"><a id="__codelineno-0-225" name="__codelineno-0-225"></a>
-</span><span id="__span-0-226"><a id="__codelineno-0-226" name="__codelineno-0-226"></a>            <span class="c1"># Ensure that we always have n_submitted_jobs in the queue or running</span>
-</span><span id="__span-0-227"><a id="__codelineno-0-227" name="__codelineno-0-227"></a>            <span class="n">n_remaining_slots</span> <span class="o">=</span> <span class="n">n_submitted_jobs</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">pending</span><span class="p">)</span>
-</span><span id="__span-0-228"><a id="__codelineno-0-228" name="__codelineno-0-228"></a>            <span class="n">seeds</span> <span class="o">=</span> <span class="n">seed_sequence</span><span class="o">.</span><span class="n">spawn</span><span class="p">(</span><span class="n">n_remaining_slots</span><span class="p">)</span>
-</span><span id="__span-0-229"><a id="__codelineno-0-229" name="__codelineno-0-229"></a>            <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_remaining_slots</span><span class="p">):</span>
-</span><span id="__span-0-230"><a id="__codelineno-0-230" name="__codelineno-0-230"></a>                <span class="n">future</span> <span class="o">=</span> <span class="n">executor</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span>
-</span><span id="__span-0-231"><a id="__codelineno-0-231" name="__codelineno-0-231"></a>                    <span class="n">_permutation_montecarlo_one_step</span><span class="p">,</span>
-</span><span id="__span-0-232"><a id="__codelineno-0-232" name="__codelineno-0-232"></a>                    <span class="n">u</span><span class="p">,</span>
-</span><span id="__span-0-233"><a id="__codelineno-0-233" name="__codelineno-0-233"></a>                    <span class="n">truncation</span><span class="p">,</span>
-</span><span id="__span-0-234"><a id="__codelineno-0-234" name="__codelineno-0-234"></a>                    <span class="n">algorithm</span><span class="p">,</span>
-</span><span id="__span-0-235"><a id="__codelineno-0-235" name="__codelineno-0-235"></a>                    <span class="n">seed</span><span class="o">=</span><span class="n">seeds</span><span class="p">[</span><span class="n">i</span><span class="p">],</span>
-</span><span id="__span-0-236"><a id="__codelineno-0-236" name="__codelineno-0-236"></a>                <span class="p">)</span>
-</span><span id="__span-0-237"><a id="__codelineno-0-237" name="__codelineno-0-237"></a>                <span class="n">pending</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">future</span><span class="p">)</span>
+</span><span id="__span-0-208"><a id="__codelineno-0-208" name="__codelineno-0-208"></a>    <span class="n">pbar</span> <span class="o">=</span> <span class="n">tqdm</span><span class="p">(</span><span class="n">disable</span><span class="o">=</span><span class="ow">not</span> <span class="n">progress</span><span class="p">,</span> <span class="n">total</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">unit</span><span class="o">=</span><span class="s2">&quot;%&quot;</span><span class="p">)</span>
+</span><span id="__span-0-209"><a id="__codelineno-0-209" name="__codelineno-0-209"></a>
+</span><span id="__span-0-210"><a id="__codelineno-0-210" name="__codelineno-0-210"></a>    <span class="k">with</span> <span class="n">init_executor</span><span class="p">(</span>
+</span><span id="__span-0-211"><a id="__codelineno-0-211" name="__codelineno-0-211"></a>        <span class="n">max_workers</span><span class="o">=</span><span class="n">max_workers</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span> <span class="n">cancel_futures</span><span class="o">=</span><span class="n">CancellationPolicy</span><span class="o">.</span><span class="n">ALL</span>
+</span><span id="__span-0-212"><a id="__codelineno-0-212" name="__codelineno-0-212"></a>    <span class="p">)</span> <span class="k">as</span> <span class="n">executor</span><span class="p">:</span>
+</span><span id="__span-0-213"><a id="__codelineno-0-213" name="__codelineno-0-213"></a>        <span class="n">pending</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="n">Future</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
+</span><span id="__span-0-214"><a id="__codelineno-0-214" name="__codelineno-0-214"></a>        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+</span><span id="__span-0-215"><a id="__codelineno-0-215" name="__codelineno-0-215"></a>            <span class="n">pbar</span><span class="o">.</span><span class="n">n</span> <span class="o">=</span> <span class="mi">100</span> <span class="o">*</span> <span class="n">done</span><span class="o">.</span><span class="n">completion</span><span class="p">()</span>
+</span><span id="__span-0-216"><a id="__codelineno-0-216" name="__codelineno-0-216"></a>            <span class="n">pbar</span><span class="o">.</span><span class="n">refresh</span><span class="p">()</span>
+</span><span id="__span-0-217"><a id="__codelineno-0-217" name="__codelineno-0-217"></a>
+</span><span id="__span-0-218"><a id="__codelineno-0-218" name="__codelineno-0-218"></a>            <span class="n">completed</span><span class="p">,</span> <span class="n">pending</span> <span class="o">=</span> <span class="n">wait</span><span class="p">(</span>
+</span><span id="__span-0-219"><a id="__codelineno-0-219" name="__codelineno-0-219"></a>                <span class="n">pending</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">wait_timeout</span><span class="p">,</span> <span class="n">return_when</span><span class="o">=</span><span class="n">FIRST_COMPLETED</span>
+</span><span id="__span-0-220"><a id="__codelineno-0-220" name="__codelineno-0-220"></a>            <span class="p">)</span>
+</span><span id="__span-0-221"><a id="__codelineno-0-221" name="__codelineno-0-221"></a>            <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">completed</span><span class="p">:</span>
+</span><span id="__span-0-222"><a id="__codelineno-0-222" name="__codelineno-0-222"></a>                <span class="n">result</span> <span class="o">+=</span> <span class="n">future</span><span class="o">.</span><span class="n">result</span><span class="p">()</span>
+</span><span id="__span-0-223"><a id="__codelineno-0-223" name="__codelineno-0-223"></a>                <span class="c1"># we could check outside the loop, but that means more</span>
+</span><span id="__span-0-224"><a id="__codelineno-0-224" name="__codelineno-0-224"></a>                <span class="c1"># submissions if the stopping criterion is unstable</span>
+</span><span id="__span-0-225"><a id="__codelineno-0-225" name="__codelineno-0-225"></a>                <span class="k">if</span> <span class="n">done</span><span class="p">(</span><span class="n">result</span><span class="p">):</span>
+</span><span id="__span-0-226"><a id="__codelineno-0-226" name="__codelineno-0-226"></a>                    <span class="k">return</span> <span class="n">result</span>
+</span><span id="__span-0-227"><a id="__codelineno-0-227" name="__codelineno-0-227"></a>
+</span><span id="__span-0-228"><a id="__codelineno-0-228" name="__codelineno-0-228"></a>            <span class="c1"># Ensure that we always have n_submitted_jobs in the queue or running</span>
+</span><span id="__span-0-229"><a id="__codelineno-0-229" name="__codelineno-0-229"></a>            <span class="n">n_remaining_slots</span> <span class="o">=</span> <span class="n">n_submitted_jobs</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">pending</span><span class="p">)</span>
+</span><span id="__span-0-230"><a id="__codelineno-0-230" name="__codelineno-0-230"></a>            <span class="n">seeds</span> <span class="o">=</span> <span class="n">seed_sequence</span><span class="o">.</span><span class="n">spawn</span><span class="p">(</span><span class="n">n_remaining_slots</span><span class="p">)</span>
+</span><span id="__span-0-231"><a id="__codelineno-0-231" name="__codelineno-0-231"></a>            <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_remaining_slots</span><span class="p">):</span>
+</span><span id="__span-0-232"><a id="__codelineno-0-232" name="__codelineno-0-232"></a>                <span class="n">future</span> <span class="o">=</span> <span class="n">executor</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span>
+</span><span id="__span-0-233"><a id="__codelineno-0-233" name="__codelineno-0-233"></a>                    <span class="n">_permutation_montecarlo_one_step</span><span class="p">,</span>
+</span><span id="__span-0-234"><a id="__codelineno-0-234" name="__codelineno-0-234"></a>                    <span class="n">u</span><span class="p">,</span>
+</span><span id="__span-0-235"><a id="__codelineno-0-235" name="__codelineno-0-235"></a>                    <span class="n">truncation</span><span class="p">,</span>
+</span><span id="__span-0-236"><a id="__codelineno-0-236" name="__codelineno-0-236"></a>                    <span class="n">algorithm</span><span class="p">,</span>
+</span><span id="__span-0-237"><a id="__codelineno-0-237" name="__codelineno-0-237"></a>                    <span class="n">seed</span><span class="o">=</span><span class="n">seeds</span><span class="p">[</span><span class="n">i</span><span class="p">],</span>
+</span><span id="__span-0-238"><a id="__codelineno-0-238" name="__codelineno-0-238"></a>                <span class="p">)</span>
+</span><span id="__span-0-239"><a id="__codelineno-0-239" name="__codelineno-0-239"></a>                <span class="n">pending</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">future</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3223,9 +3227,7 @@ <h2 id="pydvl.value.shapley.montecarlo.combinatorial_montecarlo_shapley" class="
 
           <details class="quote">
             <summary>Source code in <code>src/pydvl/value/shapley/montecarlo.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-296">296</a></span>
-<span class="normal"><a href="#__codelineno-0-297">297</a></span>
-<span class="normal"><a href="#__codelineno-0-298">298</a></span>
+            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-298">298</a></span>
 <span class="normal"><a href="#__codelineno-0-299">299</a></span>
 <span class="normal"><a href="#__codelineno-0-300">300</a></span>
 <span class="normal"><a href="#__codelineno-0-301">301</a></span>
@@ -3273,57 +3275,59 @@ <h2 id="pydvl.value.shapley.montecarlo.combinatorial_montecarlo_shapley" class="
 <span class="normal"><a href="#__codelineno-0-343">343</a></span>
 <span class="normal"><a href="#__codelineno-0-344">344</a></span>
 <span class="normal"><a href="#__codelineno-0-345">345</a></span>
-<span class="normal"><a href="#__codelineno-0-346">346</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-296"><a id="__codelineno-0-296" name="__codelineno-0-296"></a><span class="k">def</span> <span class="nf">combinatorial_montecarlo_shapley</span><span class="p">(</span>
-</span><span id="__span-0-297"><a id="__codelineno-0-297" name="__codelineno-0-297"></a>    <span class="n">u</span><span class="p">:</span> <span class="n">Utility</span><span class="p">,</span>
-</span><span id="__span-0-298"><a id="__codelineno-0-298" name="__codelineno-0-298"></a>    <span class="n">done</span><span class="p">:</span> <span class="n">StoppingCriterion</span><span class="p">,</span>
-</span><span id="__span-0-299"><a id="__codelineno-0-299" name="__codelineno-0-299"></a>    <span class="o">*</span><span class="p">,</span>
-</span><span id="__span-0-300"><a id="__codelineno-0-300" name="__codelineno-0-300"></a>    <span class="n">n_jobs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-</span><span id="__span-0-301"><a id="__codelineno-0-301" name="__codelineno-0-301"></a>    <span class="n">config</span><span class="p">:</span> <span class="n">ParallelConfig</span> <span class="o">=</span> <span class="n">ParallelConfig</span><span class="p">(),</span>
-</span><span id="__span-0-302"><a id="__codelineno-0-302" name="__codelineno-0-302"></a>    <span class="n">progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-</span><span id="__span-0-303"><a id="__codelineno-0-303" name="__codelineno-0-303"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-</span><span id="__span-0-304"><a id="__codelineno-0-304" name="__codelineno-0-304"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">:</span>
-</span><span id="__span-0-305"><a id="__codelineno-0-305" name="__codelineno-0-305"></a><span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Computes an approximate Shapley value using the combinatorial</span>
-</span><span id="__span-0-306"><a id="__codelineno-0-306" name="__codelineno-0-306"></a><span class="sd">    definition:</span>
-</span><span id="__span-0-307"><a id="__codelineno-0-307" name="__codelineno-0-307"></a>
-</span><span id="__span-0-308"><a id="__codelineno-0-308" name="__codelineno-0-308"></a><span class="sd">    $$v_u(i) = \frac{1}{n} \sum_{S \subseteq N \setminus \{i\}}</span>
-</span><span id="__span-0-309"><a id="__codelineno-0-309" name="__codelineno-0-309"></a><span class="sd">    \binom{n-1}{ | S | }^{-1} [u(S \cup \{i\}) − u(S)]$$</span>
-</span><span id="__span-0-310"><a id="__codelineno-0-310" name="__codelineno-0-310"></a>
-</span><span id="__span-0-311"><a id="__codelineno-0-311" name="__codelineno-0-311"></a><span class="sd">    This consists of randomly sampling subsets of the power set of the training</span>
-</span><span id="__span-0-312"><a id="__codelineno-0-312" name="__codelineno-0-312"></a><span class="sd">    indices in [u.data][pydvl.utils.utility.Utility], and computing their</span>
-</span><span id="__span-0-313"><a id="__codelineno-0-313" name="__codelineno-0-313"></a><span class="sd">    marginal utilities. See [Data valuation][computing-data-values] for details.</span>
-</span><span id="__span-0-314"><a id="__codelineno-0-314" name="__codelineno-0-314"></a>
-</span><span id="__span-0-315"><a id="__codelineno-0-315" name="__codelineno-0-315"></a><span class="sd">    Note that because sampling is done with replacement, the approximation is</span>
-</span><span id="__span-0-316"><a id="__codelineno-0-316" name="__codelineno-0-316"></a><span class="sd">    poor even for $2^{m}$ subsets with $m&gt;n$, even though there are $2^{n-1}$</span>
-</span><span id="__span-0-317"><a id="__codelineno-0-317" name="__codelineno-0-317"></a><span class="sd">    subsets for each $i$. Prefer</span>
-</span><span id="__span-0-318"><a id="__codelineno-0-318" name="__codelineno-0-318"></a><span class="sd">    [permutation_montecarlo_shapley()][pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley].</span>
-</span><span id="__span-0-319"><a id="__codelineno-0-319" name="__codelineno-0-319"></a>
-</span><span id="__span-0-320"><a id="__codelineno-0-320" name="__codelineno-0-320"></a><span class="sd">    Parallelization is done by splitting the set of indices across processes and</span>
-</span><span id="__span-0-321"><a id="__codelineno-0-321" name="__codelineno-0-321"></a><span class="sd">    computing the sum over subsets $S \subseteq N \setminus \{i\}$ separately.</span>
-</span><span id="__span-0-322"><a id="__codelineno-0-322" name="__codelineno-0-322"></a>
-</span><span id="__span-0-323"><a id="__codelineno-0-323" name="__codelineno-0-323"></a><span class="sd">    Args:</span>
-</span><span id="__span-0-324"><a id="__codelineno-0-324" name="__codelineno-0-324"></a><span class="sd">        u: Utility object with model, data, and scoring function</span>
-</span><span id="__span-0-325"><a id="__codelineno-0-325" name="__codelineno-0-325"></a><span class="sd">        done: Stopping criterion for the computation.</span>
-</span><span id="__span-0-326"><a id="__codelineno-0-326" name="__codelineno-0-326"></a><span class="sd">        n_jobs: number of parallel jobs across which to distribute the</span>
-</span><span id="__span-0-327"><a id="__codelineno-0-327" name="__codelineno-0-327"></a><span class="sd">            computation. Each worker receives a chunk of</span>
-</span><span id="__span-0-328"><a id="__codelineno-0-328" name="__codelineno-0-328"></a><span class="sd">            [indices][pydvl.utils.dataset.Dataset.indices]</span>
-</span><span id="__span-0-329"><a id="__codelineno-0-329" name="__codelineno-0-329"></a><span class="sd">        config: Object configuring parallel computation, with cluster address,</span>
-</span><span id="__span-0-330"><a id="__codelineno-0-330" name="__codelineno-0-330"></a><span class="sd">            number of cpus, etc.</span>
-</span><span id="__span-0-331"><a id="__codelineno-0-331" name="__codelineno-0-331"></a><span class="sd">        progress: Whether to display progress bars for each job.</span>
-</span><span id="__span-0-332"><a id="__codelineno-0-332" name="__codelineno-0-332"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
-</span><span id="__span-0-333"><a id="__codelineno-0-333" name="__codelineno-0-333"></a>
-</span><span id="__span-0-334"><a id="__codelineno-0-334" name="__codelineno-0-334"></a><span class="sd">    Returns:</span>
-</span><span id="__span-0-335"><a id="__codelineno-0-335" name="__codelineno-0-335"></a><span class="sd">        Object with the data values.</span>
-</span><span id="__span-0-336"><a id="__codelineno-0-336" name="__codelineno-0-336"></a><span class="sd">    &quot;&quot;&quot;</span>
-</span><span id="__span-0-337"><a id="__codelineno-0-337" name="__codelineno-0-337"></a>
-</span><span id="__span-0-338"><a id="__codelineno-0-338" name="__codelineno-0-338"></a>    <span class="n">map_reduce_job</span><span class="p">:</span> <span class="n">MapReduceJob</span><span class="p">[</span><span class="n">NDArray</span><span class="p">,</span> <span class="n">ValuationResult</span><span class="p">]</span> <span class="o">=</span> <span class="n">MapReduceJob</span><span class="p">(</span>
-</span><span id="__span-0-339"><a id="__codelineno-0-339" name="__codelineno-0-339"></a>        <span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span>
-</span><span id="__span-0-340"><a id="__codelineno-0-340" name="__codelineno-0-340"></a>        <span class="n">map_func</span><span class="o">=</span><span class="n">_combinatorial_montecarlo_shapley</span><span class="p">,</span>
-</span><span id="__span-0-341"><a id="__codelineno-0-341" name="__codelineno-0-341"></a>        <span class="n">reduce_func</span><span class="o">=</span><span class="k">lambda</span> <span class="n">results</span><span class="p">:</span> <span class="n">reduce</span><span class="p">(</span><span class="n">operator</span><span class="o">.</span><span class="n">add</span><span class="p">,</span> <span class="n">results</span><span class="p">),</span>
-</span><span id="__span-0-342"><a id="__codelineno-0-342" name="__codelineno-0-342"></a>        <span class="n">map_kwargs</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">u</span><span class="o">=</span><span class="n">u</span><span class="p">,</span> <span class="n">done</span><span class="o">=</span><span class="n">done</span><span class="p">,</span> <span class="n">progress</span><span class="o">=</span><span class="n">progress</span><span class="p">),</span>
-</span><span id="__span-0-343"><a id="__codelineno-0-343" name="__codelineno-0-343"></a>        <span class="n">n_jobs</span><span class="o">=</span><span class="n">n_jobs</span><span class="p">,</span>
-</span><span id="__span-0-344"><a id="__codelineno-0-344" name="__codelineno-0-344"></a>        <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span>
-</span><span id="__span-0-345"><a id="__codelineno-0-345" name="__codelineno-0-345"></a>    <span class="p">)</span>
-</span><span id="__span-0-346"><a id="__codelineno-0-346" name="__codelineno-0-346"></a>    <span class="k">return</span> <span class="n">map_reduce_job</span><span class="p">(</span><span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-346">346</a></span>
+<span class="normal"><a href="#__codelineno-0-347">347</a></span>
+<span class="normal"><a href="#__codelineno-0-348">348</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-298"><a id="__codelineno-0-298" name="__codelineno-0-298"></a><span class="k">def</span> <span class="nf">combinatorial_montecarlo_shapley</span><span class="p">(</span>
+</span><span id="__span-0-299"><a id="__codelineno-0-299" name="__codelineno-0-299"></a>    <span class="n">u</span><span class="p">:</span> <span class="n">Utility</span><span class="p">,</span>
+</span><span id="__span-0-300"><a id="__codelineno-0-300" name="__codelineno-0-300"></a>    <span class="n">done</span><span class="p">:</span> <span class="n">StoppingCriterion</span><span class="p">,</span>
+</span><span id="__span-0-301"><a id="__codelineno-0-301" name="__codelineno-0-301"></a>    <span class="o">*</span><span class="p">,</span>
+</span><span id="__span-0-302"><a id="__codelineno-0-302" name="__codelineno-0-302"></a>    <span class="n">n_jobs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+</span><span id="__span-0-303"><a id="__codelineno-0-303" name="__codelineno-0-303"></a>    <span class="n">config</span><span class="p">:</span> <span class="n">ParallelConfig</span> <span class="o">=</span> <span class="n">ParallelConfig</span><span class="p">(),</span>
+</span><span id="__span-0-304"><a id="__codelineno-0-304" name="__codelineno-0-304"></a>    <span class="n">progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+</span><span id="__span-0-305"><a id="__codelineno-0-305" name="__codelineno-0-305"></a>    <span class="n">seed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Seed</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+</span><span id="__span-0-306"><a id="__codelineno-0-306" name="__codelineno-0-306"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ValuationResult</span><span class="p">:</span>
+</span><span id="__span-0-307"><a id="__codelineno-0-307" name="__codelineno-0-307"></a><span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Computes an approximate Shapley value using the combinatorial</span>
+</span><span id="__span-0-308"><a id="__codelineno-0-308" name="__codelineno-0-308"></a><span class="sd">    definition:</span>
+</span><span id="__span-0-309"><a id="__codelineno-0-309" name="__codelineno-0-309"></a>
+</span><span id="__span-0-310"><a id="__codelineno-0-310" name="__codelineno-0-310"></a><span class="sd">    $$v_u(i) = \frac{1}{n} \sum_{S \subseteq N \setminus \{i\}}</span>
+</span><span id="__span-0-311"><a id="__codelineno-0-311" name="__codelineno-0-311"></a><span class="sd">    \binom{n-1}{ | S | }^{-1} [u(S \cup \{i\}) − u(S)]$$</span>
+</span><span id="__span-0-312"><a id="__codelineno-0-312" name="__codelineno-0-312"></a>
+</span><span id="__span-0-313"><a id="__codelineno-0-313" name="__codelineno-0-313"></a><span class="sd">    This consists of randomly sampling subsets of the power set of the training</span>
+</span><span id="__span-0-314"><a id="__codelineno-0-314" name="__codelineno-0-314"></a><span class="sd">    indices in [u.data][pydvl.utils.utility.Utility], and computing their</span>
+</span><span id="__span-0-315"><a id="__codelineno-0-315" name="__codelineno-0-315"></a><span class="sd">    marginal utilities. See [Data valuation][computing-data-values] for details.</span>
+</span><span id="__span-0-316"><a id="__codelineno-0-316" name="__codelineno-0-316"></a>
+</span><span id="__span-0-317"><a id="__codelineno-0-317" name="__codelineno-0-317"></a><span class="sd">    Note that because sampling is done with replacement, the approximation is</span>
+</span><span id="__span-0-318"><a id="__codelineno-0-318" name="__codelineno-0-318"></a><span class="sd">    poor even for $2^{m}$ subsets with $m&gt;n$, even though there are $2^{n-1}$</span>
+</span><span id="__span-0-319"><a id="__codelineno-0-319" name="__codelineno-0-319"></a><span class="sd">    subsets for each $i$. Prefer</span>
+</span><span id="__span-0-320"><a id="__codelineno-0-320" name="__codelineno-0-320"></a><span class="sd">    [permutation_montecarlo_shapley()][pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley].</span>
+</span><span id="__span-0-321"><a id="__codelineno-0-321" name="__codelineno-0-321"></a>
+</span><span id="__span-0-322"><a id="__codelineno-0-322" name="__codelineno-0-322"></a><span class="sd">    Parallelization is done by splitting the set of indices across processes and</span>
+</span><span id="__span-0-323"><a id="__codelineno-0-323" name="__codelineno-0-323"></a><span class="sd">    computing the sum over subsets $S \subseteq N \setminus \{i\}$ separately.</span>
+</span><span id="__span-0-324"><a id="__codelineno-0-324" name="__codelineno-0-324"></a>
+</span><span id="__span-0-325"><a id="__codelineno-0-325" name="__codelineno-0-325"></a><span class="sd">    Args:</span>
+</span><span id="__span-0-326"><a id="__codelineno-0-326" name="__codelineno-0-326"></a><span class="sd">        u: Utility object with model, data, and scoring function</span>
+</span><span id="__span-0-327"><a id="__codelineno-0-327" name="__codelineno-0-327"></a><span class="sd">        done: Stopping criterion for the computation.</span>
+</span><span id="__span-0-328"><a id="__codelineno-0-328" name="__codelineno-0-328"></a><span class="sd">        n_jobs: number of parallel jobs across which to distribute the</span>
+</span><span id="__span-0-329"><a id="__codelineno-0-329" name="__codelineno-0-329"></a><span class="sd">            computation. Each worker receives a chunk of</span>
+</span><span id="__span-0-330"><a id="__codelineno-0-330" name="__codelineno-0-330"></a><span class="sd">            [indices][pydvl.utils.dataset.Dataset.indices]</span>
+</span><span id="__span-0-331"><a id="__codelineno-0-331" name="__codelineno-0-331"></a><span class="sd">        config: Object configuring parallel computation, with cluster address,</span>
+</span><span id="__span-0-332"><a id="__codelineno-0-332" name="__codelineno-0-332"></a><span class="sd">            number of cpus, etc.</span>
+</span><span id="__span-0-333"><a id="__codelineno-0-333" name="__codelineno-0-333"></a><span class="sd">        progress: Whether to display progress bars for each job.</span>
+</span><span id="__span-0-334"><a id="__codelineno-0-334" name="__codelineno-0-334"></a><span class="sd">        seed: Either an instance of a numpy random number generator or a seed for it.</span>
+</span><span id="__span-0-335"><a id="__codelineno-0-335" name="__codelineno-0-335"></a>
+</span><span id="__span-0-336"><a id="__codelineno-0-336" name="__codelineno-0-336"></a><span class="sd">    Returns:</span>
+</span><span id="__span-0-337"><a id="__codelineno-0-337" name="__codelineno-0-337"></a><span class="sd">        Object with the data values.</span>
+</span><span id="__span-0-338"><a id="__codelineno-0-338" name="__codelineno-0-338"></a><span class="sd">    &quot;&quot;&quot;</span>
+</span><span id="__span-0-339"><a id="__codelineno-0-339" name="__codelineno-0-339"></a>
+</span><span id="__span-0-340"><a id="__codelineno-0-340" name="__codelineno-0-340"></a>    <span class="n">map_reduce_job</span><span class="p">:</span> <span class="n">MapReduceJob</span><span class="p">[</span><span class="n">NDArray</span><span class="p">,</span> <span class="n">ValuationResult</span><span class="p">]</span> <span class="o">=</span> <span class="n">MapReduceJob</span><span class="p">(</span>
+</span><span id="__span-0-341"><a id="__codelineno-0-341" name="__codelineno-0-341"></a>        <span class="n">u</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span>
+</span><span id="__span-0-342"><a id="__codelineno-0-342" name="__codelineno-0-342"></a>        <span class="n">map_func</span><span class="o">=</span><span class="n">_combinatorial_montecarlo_shapley</span><span class="p">,</span>
+</span><span id="__span-0-343"><a id="__codelineno-0-343" name="__codelineno-0-343"></a>        <span class="n">reduce_func</span><span class="o">=</span><span class="k">lambda</span> <span class="n">results</span><span class="p">:</span> <span class="n">reduce</span><span class="p">(</span><span class="n">operator</span><span class="o">.</span><span class="n">add</span><span class="p">,</span> <span class="n">results</span><span class="p">),</span>
+</span><span id="__span-0-344"><a id="__codelineno-0-344" name="__codelineno-0-344"></a>        <span class="n">map_kwargs</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">u</span><span class="o">=</span><span class="n">u</span><span class="p">,</span> <span class="n">done</span><span class="o">=</span><span class="n">done</span><span class="p">,</span> <span class="n">progress</span><span class="o">=</span><span class="n">progress</span><span class="p">),</span>
+</span><span id="__span-0-345"><a id="__codelineno-0-345" name="__codelineno-0-345"></a>        <span class="n">n_jobs</span><span class="o">=</span><span class="n">n_jobs</span><span class="p">,</span>
+</span><span id="__span-0-346"><a id="__codelineno-0-346" name="__codelineno-0-346"></a>        <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span>
+</span><span id="__span-0-347"><a id="__codelineno-0-347" name="__codelineno-0-347"></a>    <span class="p">)</span>
+</span><span id="__span-0-348"><a id="__codelineno-0-348" name="__codelineno-0-348"></a>    <span class="k">return</span> <span class="n">map_reduce_job</span><span class="p">(</span><span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">)</span>
 </span></code></pre></div></td></tr></table></div>
           </details>
   </div>
@@ -3343,11 +3347,11 @@ <h2 id="pydvl.value.shapley.montecarlo.combinatorial_montecarlo_shapley" class="
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/shapley/naive/index.html b/devel/api/pydvl/value/shapley/naive/index.html
index fec476ea1..c093aa14b 100644
--- a/devel/api/pydvl/value/shapley/naive/index.html
+++ b/devel/api/pydvl/value/shapley/naive/index.html
@@ -3003,11 +3003,11 @@ <h2 id="pydvl.value.shapley.naive.combinatorial_exact_shapley" class="doc doc-he
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/shapley/owen/index.html b/devel/api/pydvl/value/shapley/owen/index.html
index 22adbf023..982b8b4b5 100644
--- a/devel/api/pydvl/value/shapley/owen/index.html
+++ b/devel/api/pydvl/value/shapley/owen/index.html
@@ -2988,11 +2988,11 @@ <h2 id="pydvl.value.shapley.owen.owen_sampling_shapley" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/shapley/truncated/index.html b/devel/api/pydvl/value/shapley/truncated/index.html
index 289fa2cd0..b58758a6c 100644
--- a/devel/api/pydvl/value/shapley/truncated/index.html
+++ b/devel/api/pydvl/value/shapley/truncated/index.html
@@ -2667,7 +2667,7 @@ <h3 id="pydvl.value.shapley.truncated--references">References<a class="headerlin
 <ol>
 <li id="pydvl.value.shapley.truncated--fn:1">
 <p><a name="ghorbani_data_2019"></a>Ghorbani, A., Zou, J., 2019.
-<a href="http://proceedings.mlr.press/v97/ghorbani19c.html">Data Shapley: Equitable Valuation of Data for Machine Learning</a>.
+<a href="https://proceedings.mlr.press/v97/ghorbani19c.html">Data Shapley: Equitable Valuation of Data for Machine Learning</a>.
 In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242–2251.&#160;<a class="footnote-backref" href="#pydvl.value.shapley.truncated--fnref:1" title="Jump back to footnote 1 in the text">&#8617;</a></p>
 </li>
 </ol>
@@ -3524,11 +3524,11 @@ <h2 id="pydvl.value.shapley.truncated.truncated_montecarlo_shapley" class="doc d
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/shapley/types/index.html b/devel/api/pydvl/value/shapley/types/index.html
index 8d254b677..aec9c2bda 100644
--- a/devel/api/pydvl/value/shapley/types/index.html
+++ b/devel/api/pydvl/value/shapley/types/index.html
@@ -2592,11 +2592,11 @@ <h2 id="pydvl.value.shapley.types.ShapleyMode" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/api/pydvl/value/stopping/index.html b/devel/api/pydvl/value/stopping/index.html
index fd52cbbcb..3225d6e2b 100644
--- a/devel/api/pydvl/value/stopping/index.html
+++ b/devel/api/pydvl/value/stopping/index.html
@@ -2788,7 +2788,7 @@ <h3 id="pydvl.value.stopping--references">References<a class="headerlink" href="
 <ol>
 <li id="pydvl.value.stopping--fn:1">
 <p><a name="ghorbani_data_2019"></a>Ghorbani, A., Zou, J., 2019.
-<a href="http://proceedings.mlr.press/v97/ghorbani19c.html">Data Shapley: Equitable Valuation of Data for Machine Learning</a>.
+<a href="https://proceedings.mlr.press/v97/ghorbani19c.html">Data Shapley: Equitable Valuation of Data for Machine Learning</a>.
 In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242–2251.&#160;<a class="footnote-backref" href="#pydvl.value.stopping--fnref:1" title="Jump back to footnote 1 in the text">&#8617;</a></p>
 </li>
 </ol>
@@ -3996,11 +3996,11 @@ <h2 id="pydvl.value.stopping.make_criterion" class="doc doc-heading">
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/examples/data_oob/index.html b/devel/examples/data_oob/index.html
index a29e1b141..be680cd8a 100644
--- a/devel/examples/data_oob/index.html
+++ b/devel/examples/data_oob/index.html
@@ -2690,11 +2690,11 @@ <h1 id="point-removal-experiments">Point removal experiments<a class="headerlink
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/examples/influence_imagenet/index.html b/devel/examples/influence_imagenet/index.html
index ba0779eab..7331f9c8f 100644
--- a/devel/examples/influence_imagenet/index.html
+++ b/devel/examples/influence_imagenet/index.html
@@ -3492,11 +3492,11 @@ <h2 id="regularizing-the-hessian">Regularizing the Hessian<a class="headerlink"
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/examples/influence_synthetic/index.html b/devel/examples/influence_synthetic/index.html
index 12e24e986..8f89176fe 100644
--- a/devel/examples/influence_synthetic/index.html
+++ b/devel/examples/influence_synthetic/index.html
@@ -3311,11 +3311,11 @@ <h2 id="appendix-calculating-the-decision-boundary">Appendix: Calculating the de
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/examples/influence_wine/index.html b/devel/examples/influence_wine/index.html
index 903943b4c..ee7ab35af 100644
--- a/devel/examples/influence_wine/index.html
+++ b/devel/examples/influence_wine/index.html
@@ -3194,11 +3194,11 @@ <h2 id="speeding-up-influences-for-big-models">Speeding up influences for big mo
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/examples/least_core_basic/index.html b/devel/examples/least_core_basic/index.html
index 7cc5130cc..538dca207 100644
--- a/devel/examples/least_core_basic/index.html
+++ b/devel/examples/least_core_basic/index.html
@@ -3078,11 +3078,11 @@ <h3 id="remove-worst">Remove Worst<a class="headerlink" href="#remove-worst" tit
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/examples/shapley_basic_spotify/index.html b/devel/examples/shapley_basic_spotify/index.html
index a91d714b8..d02269767 100644
--- a/devel/examples/shapley_basic_spotify/index.html
+++ b/devel/examples/shapley_basic_spotify/index.html
@@ -3161,11 +3161,11 @@ <h2 id="evaluation-on-anomalous-data">Evaluation on anomalous data<a class="head
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/examples/shapley_knn_flowers/index.html b/devel/examples/shapley_knn_flowers/index.html
index 03ffe3890..871f3aefc 100644
--- a/devel/examples/shapley_knn_flowers/index.html
+++ b/devel/examples/shapley_knn_flowers/index.html
@@ -2819,11 +2819,11 @@ <h2 id="corrupting-labels">Corrupting labels<a class="headerlink" href="#corrupt
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/examples/shapley_utility_learning/index.html b/devel/examples/shapley_utility_learning/index.html
index 1fe134178..feedc4a1c 100644
--- a/devel/examples/shapley_utility_learning/index.html
+++ b/devel/examples/shapley_utility_learning/index.html
@@ -3032,11 +3032,11 @@ <h2 id="evaluation-on-anomalous-data">Evaluation on anomalous data<a class="head
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/getting-started/first-steps/index.html b/devel/getting-started/first-steps/index.html
index cacc09c84..b98b0dfa7 100644
--- a/devel/getting-started/first-steps/index.html
+++ b/devel/getting-started/first-steps/index.html
@@ -2588,11 +2588,11 @@ <h2 id="parallelization">Parallelization<a class="headerlink" href="#paralleliza
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/getting-started/installation/index.html b/devel/getting-started/installation/index.html
index 9e34f295c..98b2b0e76 100644
--- a/devel/getting-started/installation/index.html
+++ b/devel/getting-started/installation/index.html
@@ -2575,11 +2575,11 @@ <h2 id="setting-up-the-cache">Setting up the cache<a class="headerlink" href="#s
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/index.html b/devel/index.html
index e0ed5342e..3730fe53f 100644
--- a/devel/index.html
+++ b/devel/index.html
@@ -2466,11 +2466,11 @@ <h1 id="the-python-library-for-data-valuation">The python library for data valua
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/influence/index.html b/devel/influence/index.html
index 7f62c5b51..c634af32f 100644
--- a/devel/influence/index.html
+++ b/devel/influence/index.html
@@ -3125,11 +3125,11 @@ <h3 id="arnoldi-solver">Arnoldi solver<a class="headerlink" href="#arnoldi-solve
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/objects.inv b/devel/objects.inv
index dbb0ee2a0..6ddc62c0d 100644
Binary files a/devel/objects.inv and b/devel/objects.inv differ
diff --git a/devel/search/search_index.json b/devel/search/search_index.json
index 83512c13e..99e9df878 100644
--- a/devel/search/search_index.json
+++ b/devel/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"The python library for data valuation","text":"<p>pyDVL collects algorithms for data valuation and influence function computation. It runs most of them in parallel either locally or in a cluster and supports distributed caching of results.</p> <p>If you're a first time user of pyDVL, we recommend you to go through the Getting Started and Installing pyDVL guides.</p> <p>Installation</p><p>Steps to install and requirements</p> <p>Data valuation</p><p>Basics of data valuation and description of the main algorithms </p> <p>Influence Function</p><p>An introduction to the influence function and its computation with pyDVL </p> <p>Browse the API</p><p>Full documentation of the API</p>"},{"location":"CHANGELOG/","title":"Changelog","text":""},{"location":"CHANGELOG/#unreleased","title":"Unreleased","text":"<ul> <li>Refactoring of parallel module. Old imports will stop working in v0.9.0   PR #421</li> </ul>"},{"location":"CHANGELOG/#070-documentation-and-if-overhaul-new-methods-and-bug-fixes","title":"0.7.0 - \ud83d\udcda\ud83c\udd95 Documentation and IF overhaul, new methods and bug fixes \ud83d\udca5\ud83d\udc1e","text":"<p>This is our first \u03b2 release! We have worked hard to deliver improvements across the board, with a focus on documentation and usability. We have also reworked the internals of the <code>influence</code> module, improved parallelism and handling of randomness.</p>"},{"location":"CHANGELOG/#added","title":"Added","text":"<ul> <li>Implemented solving the Hessian equation via spectral low-rank approximation   PR #365</li> <li>Enabled parallel computation for Leave-One-Out values   PR #406</li> <li>Added more abbreviations to documentation   PR #415</li> <li>Added seed to functions from <code>pydvl.utils.numeric</code>, <code>pydvl.value.shapley</code> and   <code>pydvl.value.semivalues</code>. Introduced new type <code>Seed</code> and conversion function    <code>ensure_seed_sequence</code>.   PR #396</li> </ul>"},{"location":"CHANGELOG/#changed","title":"Changed","text":"<ul> <li>Replaced sphinx with mkdocs for documentation. Major overhaul of documentation   PR #352</li> <li>Made ray an optional dependency, relying on joblib as default parallel backend   PR #408</li> <li>Decoupled <code>ray.init</code> from <code>ParallelConfig</code> PR #373</li> <li>Breaking Changes</li> <li>Signature change: return information about Hessian inversion from     <code>compute_influence_factors</code> PR #375</li> <li>Major changes to IF interface and functionality. Foundation for a framework     abstraction for IF computation.     PR #278 PR #394</li> <li>Renamed <code>semivalues</code> to <code>compute_generic_semivalues</code> PR #413</li> <li>New <code>joblib</code> backend as default instead of ray. Simplify MapReduceJob.     PR #355</li> <li>Bump torch dependency for influence package to 2.0     PR #365</li> </ul>"},{"location":"CHANGELOG/#fixed","title":"Fixed","text":"<ul> <li>Fixes to parallel computation of generic semi-values: properly handle all   samplers and stopping criteria, irrespective of parallel backend.   PR #372</li> <li>Optimises memory usage in IF calculation   PR #375</li> <li>Fix adding valuation results with overlapping indices and different lengths   PR #370</li> <li>Fixed bugs in conjugate gradient and <code>linear_solve</code> PR #358</li> <li>Fix installation of dev requirements for Python3.10   PR #382</li> <li>Improvements to IF documentation   PR #371</li> </ul>"},{"location":"CHANGELOG/#061-bug-fixes-and-small-improvements","title":"0.6.1 - \ud83c\udfd7 Bug fixes and small improvements","text":"<ul> <li>Fix parsing keyword arguments of <code>compute_semivalues</code> dispatch function   PR #333</li> <li>Create new <code>RayExecutor</code> class based on the concurrent.futures API,   use the new class to fix an issue with Truncated Monte Carlo Shapley   (TMCS) starting too many processes and dying, plus other small changes   PR #329</li> <li>Fix creation of GroupedDataset objects using the <code>from_arrays</code>   and <code>from_sklearn</code> class methods    PR #324</li> <li>Fix release job not triggering on CI when a new tag is pushed   PR #331</li> <li>Added alias <code>ApproShapley</code> from Castro et al. 2009 for permutation Shapley   PR #332</li> </ul>"},{"location":"CHANGELOG/#060-new-algorithms-cleanup-and-bug-fixes","title":"0.6.0 - \ud83c\udd95 New algorithms, cleanup and bug fixes \ud83c\udfd7","text":"<ul> <li>Fixes in <code>ValuationResult</code>: bugs around data names, semantics of   <code>empty()</code>, new method <code>zeros()</code> and normalised random values   PR #327</li> <li>New method: Implements generalised semi-values for data valuation,   including Data Banzhaf and Beta Shapley, with configurable sampling strategies   PR #319</li> <li>Adds kwargs parameter to <code>from_array</code> and <code>from_sklearn</code> Dataset and   GroupedDataset class methods   PR #316</li> <li>PEP-561 conformance: added <code>py.typed</code> PR #307</li> <li>Removed default non-negativity constraint on least core subsidy   and added instead a <code>non_negative_subsidy</code> boolean flag.   Renamed <code>options</code> to <code>solver_options</code> and pass it as dict.   Change default least-core solver to SCS with 10000 max_iters.   PR #304</li> <li>Cleanup: removed unnecessary decorator <code>@unpackable</code> PR #233</li> <li>Stopping criteria: fixed problem with <code>StandardError</code> and enable proper   composition of index convergence statuses. Fixed a bug with <code>n_jobs</code> in   <code>truncated_montecarlo_shapley</code>.   PR #300 and   PR #305</li> <li>Shuffling code around to allow for simpler user imports, some cleanup and   documentation fixes.   PR #284</li> <li>Bug fix: Warn instead of raising an error when <code>n_iterations</code>   is less than the size of the dataset in Monte Carlo Least Core   PR #281</li> </ul>"},{"location":"CHANGELOG/#050-fixes-nicer-interfaces-and-more-breaking-changes","title":"0.5.0 - \ud83d\udca5 Fixes, nicer interfaces and... more breaking changes \ud83d\ude12","text":"<ul> <li>Fixed parallel and antithetic Owen sampling for Shapley values. Simplified   and extended tests.   PR #267</li> <li>Added <code>Scorer</code> class for a cleaner interface. Fixed minor bugs around   Group-Testing Shapley, added more tests and switched to cvxpy for the solver.   PR #264</li> <li>Generalised stopping criteria for valuation algorithms. Improved classes   <code>ValuationResult</code> and <code>Status</code> with more operations. Some minor issues fixed.   PR #252</li> <li>Fixed a bug whereby <code>compute_shapley_values</code> would only spawn one process when   using <code>n_jobs=-1</code> and Monte Carlo methods.   PR #270</li> <li>Bugfix in <code>RayParallelBackend</code>: wrong semantics for <code>kwargs</code>.   PR #268</li> <li>Splitting of problem preparation and solution in Least-Core computation.   Umbrella function for LC methods.   PR #257 </li> <li>Operations on <code>ValuationResult</code> and <code>Status</code> and some cleanup   PR #248</li> <li>Bug fix and minor improvements: Fixes bug in TMCS with remote Ray cluster,   raises an error for dummy sequential parallel backend with TMCS, clones model   inside <code>Utility</code> before fitting by default, with flag <code>clone_before_fit</code>    to disable it, catches all warnings in <code>Utility</code> when <code>show_warnings</code> is    <code>False</code>. Adds Miner and Gloves toy games utilities   PR #247</li> </ul>"},{"location":"CHANGELOG/#040-new-algorithms-and-more-breaking-changes","title":"0.4.0 - \ud83c\udfed\ud83d\udca5 New algorithms and more breaking changes","text":"<ul> <li>GH action to mark issues as stale   PR #201</li> <li>Disabled caching of Utility values as well as repeated evaluations by default   PR #211</li> <li>Test and officially support Python version 3.9 and 3.10    PR #208</li> <li>Breaking change: Introduces a class ValuationResult to gather and inspect   results from all valuation algorithms   PR #214</li> <li>Fixes bug in Influence calculation with multidimensional input and adds new   example notebook   PR #195</li> <li>Breaking change: Passes the input to <code>MapReduceJob</code> at initialization,   removes <code>chunkify_inputs</code> argument from <code>MapReduceJob</code>, removes <code>n_runs</code>   argument from <code>MapReduceJob</code>, calls the parallel backend's <code>put()</code> method for   each generated chunk in <code>_chunkify()</code>, renames ParallelConfig's <code>num_workers</code>   attribute to <code>n_local_workers</code>, fixes a bug in <code>MapReduceJob</code>'s chunkification   when <code>n_runs</code> &gt;= <code>n_jobs</code>, and defines a sequential parallel backend to run   all jobs in the current thread   PR #232</li> <li>New method: Implements exact and monte carlo Least Core for data valuation,   adds <code>from_arrays()</code> class method to the <code>Dataset</code> and <code>GroupedDataset</code>   classes, adds <code>extra_values</code> argument to <code>ValuationResult</code>, adds   <code>compute_removal_score()</code> and <code>compute_random_removal_score()</code> helper functions   PR #237</li> <li>New method: Group Testing Shapley for valuation, from Jia et al. 2019 PR #240</li> <li>Fixes bug in ray initialization in <code>RayParallelBackend</code> class   PR #239</li> <li>Implements \"Egalitarian Least Core\", adds cvxpy as a   dependency and uses it instead of scipy as optimizer   PR #243</li> </ul>"},{"location":"CHANGELOG/#030-breaking-changes","title":"0.3.0 - \ud83d\udca5 Breaking changes","text":"<ul> <li>Simplified and fixed powerset sampling and testing   PR #181</li> <li>Simplified and fixed publishing to PyPI from CI   PR #183</li> <li>Fixed bug in release script and updated contributing docs.   PR #184</li> <li>Added Pull Request template   PR #185</li> <li>Modified Pull Request template to automatically link PR to issue   PR ##186</li> <li>First implementation of Owen Sampling, squashed scores, better testing   PR #194</li> <li>Improved documentation on caching, Shapley, caveats of values, bibtex   PR #194</li> <li>Breaking change: Rearranging of modules to accommodate for new methods   PR #194</li> </ul>"},{"location":"CHANGELOG/#020-better-docs","title":"0.2.0 - \ud83d\udcda Better docs","text":"<p>Mostly API documentation and notebooks, plus some bugfixes.</p>"},{"location":"CHANGELOG/#added_1","title":"Added","text":"<p>In PR #161: - Support for $$ math in sphinx docs. - Usage of sphinx extension for external links (introducing new directives like   <code>:gh:</code>, <code>:issue:</code> and <code>:tfl:</code> to construct standardised links to external   resources). - Only update auto-generated documentation files if there are changes. Some   minor additions to <code>update_docs.py</code>. - Parallelization of exact combinatorial Shapley. - Integrated KNN shapley into the main interface <code>compute_shapley_values</code>.</p>"},{"location":"CHANGELOG/#changed_1","title":"Changed","text":"<p>In PR #161: - Improved main docs and Shapley notebooks. Added or fixed many docstrings,   readme and documentation for contributors. Typos, grammar and style in code,   documentation and notebooks. - Internal renaming and rearranging in the parallelization and caching modules.</p>"},{"location":"CHANGELOG/#fixed_1","title":"Fixed","text":"<ul> <li>Bug in random matrix generation   PR #161.</li> <li>Bugs in MapReduceJob's <code>_chunkify</code> and <code>_backpressure</code> methods   PR #176.</li> </ul>"},{"location":"CHANGELOG/#010-first-release","title":"0.1.0 - \ud83c\udf89 first release","text":"<p>This is very first release of pyDVL.</p> <p>It contains:</p> <ul> <li> <p>Data Valuation Methods:</p> </li> <li> <p>Leave-One-Out</p> </li> <li>Influence Functions</li> <li>Shapley:<ul> <li>Exact Permutation and Combinatorial</li> <li>Montecarlo Permutation and Combinatorial</li> <li>Truncated Montecarlo Permutation</li> </ul> </li> <li>Caching of results with Memcached</li> <li>Parallelization of computations with Ray</li> <li>Documentation</li> <li>Notebooks containing examples of different use cases</li> </ul>"},{"location":"api/pydvl/","title":"API","text":""},{"location":"api/pydvl/#pydvl--the-python-data-valuation-library-api","title":"The Python Data Valuation Library API","text":"<p>This is the API documentation for the Python Data Valuation Library (PyDVL). Use the table of contents to access the documentation for each module.</p> <p>The two main modules you will want to look at are value and influence.</p>"},{"location":"api/pydvl/influence/","title":"Influence","text":"<p>This package contains algorithms for the computation of the influence function.</p> <p>Warning: Much of the code in this package is experimental or untested and is subject to modification. In particular, the package structure and basic API will probably change.</p>"},{"location":"api/pydvl/influence/general/","title":"General","text":"<p>This module contains influence calculation functions for general models, as introduced in (Koh and Liang, 2017)<sup>1</sup>.</p>"},{"location":"api/pydvl/influence/general/#pydvl.influence.general--references","title":"References","text":"<ol> <li> <p>Koh, P.W., Liang, P., 2017. Understanding Black-box Predictions via Influence Functions. In: Proceedings of the 34th International Conference on Machine Learning, pp. 1885\u20131894. PMLR.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/influence/general/#pydvl.influence.general.InfluenceType","title":"<code>InfluenceType</code>","text":"<p>             Bases: <code>str</code>, <code>Enum</code></p> <p>Enum representation for the types of influence.</p> ATTRIBUTE DESCRIPTION <code>Up</code> <p>Up-weighting a training point, see section 2.1 of (Koh and Liang, 2017)<sup>1</sup></p> <p> </p> <code>Perturbation</code> <p>Perturb a training point, see section 2.2 of (Koh and Liang, 2017)<sup>1</sup></p> <p> </p>"},{"location":"api/pydvl/influence/general/#pydvl.influence.general.compute_influence_factors","title":"<code>compute_influence_factors(model, training_data, test_data, inversion_method, *, hessian_perturbation=0.0, progress=False, **kwargs)</code>","text":"<p>Calculates influence factors of a model for training and test data.</p> <p>Given a test point \\(z_{test} = (x_{test}, y_{test})\\), a loss \\(L(z_{test}, \\theta)\\) (\\(\\theta\\) being the parameters of the model) and the Hessian of the model \\(H_{\\theta}\\), influence factors are defined as:</p> \\[ s_{test} = H_{\\theta}^{-1} \\operatorname{grad}_{\\theta} L(z_{test}, \\theta). \\] <p>They are used for efficient influence calculation. This method first (implicitly) calculates the Hessian and then (explicitly) finds the influence factors for the model using the given inversion method. The parameter <code>hessian_perturbation</code> is used to regularize the inversion of the Hessian. For more info, refer to (Koh and Liang, 2017)<sup>1</sup>, paragraph 3.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A model wrapped in the TwiceDifferentiable interface.</p> <p> TYPE: <code>TwiceDifferentiable</code> </p> <code>training_data</code> <p>DataLoader containing the training data.</p> <p> TYPE: <code>DataLoaderType</code> </p> <code>test_data</code> <p>DataLoader containing the test data.</p> <p> TYPE: <code>DataLoaderType</code> </p> <code>inversion_method</code> <p>Name of method for computing inverse hessian vector products.</p> <p> TYPE: <code>InversionMethod</code> </p> <code>hessian_perturbation</code> <p>Regularization of the hessian.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>progress</code> <p>If True, display progress bars.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>array</code> <p>An array of size (N, D) containing the influence factors for each dimension (D) and test sample (N).</p> <p> TYPE: <code>InverseHvpResult</code> </p> Source code in <code>src/pydvl/influence/general.py</code> <pre><code>def compute_influence_factors(\nmodel: TwiceDifferentiable,\ntraining_data: DataLoaderType,\ntest_data: DataLoaderType,\ninversion_method: InversionMethod,\n*,\nhessian_perturbation: float = 0.0,\nprogress: bool = False,\n**kwargs: Any,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Calculates influence factors of a model for training and test data.\n    Given a test point \\(z_{test} = (x_{test}, y_{test})\\), a loss \\(L(z_{test}, \\theta)\\)\n    (\\(\\theta\\) being the parameters of the model) and the Hessian of the model \\(H_{\\theta}\\),\n    influence factors are defined as:\n    \\[\n    s_{test} = H_{\\theta}^{-1} \\operatorname{grad}_{\\theta} L(z_{test}, \\theta).\n    \\]\n    They are used for efficient influence calculation. This method first (implicitly) calculates\n    the Hessian and then (explicitly) finds the influence factors for the model using the given\n    inversion method. The parameter `hessian_perturbation` is used to regularize the inversion of\n    the Hessian. For more info, refer to (Koh and Liang, 2017)&lt;sup&gt;&lt;a href=\"#koh_liang_2017\"&gt;1&lt;/a&gt;&lt;/sup&gt;, paragraph 3.\n    Args:\n        model: A model wrapped in the TwiceDifferentiable interface.\n        training_data: DataLoader containing the training data.\n        test_data: DataLoader containing the test data.\n        inversion_method: Name of method for computing inverse hessian vector products.\n        hessian_perturbation: Regularization of the hessian.\n        progress: If True, display progress bars.\n    Returns:\n        array: An array of size (N, D) containing the influence factors for each dimension (D) and test sample (N).\n    \"\"\"\ntensor_util: Type[TensorUtilities] = TensorUtilities.from_twice_differentiable(\nmodel\n)\nstack = tensor_util.stack\nunsqueeze = tensor_util.unsqueeze\ncat_gen = tensor_util.cat_gen\ncat = tensor_util.cat\ndef test_grads() -&gt; Generator[TensorType, None, None]:\nfor x_test, y_test in maybe_progress(\ntest_data, progress, desc=\"Batch Test Gradients\"\n):\nyield stack(\n[\nmodel.grad(inpt, target)\nfor inpt, target in zip(unsqueeze(x_test, 1), y_test)\n]\n)  # type:ignore\ntry:\n# if provided input_data implements __len__, pre-allocate the result tensor to reduce memory consumption\nresulting_shape = (len(test_data), model.num_params)  # type:ignore\nrhs = cat_gen(\ntest_grads(), resulting_shape, model  # type:ignore\n)  # type:ignore\nexcept Exception as e:\nlogger.warning(\nf\"Failed to pre-allocate result tensor: {e}\\n\"\nf\"Evaluate all resulting tensor and concatenate\"\n)\nrhs = cat(list(test_grads()))\nreturn solve_hvp(\ninversion_method,\nmodel,\ntraining_data,\nrhs,\nhessian_perturbation=hessian_perturbation,\n**kwargs,\n)\n</code></pre>"},{"location":"api/pydvl/influence/general/#pydvl.influence.general.compute_influences_up","title":"<code>compute_influences_up(model, input_data, influence_factors, *, progress=False)</code>","text":"<p>Given the model, the training points, and the influence factors, this function calculates the influences using the up-weighting method.</p> <p>The procedure involves two main steps: 1. Calculating the gradients of the model with respect to each training sample    (\\(\\operatorname{grad}_{\\theta} L\\), where \\(L\\) is the loss of a single point and \\(\\theta\\) are the    parameters of the model). 2. Multiplying each gradient with the influence factors.</p> <p>For a detailed description of the methodology, see section 2.1 of (Koh and Liang, 2017)<sup>1</sup>.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A model that implements the TwiceDifferentiable interface.</p> <p> TYPE: <code>TwiceDifferentiable</code> </p> <code>input_data</code> <p>DataLoader containing the samples for which the influence will be calculated.</p> <p> TYPE: <code>DataLoaderType</code> </p> <code>influence_factors</code> <p>Array containing pre-computed influence factors.</p> <p> TYPE: <code>TensorType</code> </p> <code>progress</code> <p>If set to True, progress bars will be displayed during computation.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>TensorType</code> <p>An array of shape [NxM], where N is the number of influence factors, and M is the number of input samples.</p> Source code in <code>src/pydvl/influence/general.py</code> <pre><code>def compute_influences_up(\nmodel: TwiceDifferentiable,\ninput_data: DataLoaderType,\ninfluence_factors: TensorType,\n*,\nprogress: bool = False,\n) -&gt; TensorType:\nr\"\"\"\n    Given the model, the training points, and the influence factors, this function calculates the\n    influences using the up-weighting method.\n    The procedure involves two main steps:\n    1. Calculating the gradients of the model with respect to each training sample\n       (\\(\\operatorname{grad}_{\\theta} L\\), where \\(L\\) is the loss of a single point and \\(\\theta\\) are the\n       parameters of the model).\n    2. Multiplying each gradient with the influence factors.\n    For a detailed description of the methodology, see section 2.1 of (Koh and Liang, 2017)&lt;sup&gt;&lt;a href=\"#koh_liang_2017\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    Args:\n        model: A model that implements the TwiceDifferentiable interface.\n        input_data: DataLoader containing the samples for which the influence will be calculated.\n        influence_factors: Array containing pre-computed influence factors.\n        progress: If set to True, progress bars will be displayed during computation.\n    Returns:\n        An array of shape [NxM], where N is the number of influence factors, and M is the number of input samples.\n    \"\"\"\ntensor_util: Type[TensorUtilities] = TensorUtilities.from_twice_differentiable(\nmodel\n)\nstack = tensor_util.stack\nunsqueeze = tensor_util.unsqueeze\ncat_gen = tensor_util.cat_gen\ncat = tensor_util.cat\neinsum = tensor_util.einsum\ndef train_grads() -&gt; Generator[TensorType, None, None]:\nfor x, y in maybe_progress(\ninput_data, progress, desc=\"Batch Split Input Gradients\"\n):\nyield stack(\n[model.grad(inpt, target) for inpt, target in zip(unsqueeze(x, 1), y)]\n)  # type:ignore\ntry:\n# if provided input_data implements __len__, pre-allocate the result tensor to reduce memory consumption\nresulting_shape = (len(input_data), model.num_params)  # type:ignore\ntrain_grad_tensor = cat_gen(\ntrain_grads(), resulting_shape, model  # type:ignore\n)  # type:ignore\nexcept Exception as e:\nlogger.warning(\nf\"Failed to pre-allocate result tensor: {e}\\n\"\nf\"Evaluate all resulting tensor and concatenate\"\n)\ntrain_grad_tensor = cat([x for x in train_grads()])  # type:ignore\nreturn einsum(\"ta,va-&gt;tv\", influence_factors, train_grad_tensor)  # type:ignore\n</code></pre>"},{"location":"api/pydvl/influence/general/#pydvl.influence.general.compute_influences_pert","title":"<code>compute_influences_pert(model, input_data, influence_factors, *, progress=False)</code>","text":"<p>Calculates the influence values based on the influence factors and training samples using the perturbation method.</p> <p>The process involves two main steps: 1. Calculating the gradient of the model with respect to each training sample    (\\(\\operatorname{grad}_{\\theta} L\\), where \\(L\\) is the loss of the model for a single data point and \\(\\theta\\)    are the parameters of the model). 2. Using the method TwiceDifferentiable.mvp    to efficiently compute the product of the    influence factors and \\(\\operatorname{grad}_x \\operatorname{grad}_{\\theta} L\\).</p> <p>For a detailed methodology, see section 2.2 of (Koh and Liang, 2017)<sup>1</sup>.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A model that implements the TwiceDifferentiable interface.</p> <p> TYPE: <code>TwiceDifferentiable</code> </p> <code>input_data</code> <p>DataLoader containing the samples for which the influence will be calculated.</p> <p> TYPE: <code>DataLoaderType</code> </p> <code>influence_factors</code> <p>Array containing pre-computed influence factors.</p> <p> TYPE: <code>TensorType</code> </p> <code>progress</code> <p>If set to True, progress bars will be displayed during computation.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>TensorType</code> <p>A 3D array with shape [NxMxP], where N is the number of influence factors, M is the number of input samples, and P is the number of features.</p> Source code in <code>src/pydvl/influence/general.py</code> <pre><code>def compute_influences_pert(\nmodel: TwiceDifferentiable,\ninput_data: DataLoaderType,\ninfluence_factors: TensorType,\n*,\nprogress: bool = False,\n) -&gt; TensorType:\nr\"\"\"\n    Calculates the influence values based on the influence factors and training samples using the perturbation method.\n    The process involves two main steps:\n    1. Calculating the gradient of the model with respect to each training sample\n       (\\(\\operatorname{grad}_{\\theta} L\\), where \\(L\\) is the loss of the model for a single data point and \\(\\theta\\)\n       are the parameters of the model).\n    2. Using the method [TwiceDifferentiable.mvp][pydvl.influence.twice_differentiable.TwiceDifferentiable.mvp]\n       to efficiently compute the product of the\n       influence factors and \\(\\operatorname{grad}_x \\operatorname{grad}_{\\theta} L\\).\n    For a detailed methodology, see section 2.2 of (Koh and Liang, 2017)&lt;sup&gt;&lt;a href=\"#koh_liang_2017\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    Args:\n        model: A model that implements the TwiceDifferentiable interface.\n        input_data: DataLoader containing the samples for which the influence will be calculated.\n        influence_factors: Array containing pre-computed influence factors.\n        progress: If set to True, progress bars will be displayed during computation.\n    Returns:\n        A 3D array with shape [NxMxP], where N is the number of influence factors,\n            M is the number of input samples, and P is the number of features.\n    \"\"\"\ntensor_util: Type[TensorUtilities] = TensorUtilities.from_twice_differentiable(\nmodel\n)\nstack = tensor_util.stack\ntu_slice = tensor_util.slice\nreshape = tensor_util.reshape\nget_element = tensor_util.get_element\nshape = tensor_util.shape\nall_pert_influences = []\nfor x, y in maybe_progress(\ninput_data,\nprogress,\ndesc=\"Batch Influence Perturbation\",\n):\nfor i in range(len(x)):\ntensor_x = tu_slice(x, i, i + 1)\ngrad_xy = model.grad(tensor_x, get_element(y, i), create_graph=True)\nperturbation_influences = model.mvp(\ngrad_xy,\ninfluence_factors,\nbackprop_on=tensor_x,\n)\nall_pert_influences.append(\nreshape(perturbation_influences, (-1, *shape(get_element(x, i))))\n)\nreturn stack(all_pert_influences, axis=1)  # type:ignore\n</code></pre>"},{"location":"api/pydvl/influence/general/#pydvl.influence.general.compute_influences","title":"<code>compute_influences(differentiable_model, training_data, *, test_data=None, input_data=None, inversion_method=InversionMethod.Direct, influence_type=InfluenceType.Up, hessian_regularization=0.0, progress=False, **kwargs)</code>","text":"<p>Calculates the influence of each input data point on the specified test points.</p> <p>This method operates in two primary stages: 1. Computes the influence factors for all test points concerning the model and its training data. 2. Uses these factors to derive the influences over the complete set of input data.</p> <p>The influence calculation relies on the twice-differentiable nature of the provided model.</p> PARAMETER  DESCRIPTION <code>differentiable_model</code> <p>A model bundled with its corresponding loss in the <code>TwiceDifferentiable</code> wrapper.</p> <p> TYPE: <code>TwiceDifferentiable</code> </p> <code>training_data</code> <p>DataLoader instance supplying the training data. This data is pivotal in computing the            Hessian matrix for the model's loss.</p> <p> TYPE: <code>DataLoaderType</code> </p> <code>test_data</code> <p>DataLoader instance with the test samples. Defaults to <code>training_data</code> if None.</p> <p> TYPE: <code>Optional[DataLoaderType]</code> DEFAULT: <code>None</code> </p> <code>input_data</code> <p>DataLoader instance holding samples whose influences need to be computed. Defaults to         <code>training_data</code> if None.</p> <p> TYPE: <code>Optional[DataLoaderType]</code> DEFAULT: <code>None</code> </p> <code>inversion_method</code> <p>An enumeration value determining the approach for inverting matrices or computing inverse operations, see [.inversion.InversionMethod]</p> <p> TYPE: <code>InversionMethod</code> DEFAULT: <code>Direct</code> </p> <code>progress</code> <p>A boolean indicating whether progress bars should be displayed during computation.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>influence_type</code> <p>Determines the methodology for computing influences. Valid choices include 'up' (for up-weighting) and 'perturbation'. For an in-depth understanding, see (Koh and Liang, 2017)<sup>1</sup>.</p> <p> TYPE: <code>InfluenceType</code> DEFAULT: <code>Up</code> </p> <code>hessian_regularization</code> <p>A lambda value used in Hessian regularization. The regularized Hessian, \\( H_{reg} \\), is computed as \\( H + \\lambda \\times I \\), where \\( I \\) is the identity matrix and \\( H \\) is the simple, unmodified Hessian. This regularization is typically utilized for more sophisticated models to ensure that the Hessian remains positive definite.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> RETURNS DESCRIPTION <code>TensorType</code> <p>The shape of this array varies based on the <code>influence_type</code>. If 'up', the shape is [NxM], where N denotes the number of test points and M denotes the number of training points. Conversely, if the influence_type is 'perturbation', the shape is [NxMxP], with P representing the number of input features.</p> Source code in <code>src/pydvl/influence/general.py</code> <pre><code>def compute_influences(\ndifferentiable_model: TwiceDifferentiable,\ntraining_data: DataLoaderType,\n*,\ntest_data: Optional[DataLoaderType] = None,\ninput_data: Optional[DataLoaderType] = None,\ninversion_method: InversionMethod = InversionMethod.Direct,\ninfluence_type: InfluenceType = InfluenceType.Up,\nhessian_regularization: float = 0.0,\nprogress: bool = False,\n**kwargs: Any,\n) -&gt; TensorType:  # type: ignore # ToDO fix typing\nr\"\"\"\n    Calculates the influence of each input data point on the specified test points.\n    This method operates in two primary stages:\n    1. Computes the influence factors for all test points concerning the model and its training data.\n    2. Uses these factors to derive the influences over the complete set of input data.\n    The influence calculation relies on the twice-differentiable nature of the provided model.\n    Args:\n        differentiable_model: A model bundled with its corresponding loss in the `TwiceDifferentiable` wrapper.\n        training_data: DataLoader instance supplying the training data. This data is pivotal in computing the\n                       Hessian matrix for the model's loss.\n        test_data: DataLoader instance with the test samples. Defaults to `training_data` if None.\n        input_data: DataLoader instance holding samples whose influences need to be computed. Defaults to\n                    `training_data` if None.\n        inversion_method: An enumeration value determining the approach for inverting matrices\n            or computing inverse operations, see [.inversion.InversionMethod]\n        progress: A boolean indicating whether progress bars should be displayed during computation.\n        influence_type: Determines the methodology for computing influences.\n            Valid choices include 'up' (for up-weighting) and 'perturbation'.\n            For an in-depth understanding, see (Koh and Liang, 2017)&lt;sup&gt;&lt;a href=\"#koh_liang_2017\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n        hessian_regularization: A lambda value used in Hessian regularization. The regularized Hessian, \\( H_{reg} \\),\n            is computed as \\( H + \\lambda \\times I \\), where \\( I \\) is the identity matrix and \\( H \\)\n            is the simple, unmodified Hessian. This regularization is typically utilized for more\n            sophisticated models to ensure that the Hessian remains positive definite.\n    Returns:\n        The shape of this array varies based on the `influence_type`. If 'up', the shape is [NxM], where\n            N denotes the number of test points and M denotes the number of training points. Conversely, if the\n            influence_type is 'perturbation', the shape is [NxMxP], with P representing the number of input features.\n    \"\"\"\nif input_data is None:\ninput_data = deepcopy(training_data)\nif test_data is None:\ntest_data = deepcopy(training_data)\ninfluence_factors, _ = compute_influence_factors(\ndifferentiable_model,\ntraining_data,\ntest_data,\ninversion_method,\nhessian_perturbation=hessian_regularization,\nprogress=progress,\n**kwargs,\n)\nreturn influence_type_registry[influence_type](\ndifferentiable_model,\ninput_data,\ninfluence_factors,\nprogress=progress,\n)\n</code></pre>"},{"location":"api/pydvl/influence/inversion/","title":"Inversion","text":"<p>Contains methods to invert the hessian vector product.</p>"},{"location":"api/pydvl/influence/inversion/#pydvl.influence.inversion.InversionMethod","title":"<code>InversionMethod</code>","text":"<p>             Bases: <code>str</code>, <code>Enum</code></p> <p>Different inversion methods types.</p>"},{"location":"api/pydvl/influence/inversion/#pydvl.influence.inversion.InversionRegistry","title":"<code>InversionRegistry</code>","text":"<p>A registry to hold inversion methods for different models.</p>"},{"location":"api/pydvl/influence/inversion/#pydvl.influence.inversion.InversionRegistry.register","title":"<code>register(model_type, inversion_method, overwrite=False)</code>  <code>classmethod</code>","text":"<p>Register a function for a specific model type and inversion method.</p> <p>The function to be registered must conform to the following signature: <code>(model: TwiceDifferentiable, training_data: DataLoaderType, b: TensorType, hessian_perturbation: float = 0.0, ...)</code>.</p> PARAMETER  DESCRIPTION <code>model_type</code> <p>The type of the model the function should be registered for.</p> <p> TYPE: <code>Type[TwiceDifferentiable]</code> </p> <code>inversion_method</code> <p>The inversion method the function should be registered for.</p> <p> TYPE: <code>InversionMethod</code> </p> <code>overwrite</code> <p>If <code>True</code>, allows overwriting of an existing registered function for the same model type and inversion method. If <code>False</code>, logs a warning when attempting to register a function for an already registered model type and inversion method.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RAISES DESCRIPTION <code>TypeError</code> <p>If the provided model_type or inversion_method are of the wrong type.</p> <code>ValueError</code> <p>If the function to be registered does not match the required signature.</p> RETURNS DESCRIPTION <p>A decorator for registering a function.</p> Source code in <code>src/pydvl/influence/inversion.py</code> <pre><code>@classmethod\ndef register(\ncls,\nmodel_type: Type[TwiceDifferentiable],\ninversion_method: InversionMethod,\noverwrite: bool = False,\n):\n\"\"\"\n    Register a function for a specific model type and inversion method.\n    The function to be registered must conform to the following signature:\n    `(model: TwiceDifferentiable, training_data: DataLoaderType, b: TensorType,\n    hessian_perturbation: float = 0.0, ...)`.\n    Args:\n        model_type: The type of the model the function should be registered for.\n        inversion_method: The inversion method the function should be\n            registered for.\n        overwrite: If ``True``, allows overwriting of an existing registered\n            function for the same model type and inversion method. If ``False``,\n            logs a warning when attempting to register a function for an already\n            registered model type and inversion method.\n    Raises:\n        TypeError: If the provided model_type or inversion_method are of the wrong type.\n        ValueError: If the function to be registered does not match the required signature.\n    Returns:\n        A decorator for registering a function.\n    \"\"\"\nif not isinstance(model_type, type):\nraise TypeError(\nf\"'model_type' is of type {type(model_type)} but should be a Type[TwiceDifferentiable]\"\n)\nif not isinstance(inversion_method, InversionMethod):\nraise TypeError(\nf\"'inversion_method' must be an 'InversionMethod' \"\nf\"but has type {type(inversion_method)} instead.\"\n)\nkey = (model_type, inversion_method)\ndef decorator(func):\nif not overwrite and key in cls.registry:\nwarnings.warn(\nf\"There is already a function registered for model type {model_type} \"\nf\"and inversion method {inversion_method}. \"\nf\"To overwrite the existing function {cls.registry.get(key)} with {func}, set overwrite to True.\"\n)\nsig = inspect.signature(func)\nparams = list(sig.parameters.values())\nexpected_args = [\n(\"model\", model_type),\n(\"training_data\", DataLoaderType.__bound__),\n(\"b\", model_type.tensor_type()),\n(\"hessian_perturbation\", float),\n]\nfor (name, typ), param in zip(expected_args, params):\nif not (\nisinstance(param.annotation, typ)\nor issubclass(param.annotation, typ)\n):\nraise ValueError(\nf'Parameter \"{name}\" must be of type \"{typ.__name__}\"'\n)\n@functools.wraps(func)\ndef wrapper(*args, **kwargs):\nreturn func(*args, **kwargs)\ncls.registry[key] = wrapper\nreturn wrapper\nreturn decorator\n</code></pre>"},{"location":"api/pydvl/influence/inversion/#pydvl.influence.inversion.InversionRegistry.call","title":"<code>call(inversion_method, model, training_data, b, hessian_perturbation, **kwargs)</code>  <code>classmethod</code>","text":"<p>Call a registered function with the provided parameters.</p> PARAMETER  DESCRIPTION <code>inversion_method</code> <p>The inversion method to use.</p> <p> TYPE: <code>InversionMethod</code> </p> <code>model</code> <p>A model wrapped in the TwiceDifferentiable interface.</p> <p> TYPE: <code>TwiceDifferentiable</code> </p> <code>training_data</code> <p>The training data to use.</p> <p> TYPE: <code>DataLoaderType</code> </p> <code>b</code> <p>Array as the right hand side of the equation \\(Ax = b\\).</p> <p> TYPE: <code>TensorType</code> </p> <code>hessian_perturbation</code> <p>Regularization of the hessian.</p> <p> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the inversion method.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>InverseHvpResult</code> <p>An instance of InverseHvpResult, that contains an array, which solves the inverse problem, i.e. it returns \\(x\\) such that \\(Ax = b\\), and a dictionary containing information about the inversion process.</p> Source code in <code>src/pydvl/influence/inversion.py</code> <pre><code>@classmethod\ndef call(\ncls,\ninversion_method: InversionMethod,\nmodel: TwiceDifferentiable,\ntraining_data: DataLoaderType,\nb: TensorType,\nhessian_perturbation,\n**kwargs,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Call a registered function with the provided parameters.\n    Args:\n        inversion_method: The inversion method to use.\n        model: A model wrapped in the TwiceDifferentiable interface.\n        training_data: The training data to use.\n        b: Array as the right hand side of the equation \\(Ax = b\\).\n        hessian_perturbation: Regularization of the hessian.\n        kwargs: Additional keyword arguments to pass to the inversion method.\n    Returns:\n        An instance of [InverseHvpResult][pydvl.influence.twice_differentiable.InverseHvpResult],\n            that contains an array, which solves the inverse problem,\n            i.e. it returns \\(x\\) such that \\(Ax = b\\), and a dictionary containing information\n            about the inversion process.\n    \"\"\"\nreturn cls.get(type(model), inversion_method)(\nmodel, training_data, b, hessian_perturbation, **kwargs\n)\n</code></pre>"},{"location":"api/pydvl/influence/inversion/#pydvl.influence.inversion.solve_hvp","title":"<code>solve_hvp(inversion_method, model, training_data, b, *, hessian_perturbation=0.0, **kwargs)</code>","text":"<p>Finds \\( x \\) such that \\( Ax = b \\), where \\( A \\) is the hessian of the model, and \\( b \\) a vector. Depending on the inversion method, the hessian is either calculated directly and then inverted, or implicitly and then inverted through matrix vector product. The method also allows to add a small regularization term (hessian_perturbation) to facilitate inversion of non fully trained models.</p> PARAMETER  DESCRIPTION <code>inversion_method</code> <p> TYPE: <code>InversionMethod</code> </p> <code>model</code> <p>A model wrapped in the TwiceDifferentiable interface.</p> <p> TYPE: <code>TwiceDifferentiable</code> </p> <code>training_data</code> <p> TYPE: <code>DataLoaderType</code> </p> <code>b</code> <p>Array as the right hand side of the equation \\( Ax = b \\)</p> <p> TYPE: <code>TensorType</code> </p> <code>hessian_perturbation</code> <p>regularization of the hessian.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>kwargs</code> <p>kwargs to pass to the inversion method.</p> <p> TYPE: <code>Any</code> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>InverseHvpResult</code> <p>Instance of InverseHvpResult, with an array that solves the inverse problem, i.e., it returns \\( x \\) such that \\( Ax = b \\) and a dictionary containing information about the inversion process.</p> Source code in <code>src/pydvl/influence/inversion.py</code> <pre><code>def solve_hvp(\ninversion_method: InversionMethod,\nmodel: TwiceDifferentiable,\ntraining_data: DataLoaderType,\nb: TensorType,\n*,\nhessian_perturbation: float = 0.0,\n**kwargs: Any,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Finds \\( x \\) such that \\( Ax = b \\), where \\( A \\) is the hessian of the model,\n    and \\( b \\) a vector. Depending on the inversion method, the hessian is either\n    calculated directly and then inverted, or implicitly and then inverted through\n    matrix vector product. The method also allows to add a small regularization term\n    (hessian_perturbation) to facilitate inversion of non fully trained models.\n    Args:\n        inversion_method:\n        model: A model wrapped in the TwiceDifferentiable interface.\n        training_data:\n        b: Array as the right hand side of the equation \\( Ax = b \\)\n        hessian_perturbation: regularization of the hessian.\n        kwargs: kwargs to pass to the inversion method.\n    Returns:\n        Instance of [InverseHvpResult][pydvl.influence.twice_differentiable.InverseHvpResult], with\n            an array that solves the inverse problem, i.e., it returns \\( x \\) such that \\( Ax = b \\)\n            and a dictionary containing information about the inversion process.\n    \"\"\"\nreturn InversionRegistry.call(\ninversion_method,\nmodel,\ntraining_data,\nb,\nhessian_perturbation=hessian_perturbation,\n**kwargs,\n)\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/","title":"Twice differentiable","text":""},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorType","title":"<code>TensorType = TypeVar('TensorType', bound=Sequence)</code>  <code>module-attribute</code>","text":"<p>Type variable for tensors, i.e. sequences of numbers</p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.ModelType","title":"<code>ModelType = TypeVar('ModelType', bound='TwiceDifferentiable')</code>  <code>module-attribute</code>","text":"<p>Type variable for twice differentiable models</p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.DataLoaderType","title":"<code>DataLoaderType = TypeVar('DataLoaderType', bound=Iterable)</code>  <code>module-attribute</code>","text":"<p>Type variable for data loaders</p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.InverseHvpResult","title":"<code>InverseHvpResult</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Generic[TensorType]</code></p> <p>Container class for results of solving a problem \\(Ax=b\\)</p> PARAMETER  DESCRIPTION <code>x</code> <p>solution of a problem \\(Ax=b\\)</p> <p> TYPE: <code>TensorType</code> </p> <code>info</code> <p>additional information, to couple with the solution itself</p> <p> TYPE: <code>Dict[str, Any]</code> </p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TwiceDifferentiable","title":"<code>TwiceDifferentiable</code>","text":"<p>             Bases: <code>ABC</code>, <code>Generic[TensorType]</code></p> <p>Abstract base class for wrappers of differentiable models and losses. Meant to be subclassed for each supported framework. Provides methods to compute gradients and second derivative of the loss wrt. the model parameters</p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TwiceDifferentiable.num_params","title":"<code>num_params: int</code>  <code>abstractmethod</code> <code>property</code>","text":"<p>Returns the number of parameters of the model</p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TwiceDifferentiable.parameters","title":"<code>parameters: List[TensorType]</code>  <code>abstractmethod</code> <code>property</code>","text":"<p>Returns all the model parameters that require differentiation</p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TwiceDifferentiable.grad","title":"<code>grad(x, y, create_graph=False)</code>","text":"<p>Calculates gradient of model parameters with respect to the model parameters.</p> PARAMETER  DESCRIPTION <code>x</code> <p>A matrix representing the features \\(x_i\\).</p> <p> TYPE: <code>TensorType</code> </p> <code>y</code> <p>A matrix representing the target values \\(y_i\\).</p> <p> TYPE: <code>TensorType</code> </p> <code>create_graph</code> <p>Used for further differentiation on input parameters.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>TensorType</code> <p>An array with the gradients of the model.</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>def grad(\nself, x: TensorType, y: TensorType, create_graph: bool = False\n) -&gt; TensorType:\nr\"\"\"\n    Calculates gradient of model parameters with respect to the model parameters.\n    Args:\n        x: A matrix representing the features \\(x_i\\).\n        y: A matrix representing the target values \\(y_i\\).\n        create_graph: Used for further differentiation on input parameters.\n    Returns:\n        An array with the gradients of the model.\n    \"\"\"\npass\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TwiceDifferentiable.hessian","title":"<code>hessian(x, y)</code>","text":"<p>Calculates the full Hessian of \\(L(f(x),y)\\) with respect to the model parameters given data \\(x\\) and \\(y\\).</p> PARAMETER  DESCRIPTION <code>x</code> <p>An array representing the features \\(x_i\\).</p> <p> TYPE: <code>TensorType</code> </p> <code>y</code> <p>An array representing the target values \\(y_i\\).</p> <p> TYPE: <code>TensorType</code> </p> RETURNS DESCRIPTION <code>TensorType</code> <p>A tensor representing the Hessian of the model, i.e. the second derivative with respect to the model parameters.</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>def hessian(self, x: TensorType, y: TensorType) -&gt; TensorType:\nr\"\"\"\n    Calculates the full Hessian of \\(L(f(x),y)\\) with respect to the model parameters given data \\(x\\) and \\(y\\).\n    Args:\n        x: An array representing the features \\(x_i\\).\n        y: An array representing the target values \\(y_i\\).\n    Returns:\n        A tensor representing the Hessian of the model, i.e. the second derivative\n            with respect to the model parameters.\n    \"\"\"\npass\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TwiceDifferentiable.mvp","title":"<code>mvp(grad_xy, v, backprop_on, *, progress=False)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Calculates the second order derivative of the model along directions \\(v\\). The second order derivative can be selected through the <code>backprop_on</code> argument.</p> PARAMETER  DESCRIPTION <code>grad_xy</code> <p>An array [P] holding the gradients of the model parameters with respect to input \\(x\\) and labels \\(y\\). \\(P\\) is the number of parameters of the model. Typically obtained through <code>self.grad</code>.</p> <p> TYPE: <code>TensorType</code> </p> <code>v</code> <p>An array ([DxP] or even one-dimensional [D]) which multiplies the matrix. \\(D\\) is the number of directions.</p> <p> TYPE: <code>TensorType</code> </p> <code>progress</code> <p>If <code>True</code>, progress is displayed.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>backprop_on</code> <p>Tensor used in the second backpropagation. The first one is along \\(x\\) and \\(y\\) as defined via <code>grad_xy</code>.</p> <p> TYPE: <code>TensorType</code> </p> RETURNS DESCRIPTION <code>TensorType</code> <p>A matrix representing the implicit matrix-vector product of the model along the given directions. Output shape is [DxM], where \\(M\\) is the number of elements of <code>backprop_on</code>.</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef mvp(\ngrad_xy: TensorType,\nv: TensorType,\nbackprop_on: TensorType,\n*,\nprogress: bool = False,\n) -&gt; TensorType:\nr\"\"\"\n    Calculates the second order derivative of the model along directions \\(v\\).\n    The second order derivative can be selected through the `backprop_on` argument.\n    Args:\n        grad_xy: An array [P] holding the gradients of the model parameters with respect to input \\(x\\) and\n            labels \\(y\\). \\(P\\) is the number of parameters of the model. Typically obtained through `self.grad`.\n        v: An array ([DxP] or even one-dimensional [D]) which multiplies the matrix.\n            \\(D\\) is the number of directions.\n        progress: If `True`, progress is displayed.\n        backprop_on: Tensor used in the second backpropagation. The first one is along \\(x\\) and \\(y\\)\n            as defined via `grad_xy`.\n    Returns:\n        A matrix representing the implicit matrix-vector product of the model along the given directions.\n            Output shape is [DxM], where \\(M\\) is the number of elements of `backprop_on`.\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities","title":"<code>TensorUtilities</code>","text":"<p>             Bases: <code>Generic[TensorType, ModelType]</code>, <code>ABC</code></p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.__init_subclass__","title":"<code>__init_subclass__(**kwargs)</code>","text":"<p>Automatically registers non-abstract subclasses in the registry.</p> <p>This method checks if <code>twice_differentiable_type</code> is defined in the subclass and if it is of the correct type. If either attribute is missing or incorrect, a <code>TypeError</code> is raised.</p> PARAMETER  DESCRIPTION <code>kwargs</code> <p>Additional keyword arguments.</p> <p> DEFAULT: <code>{}</code> </p> RAISES DESCRIPTION <code>TypeError</code> <p>If the subclass does not define <code>twice_differentiable_type</code>, or if it is not of the correct type.</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>def __init_subclass__(cls, **kwargs):\n\"\"\"\n    Automatically registers non-abstract subclasses in the registry.\n    This method checks if `twice_differentiable_type` is defined in the subclass and if it is of the correct type.\n    If either attribute is missing or incorrect, a `TypeError` is raised.\n    Args:\n        kwargs: Additional keyword arguments.\n    Raises:\n        TypeError: If the subclass does not define `twice_differentiable_type`, or if it is not of the correct type.\n    \"\"\"\nif not hasattr(cls, \"twice_differentiable_type\") or not isinstance(\ncls.twice_differentiable_type, type\n):\nraise TypeError(\nf\"'twice_differentiable_type' must be a Type[TwiceDifferentiable]\"\n)\ncls.registry[cls.twice_differentiable_type] = cls\nsuper().__init_subclass__(**kwargs)\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.einsum","title":"<code>einsum(equation, *operands)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Sums the product of the elements of the input <code>operands</code> along dimensions specified using a notation based on the Einstein summation convention.</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef einsum(equation, *operands) -&gt; TensorType:\n\"\"\"Sums the product of the elements of the input `operands` along dimensions specified using a notation\n    based on the Einstein summation convention.\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.cat","title":"<code>cat(a, **kwargs)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Concatenates a sequence of tensors into a single torch tensor</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef cat(a: Sequence[TensorType], **kwargs) -&gt; TensorType:\n\"\"\"Concatenates a sequence of tensors into a single torch tensor\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.stack","title":"<code>stack(a, **kwargs)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Stacks a sequence of tensors into a single torch tensor</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef stack(a: Sequence[TensorType], **kwargs) -&gt; TensorType:\n\"\"\"Stacks a sequence of tensors into a single torch tensor\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.unsqueeze","title":"<code>unsqueeze(x, dim)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Add a singleton dimension at a specified position in a tensor</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef unsqueeze(x: TensorType, dim: int) -&gt; TensorType:\n\"\"\"Add a singleton dimension at a specified position in a tensor\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.get_element","title":"<code>get_element(x, idx)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Get the tensor element x[i] from the first non-singular dimension</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef get_element(x: TensorType, idx: int) -&gt; TensorType:\n\"\"\"Get the tensor element x[i] from the first non-singular dimension\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.slice","title":"<code>slice(x, start, stop, axis=0)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Slice a tensor in the provided axis</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef slice(x: TensorType, start: int, stop: int, axis: int = 0) -&gt; TensorType:\n\"\"\"Slice a tensor in the provided axis\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.shape","title":"<code>shape(x)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Slice a tensor in the provided axis</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef shape(x: TensorType) -&gt; Tuple[int, ...]:\n\"\"\"Slice a tensor in the provided axis\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.reshape","title":"<code>reshape(x, shape)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Reshape a tensor to the provided shape</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef reshape(x: TensorType, shape: Tuple[int, ...]) -&gt; TensorType:\n\"\"\"Reshape a tensor to the provided shape\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.cat_gen","title":"<code>cat_gen(a, resulting_shape, model)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Concatenate tensors from a generator. Resulting tensor is of shape resulting_shape and compatible to model</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef cat_gen(\na: Generator[TensorType, None, None],\nresulting_shape: Tuple[int, ...],\nmodel: ModelType,\n) -&gt; TensorType:\n\"\"\"Concatenate tensors from a generator. Resulting tensor is of shape resulting_shape\n    and compatible to model\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.from_twice_differentiable","title":"<code>from_twice_differentiable(twice_diff)</code>  <code>classmethod</code>","text":"<p>Factory method to create an instance of a subclass TensorUtilities from an instance of a subclass of TwiceDifferentiable.</p> PARAMETER  DESCRIPTION <code>twice_diff</code> <p>An instance of a subclass of TwiceDifferentiable for which a corresponding TensorUtilities object is required.</p> <p> TYPE: <code>TwiceDifferentiable</code> </p> RETURNS DESCRIPTION <code>Type[TensorUtilities]</code> <p>An subclass of TensorUtilities registered to the provided subclass instance of TwiceDifferentiable object.</p> RAISES DESCRIPTION <code>KeyError</code> <p>If there's no registered TensorUtilities for the provided TwiceDifferentiable type.</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@classmethod\ndef from_twice_differentiable(\ncls,\ntwice_diff: TwiceDifferentiable,\n) -&gt; Type[\"TensorUtilities\"]:\n\"\"\"\n    Factory method to create an instance of a subclass\n    [TensorUtilities][pydvl.influence.twice_differentiable.TensorUtilities] from an instance of a subclass of\n    [TwiceDifferentiable][pydvl.influence.twice_differentiable.TwiceDifferentiable].\n    Args:\n        twice_diff: An instance of a subclass of\n            [TwiceDifferentiable][pydvl.influence.twice_differentiable.TwiceDifferentiable]\n            for which a corresponding [TensorUtilities][pydvl.influence.twice_differentiable.TensorUtilities]\n            object is required.\n    Returns:\n        An subclass of [TensorUtilities][pydvl.influence.twice_differentiable.TensorUtilities]\n            registered to the provided subclass instance of\n            [TwiceDifferentiable][pydvl.influence.twice_differentiable.TwiceDifferentiable] object.\n    Raises:\n        KeyError: If there's no registered [TensorUtilities][pydvl.influence.twice_differentiable.TensorUtilities]\n            for the provided [TwiceDifferentiable][pydvl.influence.twice_differentiable.TwiceDifferentiable] type.\n    \"\"\"\ntu = cls.registry.get(type(twice_diff), None)\nif tu is None:\nraise KeyError(\nf\"No registered TensorUtilities for the type {type(twice_diff).__name__}\"\n)\nreturn tu\n</code></pre>"},{"location":"api/pydvl/influence/torch/","title":"Torch","text":""},{"location":"api/pydvl/influence/torch/functional/","title":"Functional","text":""},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.hvp","title":"<code>hvp(func, params, vec, reverse_only=True)</code>","text":"<p>Computes the Hessian-vector product (HVP) for a given function at given parameters, i.e.</p> \\[\\nabla_{\\theta} \\nabla_{\\theta} f (\\theta)\\cdot v\\] <p>This function can operate in two modes, either reverse-mode autodiff only or both forward- and reverse-mode autodiff.</p> PARAMETER  DESCRIPTION <code>func</code> <p>The scalar-valued function for which the HVP is computed.</p> <p> TYPE: <code>Callable[[TorchTensorContainerType], Tensor]</code> </p> <code>params</code> <p>The parameters at which the HVP is computed.</p> <p> TYPE: <code>TorchTensorContainerType</code> </p> <code>vec</code> <p>The vector with which the Hessian is multiplied.</p> <p> TYPE: <code>TorchTensorContainerType</code> </p> <code>reverse_only</code> <p>Whether to use only reverse-mode autodiff (True, default) or both forward- and reverse-mode autodiff (False).</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>TorchTensorContainerType</code> <p>The HVP of the function at the given parameters with the given vector.</p> <p>Example: <pre><code>&gt;&gt;&gt; def f(z): return torch.sum(z**2)\n&gt;&gt;&gt; u = torch.ones(10, requires_grad=True)\n&gt;&gt;&gt; v = torch.ones(10)\n&gt;&gt;&gt; hvp_vec = hvp(f, u, v)\n&gt;&gt;&gt; assert torch.allclose(hvp_vec, torch.full((10, ), 2.0))\n</code></pre></p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def hvp(\nfunc: Callable[[TorchTensorContainerType], torch.Tensor],\nparams: TorchTensorContainerType,\nvec: TorchTensorContainerType,\nreverse_only: bool = True,\n) -&gt; TorchTensorContainerType:\nr\"\"\"\n    Computes the Hessian-vector product (HVP) for a given function at given parameters, i.e.\n    \\[\\nabla_{\\theta} \\nabla_{\\theta} f (\\theta)\\cdot v\\]\n    This function can operate in two modes, either reverse-mode autodiff only or both\n    forward- and reverse-mode autodiff.\n    Args:\n        func: The scalar-valued function for which the HVP is computed.\n        params: The parameters at which the HVP is computed.\n        vec: The vector with which the Hessian is multiplied.\n        reverse_only: Whether to use only reverse-mode autodiff\n            (True, default) or both forward- and reverse-mode autodiff (False).\n    Returns:\n       The HVP of the function at the given parameters with the given vector.\n    Example:\n    ```python\n    &gt;&gt;&gt; def f(z): return torch.sum(z**2)\n    &gt;&gt;&gt; u = torch.ones(10, requires_grad=True)\n    &gt;&gt;&gt; v = torch.ones(10)\n    &gt;&gt;&gt; hvp_vec = hvp(f, u, v)\n    &gt;&gt;&gt; assert torch.allclose(hvp_vec, torch.full((10, ), 2.0))\n    ```\n    \"\"\"\noutput: TorchTensorContainerType\nif reverse_only:\n_, vjp_fn = vjp(grad(func), params)\noutput = vjp_fn(vec)[0]\nelse:\noutput = jvp(grad(func), (params,), (vec,))[1]\nreturn output\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.batch_hvp_gen","title":"<code>batch_hvp_gen(model, loss, data_loader, reverse_only=True)</code>","text":"<p>Generates a sequence of batch Hessian-vector product (HVP) computations for the provided model, loss function, and data loader. If \\(f_i\\) is the model's loss on the \\(i\\)-th batch and \\(\\theta\\) the model parameters, this is the sequence of the callable matrix vector products for the matrices</p> \\[\\nabla_{\\theta}\\nabla_{\\theta}f_i(\\theta), \\quad i=1,\\dots, \\text{num_batches} \\] <p>i.e. iterating over the data_loader, yielding partial function calls for calculating HVPs.</p> PARAMETER  DESCRIPTION <code>model</code> <p>The PyTorch model for which the HVP is calculated.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>The loss function used to calculate the gradient and HVP.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>data_loader</code> <p>PyTorch DataLoader object containing the dataset for which the HVP is calculated.</p> <p> TYPE: <code>DataLoader</code> </p> <code>reverse_only</code> <p>Whether to use only reverse-mode autodiff (True, default) or both forward- and reverse-mode autodiff (False).</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> YIELDS DESCRIPTION <code>Callable[[Tensor], Tensor]</code> <p>Partial functions <code>H_{batch}(vec)=hvp(model, loss, inputs, targets, vec)</code> that when called, will compute the Hessian-vector product H(vec) for the given model and loss in a batch-wise manner, where (inputs, targets) coming from one batch.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def batch_hvp_gen(\nmodel: torch.nn.Module,\nloss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\ndata_loader: DataLoader,\nreverse_only: bool = True,\n) -&gt; Generator[Callable[[torch.Tensor], torch.Tensor], None, None]:\nr\"\"\"\n    Generates a sequence of batch Hessian-vector product (HVP) computations for the provided model, loss function,\n    and data loader. If \\(f_i\\) is the model's loss on the \\(i\\)-th batch and \\(\\theta\\) the model parameters,\n    this is the sequence of the callable matrix vector products for the matrices\n    \\[\\nabla_{\\theta}\\nabla_{\\theta}f_i(\\theta), \\quad i=1,\\dots, \\text{num_batches} \\]\n    i.e. iterating over the data_loader, yielding partial function calls for calculating HVPs.\n    Args:\n        model: The PyTorch model for which the HVP is calculated.\n        loss: The loss function used to calculate the gradient and HVP.\n        data_loader: PyTorch DataLoader object containing the dataset for which the HVP is calculated.\n        reverse_only: Whether to use only reverse-mode autodiff\n            (True, default) or both forward- and reverse-mode autodiff (False).\n    Yields:\n        Partial functions `H_{batch}(vec)=hvp(model, loss, inputs, targets, vec)` that when called,\n            will compute the Hessian-vector product H(vec) for the given model and loss in a batch-wise manner, where\n            (inputs, targets) coming from one batch.\n    \"\"\"\nfor inputs, targets in iter(data_loader):\nbatch_loss = batch_loss_function(model, loss, inputs, targets)\nmodel_params = dict(model.named_parameters())\ndef batch_hvp(vec: torch.Tensor):\nreturn flatten_tensors_to_vector(\nhvp(\nbatch_loss,\nmodel_params,\nalign_structure(model_params, vec),\nreverse_only=reverse_only,\n).values()\n)\nyield batch_hvp\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.empirical_loss_function","title":"<code>empirical_loss_function(model, loss, data_loader)</code>","text":"<p>Creates a function to compute the empirical loss of a given model on a given dataset. If we denote the model parameters with \\( \\theta \\), the resulting function approximates:</p> \\[f(\\theta) = \\frac{1}{N}\\sum_{i=1}^N \\operatorname{loss}(y_i, \\operatorname{model}(\\theta, x_i))\\] <p>Args: - model: The model for which the loss should be computed. - loss: The loss function to be used. - data_loader: The data loader for iterating over the dataset.</p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor]], Tensor]</code> <p>A function that computes the empirical loss of the model on the dataset for given model parameters.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def empirical_loss_function(\nmodel: torch.nn.Module,\nloss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\ndata_loader: DataLoader,\n) -&gt; Callable[[Dict[str, torch.Tensor]], torch.Tensor]:\nr\"\"\"\n    Creates a function to compute the empirical loss of a given model on a given dataset.\n    If we denote the model parameters with \\( \\theta \\), the resulting function approximates:\n    \\[f(\\theta) = \\frac{1}{N}\\sum_{i=1}^N \\operatorname{loss}(y_i, \\operatorname{model}(\\theta, x_i))\\]\n    Args:\n    - model: The model for which the loss should be computed.\n    - loss: The loss function to be used.\n    - data_loader: The data loader for iterating over the dataset.\n    Returns:\n        A function that computes the empirical loss of the model on the dataset for given model parameters.\n    \"\"\"\ndef empirical_loss(params: Dict[str, torch.Tensor]):\ntotal_loss = to_model_device(torch.zeros((), requires_grad=True), model)\ntotal_samples = to_model_device(torch.zeros(()), model)\nfor x, y in iter(data_loader):\noutput = functional_call(\nmodel, params, (to_model_device(x, model),), strict=True\n)\nloss_value = loss(output, to_model_device(y, model))\ntotal_loss = total_loss + loss_value * x.size(0)\ntotal_samples += x.size(0)\nreturn total_loss / total_samples\nreturn empirical_loss\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.batch_loss_function","title":"<code>batch_loss_function(model, loss, x, y)</code>","text":"<p>Creates a function to compute the loss of a given model on a given batch of data, i.e. for the \\(i\\)-th batch \\(B_i\\)</p> \\[\\frac{1}{|B_i|}\\sum_{x,y \\in B_i} \\operatorname{loss}(y, \\operatorname{model}(\\theta, x))\\] PARAMETER  DESCRIPTION <code>model</code> <p>The model for which the loss should be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>The loss function to be used.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>x</code> <p>The input data for the batch.</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>The true labels for the batch.</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor]], Tensor]</code> <p>A function that computes the loss of the model on the batch for given model parameters.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def batch_loss_function(\nmodel: torch.nn.Module,\nloss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\nx: torch.Tensor,\ny: torch.Tensor,\n) -&gt; Callable[[Dict[str, torch.Tensor]], torch.Tensor]:\nr\"\"\"\n    Creates a function to compute the loss of a given model on a given batch of data, i.e. for the $i$-th batch $B_i$\n    \\[\\frac{1}{|B_i|}\\sum_{x,y \\in B_i} \\operatorname{loss}(y, \\operatorname{model}(\\theta, x))\\]\n    Args:\n        model: The model for which the loss should be computed.\n        loss: The loss function to be used.\n        x: The input data for the batch.\n        y: The true labels for the batch.\n    Returns:\n        A function that computes the loss of the model on the batch for given model parameters.\n    \"\"\"\ndef batch_loss(params: Dict[str, torch.Tensor]):\noutputs = functional_call(\nmodel, params, (to_model_device(x, model),), strict=True\n)\nreturn loss(outputs, y)\nreturn batch_loss\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.get_hvp_function","title":"<code>get_hvp_function(model, loss, data_loader, use_hessian_avg=True, reverse_only=True, track_gradients=False)</code>","text":"<p>Returns a function that calculates the approximate Hessian-vector product for a given vector. If you want to compute the exact hessian, i.e., pulling all data into memory and compute a full gradient computation, use the function <code>hvp</code>.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch module representing the model whose loss function's Hessian is to be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that takes the model's output and target as input and returns the scalar loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>data_loader</code> <p>A DataLoader instance that provides batches of data for calculating the Hessian-vector product. Each batch from the DataLoader is assumed to return a tuple where the first element is the model's input and the second element is the target output.</p> <p> TYPE: <code>DataLoader</code> </p> <code>use_hessian_avg</code> <p>If True, the returned function uses batch-wise Hessian computation via batch_loss_function and averages the results. If False, the function uses backpropagation on the full empirical_loss_function, which is more accurate than averaging the batch hessians, but probably has a way higher memory usage.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>reverse_only</code> <p>Whether to use only reverse-mode autodiff (True, default) or both forward- and reverse-mode autodiff (False).</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>track_gradients</code> <p>Whether to track gradients for the resulting tensor of the hessian vector products are (False, default).</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Callable[[Tensor], Tensor]</code> <p>A function that takes a single argument, a vector, and returns the product of the Hessian of the <code>loss</code> function with respect to the <code>model</code>'s parameters and the input vector.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def get_hvp_function(\nmodel: torch.nn.Module,\nloss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\ndata_loader: DataLoader,\nuse_hessian_avg: bool = True,\nreverse_only: bool = True,\ntrack_gradients: bool = False,\n) -&gt; Callable[[torch.Tensor], torch.Tensor]:\n\"\"\"\n    Returns a function that calculates the approximate Hessian-vector product for a given vector. If you want to\n    compute the exact hessian, i.e., pulling all data into memory and compute a full gradient computation, use\n    the function `hvp`.\n    Args:\n        model: A PyTorch module representing the model whose loss function's Hessian is to be computed.\n        loss: A callable that takes the model's output and target as input and returns the scalar loss.\n        data_loader: A DataLoader instance that provides batches of data for calculating the Hessian-vector product.\n            Each batch from the DataLoader is assumed to return a tuple where the first element\n            is the model's input and the second element is the target output.\n        use_hessian_avg: If True, the returned function uses batch-wise Hessian computation via\n            [batch_loss_function][pydvl.influence.torch.functional.batch_loss_function] and averages the results.\n            If False, the function uses backpropagation on the full\n            [empirical_loss_function][pydvl.influence.torch.functional.empirical_loss_function],\n            which is more accurate than averaging the batch hessians, but probably has a way higher memory usage.\n        reverse_only: Whether to use only reverse-mode autodiff (True, default) or\n            both forward- and reverse-mode autodiff (False).\n        track_gradients: Whether to track gradients for the resulting tensor of the hessian vector\n            products are (False, default).\n    Returns:\n        A function that takes a single argument, a vector, and returns the product of the Hessian of the `loss`\n            function with respect to the `model`'s parameters and the input vector.\n    \"\"\"\nparams = {\nk: p if track_gradients else p.detach() for k, p in model.named_parameters()\n}\ndef hvp_function(vec: torch.Tensor) -&gt; torch.Tensor:\nv = align_structure(params, vec)\nempirical_loss = empirical_loss_function(model, loss, data_loader)\nreturn flatten_tensors_to_vector(\nhvp(empirical_loss, params, v, reverse_only=reverse_only).values()\n)\ndef avg_hvp_function(vec: torch.Tensor) -&gt; torch.Tensor:\nv = align_structure(params, vec)\nbatch_hessians_vector_products: Iterable[torch.Tensor] = map(\nlambda x: x(v), batch_hvp_gen(model, loss, data_loader, reverse_only)\n)\nnum_batches = len(data_loader)\navg_hessian = to_model_device(torch.zeros_like(vec), model)\nfor batch_hvp in batch_hessians_vector_products:\navg_hessian += batch_hvp\nreturn avg_hessian / float(num_batches)\nreturn avg_hvp_function if use_hessian_avg else hvp_function\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/","title":"Torch differentiable","text":"<p>Contains methods for differentiating  a pyTorch model. Most of the methods focus on ways to calculate matrix vector products. Moreover, it contains several methods to invert the Hessian vector product. These are used to calculate the influence of a training point on the model.</p>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable--references","title":"References","text":"<ol> <li> <p>Koh, P.W., Liang, P., 2017. Understanding Black-box Predictions via Influence Functions. In: Proceedings of the 34th International Conference on Machine Learning, pp. 1885\u20131894. PMLR.\u00a0\u21a9</p> </li> <li> <p>Agarwal, N., Bullins, B., Hazan, E., 2017. Second-Order Stochastic Optimization for Machine Learning in Linear Time. In: Journal of Machine Learning Research, Vol. 18, pp. 1\u201340. JMLR.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTwiceDifferentiable","title":"<code>TorchTwiceDifferentiable(model, loss)</code>","text":"<p>             Bases: <code>TwiceDifferentiable[Tensor]</code></p> <p>Wraps a torch.nn.Module and a loss function and provides methods to compute gradients and second derivative of the loss wrt. the model parameters</p> PARAMETER  DESCRIPTION <code>model</code> <p>A (differentiable) function.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A differentiable scalar loss \\( L(\\hat{y}, y) \\), mapping a prediction and a target to a real value.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>def __init__(\nself,\nmodel: nn.Module,\nloss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n):\nif model.training:\nlogger.warning(\n\"Passed model not in evaluation mode. This can create several issues in influence \"\n\"computation, e.g. due to batch normalization. Please call model.eval() before \"\n\"computing influences.\"\n)\nself.loss = loss\nself.model = model\nfirst_param = next(model.parameters())\nself.device = first_param.device\nself.dtype = first_param.dtype\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTwiceDifferentiable.parameters","title":"<code>parameters: List[torch.Tensor]</code>  <code>property</code>","text":"RETURNS DESCRIPTION <code>List[Tensor]</code> <p>All model parameters that require differentiating.</p>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTwiceDifferentiable.num_params","title":"<code>num_params: int</code>  <code>property</code>","text":"<p>Get the number of parameters of model f.</p> RETURNS DESCRIPTION <code>int</code> <p>Number of parameters.</p> <p> TYPE: <code>int</code> </p>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTwiceDifferentiable.grad","title":"<code>grad(x, y, create_graph=False)</code>","text":"<p>Calculates gradient of model parameters with respect to the model parameters.</p> PARAMETER  DESCRIPTION <code>x</code> <p>A matrix [NxD] representing the features \\( x_i \\).</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>A matrix [NxK] representing the target values \\( y_i \\).</p> <p> TYPE: <code>Tensor</code> </p> <code>create_graph</code> <p>If True, the resulting gradient tensor can be used for further differentiation.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>An array [P] with the gradients of the model.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>def grad(\nself, x: torch.Tensor, y: torch.Tensor, create_graph: bool = False\n) -&gt; torch.Tensor:\nr\"\"\"\n    Calculates gradient of model parameters with respect to the model parameters.\n    Args:\n        x: A matrix [NxD] representing the features \\( x_i \\).\n        y: A matrix [NxK] representing the target values \\( y_i \\).\n        create_graph (bool): If True, the resulting gradient tensor can be used for further differentiation.\n    Returns:\n        An array [P] with the gradients of the model.\n    \"\"\"\nx = x.to(self.device)\ny = y.to(self.device)\nif create_graph and not x.requires_grad:\nx = x.requires_grad_(True)\nloss_value = self.loss(torch.squeeze(self.model(x)), torch.squeeze(y))\ngrad_f = torch.autograd.grad(\nloss_value, self.parameters, create_graph=create_graph\n)\nreturn flatten_tensors_to_vector(grad_f)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTwiceDifferentiable.hessian","title":"<code>hessian(x, y)</code>","text":"<p>Calculates the explicit hessian of model parameters given data \\(x\\) and \\(y\\).</p> PARAMETER  DESCRIPTION <code>x</code> <p>A matrix [NxD] representing the features \\(x_i\\).</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>A matrix [NxK] representing the target values \\(y_i\\).</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A tensor representing the hessian of the loss with respect to the model parameters.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>def hessian(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\nr\"\"\"\n    Calculates the explicit hessian of model parameters given data \\(x\\) and \\(y\\).\n    Args:\n        x: A matrix [NxD] representing the features \\(x_i\\).\n        y: A matrix [NxK] representing the target values \\(y_i\\).\n    Returns:\n        A tensor representing the hessian of the loss with respect to the model parameters.\n    \"\"\"\ndef model_func(param):\noutputs = torch.func.functional_call(\nself.model,\nalign_structure(\n{k: p for k, p in self.model.named_parameters() if p.requires_grad},\nparam,\n),\n(x.to(self.device),),\nstrict=True,\n)\nreturn self.loss(outputs, y.to(self.device))\nparams = flatten_tensors_to_vector(\np.detach() for p in self.model.parameters() if p.requires_grad\n)\nreturn torch.func.hessian(model_func)(params)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTwiceDifferentiable.mvp","title":"<code>mvp(grad_xy, v, backprop_on, *, progress=False)</code>  <code>staticmethod</code>","text":"<p>Calculates the second-order derivative of the model along directions v. This second-order derivative can be selected through the <code>backprop_on</code> argument.</p> PARAMETER  DESCRIPTION <code>grad_xy</code> <p>An array [P] holding the gradients of the model parameters with respect to input \\(x\\) and labels \\(y\\), where P is the number of parameters of the model. It is typically obtained through <code>self.grad</code>.</p> <p> TYPE: <code>Tensor</code> </p> <code>v</code> <p>An array ([DxP] or even one-dimensional [D]) which multiplies the matrix, where D is the number of directions.</p> <p> TYPE: <code>Tensor</code> </p> <code>progress</code> <p>If True, progress will be printed.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>backprop_on</code> <p>Tensor used in the second backpropagation (the first one is defined via grad_xy).</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A matrix representing the implicit matrix-vector product of the model along the given directions. The output shape is [DxM], with M being the number of elements of <code>backprop_on</code>.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@staticmethod\ndef mvp(\ngrad_xy: torch.Tensor,\nv: torch.Tensor,\nbackprop_on: torch.Tensor,\n*,\nprogress: bool = False,\n) -&gt; torch.Tensor:\nr\"\"\"\n    Calculates the second-order derivative of the model along directions v.\n    This second-order derivative can be selected through the `backprop_on` argument.\n    Args:\n        grad_xy: An array [P] holding the gradients of the model parameters with respect to input\n            \\(x\\) and labels \\(y\\), where P is the number of parameters of the model.\n            It is typically obtained through `self.grad`.\n        v: An array ([DxP] or even one-dimensional [D]) which multiplies the matrix,\n            where D is the number of directions.\n        progress: If True, progress will be printed.\n        backprop_on: Tensor used in the second backpropagation\n            (the first one is defined via grad_xy).\n    Returns:\n        A matrix representing the implicit matrix-vector product of the model along the given directions.\n            The output shape is [DxM], with M being the number of elements of `backprop_on`.\n    \"\"\"\ndevice = grad_xy.device\nv = as_tensor(v, warn=False).to(device)\nif v.ndim == 1:\nv = v.unsqueeze(0)\nz = (grad_xy * Variable(v)).sum(dim=1)\nmvp = []\nfor i in maybe_progress(range(len(z)), progress, desc=\"MVP\"):\nmvp.append(\nflatten_tensors_to_vector(\nautograd.grad(z[i], backprop_on, retain_graph=True)\n)\n)\nreturn torch.stack([grad.contiguous().view(-1) for grad in mvp]).detach()\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.LowRankProductRepresentation","title":"<code>LowRankProductRepresentation</code>  <code>dataclass</code>","text":"<p>Representation of a low rank product of the form \\(H = V D V^T\\), where D is a diagonal matrix and V is orthogonal.</p> PARAMETER  DESCRIPTION <code>eigen_vals</code> <p>Diagonal of D.</p> <p> TYPE: <code>Tensor</code> </p> <code>projections</code> <p>The matrix V.</p> <p> TYPE: <code>Tensor</code> </p>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.LowRankProductRepresentation.to","title":"<code>to(device)</code>","text":"<p>Move the representing tensors to a device</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>def to(self, device: torch.device):\n\"\"\"\n    Move the representing tensors to a device\n    \"\"\"\nreturn LowRankProductRepresentation(\nself.eigen_vals.to(device), self.projections.to(device)\n)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTensorUtilities","title":"<code>TorchTensorUtilities</code>","text":"<p>             Bases: <code>TensorUtilities[Tensor, TorchTwiceDifferentiable]</code></p>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTensorUtilities.einsum","title":"<code>einsum(equation, *operands)</code>  <code>staticmethod</code>","text":"<p>Sums the product of the elements of the input :attr:<code>operands</code> along dimensions specified using a notation based on the Einstein summation convention.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@staticmethod\ndef einsum(equation: str, *operands) -&gt; torch.Tensor:\n\"\"\"Sums the product of the elements of the input :attr:`operands` along dimensions specified using a notation\n    based on the Einstein summation convention.\n    \"\"\"\nreturn torch.einsum(equation, *operands)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTensorUtilities.cat","title":"<code>cat(a, **kwargs)</code>  <code>staticmethod</code>","text":"<p>Concatenates a sequence of tensors into a single torch tensor</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@staticmethod\ndef cat(a: Sequence[torch.Tensor], **kwargs) -&gt; torch.Tensor:\n\"\"\"Concatenates a sequence of tensors into a single torch tensor\"\"\"\nreturn torch.cat(a, **kwargs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTensorUtilities.stack","title":"<code>stack(a, **kwargs)</code>  <code>staticmethod</code>","text":"<p>Stacks a sequence of tensors into a single torch tensor</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@staticmethod\ndef stack(a: Sequence[torch.Tensor], **kwargs) -&gt; torch.Tensor:\n\"\"\"Stacks a sequence of tensors into a single torch tensor\"\"\"\nreturn torch.stack(a, **kwargs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTensorUtilities.unsqueeze","title":"<code>unsqueeze(x, dim)</code>  <code>staticmethod</code>","text":"<p>Add a singleton dimension at a specified position in a tensor.</p> PARAMETER  DESCRIPTION <code>x</code> <p>A PyTorch tensor.</p> <p> TYPE: <code>Tensor</code> </p> <code>dim</code> <p>The position at which to add the singleton dimension. Zero-based indexing.</p> <p> TYPE: <code>int</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A new tensor with an additional singleton dimension.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@staticmethod\ndef unsqueeze(x: torch.Tensor, dim: int) -&gt; torch.Tensor:\n\"\"\"\n    Add a singleton dimension at a specified position in a tensor.\n    Args:\n        x: A PyTorch tensor.\n        dim: The position at which to add the singleton dimension. Zero-based indexing.\n    Returns:\n        A new tensor with an additional singleton dimension.\n    \"\"\"\nreturn x.unsqueeze(dim)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.lanzcos_low_rank_hessian_approx","title":"<code>lanzcos_low_rank_hessian_approx(hessian_vp, matrix_shape, hessian_perturbation=0.0, rank_estimate=10, krylov_dimension=None, tol=1e-06, max_iter=None, device=None, eigen_computation_on_gpu=False, torch_dtype=None)</code>","text":"<p>Calculates a low-rank approximation of the Hessian matrix of a scalar-valued function using the implicitly restarted Lanczos algorithm, i.e.:</p> \\[ H_{\\text{approx}} = V D V^T\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) <code>rank_estimate</code> eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors.</p> PARAMETER  DESCRIPTION <code>hessian_vp</code> <p>A function that takes a vector and returns the product of the Hessian of the loss function.</p> <p> TYPE: <code>Callable[[Tensor], Tensor]</code> </p> <code>matrix_shape</code> <p>The shape of the matrix, represented by the hessian vector product.</p> <p> TYPE: <code>Tuple[int, int]</code> </p> <code>hessian_perturbation</code> <p>Regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>rank_estimate</code> <p>The number of eigenvalues and corresponding eigenvectors to compute. Represents the desired rank of the Hessian approximation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>krylov_dimension</code> <p>The number of Krylov vectors to use for the Lanczos method. If not provided, it defaults to \\( \\min(\\text{model.num_parameters}, \\max(2 \\times \\text{rank_estimate} + 1, 20)) \\).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>tol</code> <p>The stopping criteria for the Lanczos algorithm, which stops when the difference in the approximated eigenvalue is less than <code>tol</code>. Defaults to 1e-6.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-06</code> </p> <code>max_iter</code> <p>The maximum number of iterations for the Lanczos method. If not provided, it defaults to \\( 10 \\cdot \\text{model.num_parameters}\\).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>device</code> <p>The device to use for executing the hessian vector product.</p> <p> TYPE: <code>Optional[device]</code> DEFAULT: <code>None</code> </p> <code>eigen_computation_on_gpu</code> <p>If True, tries to execute the eigen pair approximation on the provided device via cupy implementation. Ensure that either your model is small enough, or you use a small rank_estimate to fit your device's memory. If False, the eigen pair approximation is executed on the CPU with scipy's wrapper to ARPACK.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>torch_dtype</code> <p>If not provided, the current torch default dtype is used for conversion to torch.</p> <p> TYPE: <code>dtype</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>LowRankProductRepresentation</code> <p>A LowRankProductRepresentation instance that contains the top (up until rank_estimate) eigenvalues and corresponding eigenvectors of the Hessian.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>def lanzcos_low_rank_hessian_approx(\nhessian_vp: Callable[[torch.Tensor], torch.Tensor],\nmatrix_shape: Tuple[int, int],\nhessian_perturbation: float = 0.0,\nrank_estimate: int = 10,\nkrylov_dimension: Optional[int] = None,\ntol: float = 1e-6,\nmax_iter: Optional[int] = None,\ndevice: Optional[torch.device] = None,\neigen_computation_on_gpu: bool = False,\ntorch_dtype: torch.dtype = None,\n) -&gt; LowRankProductRepresentation:\nr\"\"\"\n    Calculates a low-rank approximation of the Hessian matrix of a scalar-valued\n    function using the implicitly restarted Lanczos algorithm, i.e.:\n    \\[ H_{\\text{approx}} = V D V^T\\]\n    where \\(D\\) is a diagonal matrix with the top (in absolute value) `rank_estimate` eigenvalues of the Hessian\n    and \\(V\\) contains the corresponding eigenvectors.\n    Args:\n        hessian_vp: A function that takes a vector and returns the product of\n            the Hessian of the loss function.\n        matrix_shape: The shape of the matrix, represented by the hessian vector\n            product.\n        hessian_perturbation: Regularization parameter added to the\n            Hessian-vector product for numerical stability.\n        rank_estimate: The number of eigenvalues and corresponding eigenvectors\n            to compute. Represents the desired rank of the Hessian approximation.\n        krylov_dimension: The number of Krylov vectors to use for the Lanczos\n            method. If not provided, it defaults to\n            \\( \\min(\\text{model.num_parameters}, \\max(2 \\times \\text{rank_estimate} + 1, 20)) \\).\n        tol: The stopping criteria for the Lanczos algorithm, which stops when\n            the difference in the approximated eigenvalue is less than `tol`.\n            Defaults to 1e-6.\n        max_iter: The maximum number of iterations for the Lanczos method. If\n            not provided, it defaults to \\( 10 \\cdot \\text{model.num_parameters}\\).\n        device: The device to use for executing the hessian vector product.\n        eigen_computation_on_gpu: If True, tries to execute the eigen pair\n            approximation on the provided device via [cupy](https://cupy.dev/)\n            implementation. Ensure that either your model is small enough, or you\n            use a small rank_estimate to fit your device's memory. If False, the\n            eigen pair approximation is executed on the CPU with scipy's wrapper to\n            ARPACK.\n        torch_dtype: If not provided, the current torch default dtype is used for\n            conversion to torch.\n    Returns:\n        A [LowRankProductRepresentation][pydvl.influence.torch.torch_differentiable.LowRankProductRepresentation]\n            instance that contains the top (up until rank_estimate) eigenvalues\n            and corresponding eigenvectors of the Hessian.\n    \"\"\"\ntorch_dtype = torch.get_default_dtype() if torch_dtype is None else torch_dtype\nif eigen_computation_on_gpu:\ntry:\nimport cupy as cp\nfrom cupyx.scipy.sparse.linalg import LinearOperator, eigsh\nfrom torch.utils.dlpack import from_dlpack, to_dlpack\nexcept ImportError as e:\nraise ImportError(\nf\"Try to install missing dependencies or set eigen_computation_on_gpu to False: {e}\"\n)\nif device is None:\nraise ValueError(\n\"Without setting an explicit device, cupy is not supported\"\n)\ndef to_torch_conversion_function(x):\nreturn from_dlpack(x.toDlpack()).to(torch_dtype)\ndef mv(x):\nx = to_torch_conversion_function(x)\ny = hessian_vp(x) + hessian_perturbation * x\nreturn cp.from_dlpack(to_dlpack(y))\nelse:\nfrom scipy.sparse.linalg import LinearOperator, eigsh\ndef mv(x):\nx_torch = torch.as_tensor(x, device=device, dtype=torch_dtype)\ny: NDArray = (\n(hessian_vp(x_torch) + hessian_perturbation * x_torch)\n.detach()\n.cpu()\n.numpy()\n)\nreturn y\nto_torch_conversion_function = partial(torch.as_tensor, dtype=torch_dtype)\ntry:\neigen_vals, eigen_vecs = eigsh(\nLinearOperator(matrix_shape, matvec=mv),\nk=rank_estimate,\nmaxiter=max_iter,\ntol=tol,\nncv=krylov_dimension,\nreturn_eigenvectors=True,\n)\nexcept ArpackNoConvergence as e:\nlogger.warning(\nf\"ARPACK did not converge for parameters {max_iter=}, {tol=}, {krylov_dimension=}, \"\nf\"{rank_estimate=}. \\n Returning the best approximation found so far. Use those with care or \"\nf\"modify parameters.\\n Original error: {e}\"\n)\neigen_vals, eigen_vecs = e.eigenvalues, e.eigenvectors\neigen_vals = to_torch_conversion_function(eigen_vals)\neigen_vecs = to_torch_conversion_function(eigen_vecs)\nreturn LowRankProductRepresentation(eigen_vals, eigen_vecs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.model_hessian_low_rank","title":"<code>model_hessian_low_rank(model, training_data, hessian_perturbation=0.0, rank_estimate=10, krylov_dimension=None, tol=1e-06, max_iter=None, eigen_computation_on_gpu=False)</code>","text":"<p>Calculates a low-rank approximation of the Hessian matrix of the model's loss function using the implicitly restarted Lanczos algorithm, i.e.</p> \\[ H_{\\text{approx}} = V D V^T\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) <code>rank_estimate</code> eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model instance that is twice differentiable, wrapped into <code>TorchTwiceDifferential</code>. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>TorchTwiceDifferentiable</code> </p> <code>training_data</code> <p>A DataLoader instance that provides the model's training data. Used in calculating the Hessian-vector products.</p> <p> TYPE: <code>DataLoader</code> </p> <code>hessian_perturbation</code> <p>Optional regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>rank_estimate</code> <p>The number of eigenvalues and corresponding eigenvectors to compute. Represents the desired rank of the Hessian approximation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>krylov_dimension</code> <p>The number of Krylov vectors to use for the Lanczos method. If not provided, it defaults to min(model.num_parameters, max(2*rank_estimate + 1, 20)).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>tol</code> <p>The stopping criteria for the Lanczos algorithm, which stops when the difference in the approximated eigenvalue is less than <code>tol</code>. Defaults to 1e-6.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-06</code> </p> <code>max_iter</code> <p>The maximum number of iterations for the Lanczos method. If not provided, it defaults to 10*model.num_parameters.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>eigen_computation_on_gpu</code> <p>If True, tries to execute the eigen pair approximation on the provided device via cupy implementation. Make sure, that either your model is small enough or you use a small rank_estimate to fit your device's memory. If False, the eigen pair approximation is executed on the CPU by scipy wrapper to ARPACK.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>LowRankProductRepresentation</code> <p>A LowRankProductRepresentation instance that contains the top (up until rank_estimate) eigenvalues and corresponding eigenvectors of the Hessian.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>def model_hessian_low_rank(\nmodel: TorchTwiceDifferentiable,\ntraining_data: DataLoader,\nhessian_perturbation: float = 0.0,\nrank_estimate: int = 10,\nkrylov_dimension: Optional[int] = None,\ntol: float = 1e-6,\nmax_iter: Optional[int] = None,\neigen_computation_on_gpu: bool = False,\n) -&gt; LowRankProductRepresentation:\nr\"\"\"\n    Calculates a low-rank approximation of the Hessian matrix of the model's loss function using the implicitly\n    restarted Lanczos algorithm, i.e.\n    \\[ H_{\\text{approx}} = V D V^T\\]\n    where \\(D\\) is a diagonal matrix with the top (in absolute value) `rank_estimate` eigenvalues of the Hessian\n    and \\(V\\) contains the corresponding eigenvectors.\n    Args:\n        model: A PyTorch model instance that is twice differentiable, wrapped into `TorchTwiceDifferential`.\n            The Hessian will be calculated with respect to this model's parameters.\n        training_data: A DataLoader instance that provides the model's training data.\n            Used in calculating the Hessian-vector products.\n        hessian_perturbation: Optional regularization parameter added to the Hessian-vector product\n            for numerical stability.\n        rank_estimate: The number of eigenvalues and corresponding eigenvectors to compute.\n            Represents the desired rank of the Hessian approximation.\n        krylov_dimension: The number of Krylov vectors to use for the Lanczos method.\n            If not provided, it defaults to min(model.num_parameters, max(2*rank_estimate + 1, 20)).\n        tol: The stopping criteria for the Lanczos algorithm, which stops when the difference\n            in the approximated eigenvalue is less than `tol`. Defaults to 1e-6.\n        max_iter: The maximum number of iterations for the Lanczos method. If not provided, it defaults to\n            10*model.num_parameters.\n        eigen_computation_on_gpu: If True, tries to execute the eigen pair approximation on the provided\n            device via cupy implementation.\n            Make sure, that either your model is small enough or you use a\n            small rank_estimate to fit your device's memory.\n            If False, the eigen pair approximation is executed on the CPU by scipy wrapper to\n            ARPACK.\n    Returns:\n        A [LowRankProductRepresentation][pydvl.influence.torch.torch_differentiable.LowRankProductRepresentation]\n            instance that contains the top (up until rank_estimate) eigenvalues\n            and corresponding eigenvectors of the Hessian.\n    \"\"\"\nraw_hvp = get_hvp_function(\nmodel.model, model.loss, training_data, use_hessian_avg=True\n)\nreturn lanzcos_low_rank_hessian_approx(\nhessian_vp=raw_hvp,\nmatrix_shape=(model.num_params, model.num_params),\nhessian_perturbation=hessian_perturbation,\nrank_estimate=rank_estimate,\nkrylov_dimension=krylov_dimension,\ntol=tol,\nmax_iter=max_iter,\ndevice=model.device if hasattr(model, \"device\") else None,\neigen_computation_on_gpu=eigen_computation_on_gpu,\n)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.solve_linear","title":"<code>solve_linear(model, training_data, b, hessian_perturbation=0.0)</code>","text":"<p>Given a model and training data, it finds x such that \\(Hx = b\\), with \\(H\\) being the model hessian.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A model wrapped in the TwiceDifferentiable interface.</p> <p> TYPE: <code>TorchTwiceDifferentiable</code> </p> <code>training_data</code> <p>A DataLoader containing the training data.</p> <p> TYPE: <code>DataLoader</code> </p> <code>b</code> <p>A vector or matrix, the right hand side of the equation \\(Hx = b\\).</p> <p> TYPE: <code>Tensor</code> </p> <code>hessian_perturbation</code> <p>Regularization of the hessian.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> RETURNS DESCRIPTION <code>InverseHvpResult</code> <p>Instance of InverseHvpResult, having an array that solves the inverse problem, i.e. it returns \\(x\\) such that \\(Hx = b\\), and a dictionary containing information about the solution.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@InversionRegistry.register(TorchTwiceDifferentiable, InversionMethod.Direct)\ndef solve_linear(\nmodel: TorchTwiceDifferentiable,\ntraining_data: DataLoader,\nb: torch.Tensor,\nhessian_perturbation: float = 0.0,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Given a model and training data, it finds x such that \\(Hx = b\\), with \\(H\\) being the model hessian.\n    Args:\n        model: A model wrapped in the TwiceDifferentiable interface.\n        training_data: A DataLoader containing the training data.\n        b: A vector or matrix, the right hand side of the equation \\(Hx = b\\).\n        hessian_perturbation: Regularization of the hessian.\n    Returns:\n        Instance of [InverseHvpResult][pydvl.influence.twice_differentiable.InverseHvpResult],\n            having an array that solves the inverse problem, i.e. it returns \\(x\\) such that \\(Hx = b\\),\n            and a dictionary containing information about the solution.\n    \"\"\"\nall_x, all_y = [], []\nfor x, y in training_data:\nall_x.append(x)\nall_y.append(y)\nhessian = model.hessian(torch.cat(all_x), torch.cat(all_y))\nmatrix = hessian + hessian_perturbation * torch.eye(\nmodel.num_params, device=model.device\n)\ninfo = {\"hessian\": hessian}\nreturn InverseHvpResult(x=torch.linalg.solve(matrix, b.T).T, info=info)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.solve_batch_cg","title":"<code>solve_batch_cg(model, training_data, b, hessian_perturbation=0.0, *, x0=None, rtol=1e-07, atol=1e-07, maxiter=None, progress=False)</code>","text":"<p>Given a model and training data, it uses conjugate gradient to calculate the inverse of the Hessian Vector Product. More precisely, it finds x such that \\(Hx = b\\), with \\(H\\) being the model hessian. For more info, see Wikipedia.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A model wrapped in the TwiceDifferentiable interface.</p> <p> TYPE: <code>TorchTwiceDifferentiable</code> </p> <code>training_data</code> <p>A DataLoader containing the training data.</p> <p> TYPE: <code>DataLoader</code> </p> <code>b</code> <p>A vector or matrix, the right hand side of the equation \\(Hx = b\\).</p> <p> TYPE: <code>Tensor</code> </p> <code>hessian_perturbation</code> <p>Regularization of the hessian.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>x0</code> <p>Initial guess for hvp. If None, defaults to b.</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>rtol</code> <p>Maximum relative tolerance of result.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-07</code> </p> <code>atol</code> <p>Absolute tolerance of result.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-07</code> </p> <code>maxiter</code> <p>Maximum number of iterations. If None, defaults to 10*len(b).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, display progress bars.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>InverseHvpResult</code> <p>Instance of InverseHvpResult, having a matrix of shape [NxP] with each line being a solution of \\(Ax=b\\), and a dictionary containing information about the convergence of CG, one entry for each line of the matrix.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@InversionRegistry.register(TorchTwiceDifferentiable, InversionMethod.Cg)\ndef solve_batch_cg(\nmodel: TorchTwiceDifferentiable,\ntraining_data: DataLoader,\nb: torch.Tensor,\nhessian_perturbation: float = 0.0,\n*,\nx0: Optional[torch.Tensor] = None,\nrtol: float = 1e-7,\natol: float = 1e-7,\nmaxiter: Optional[int] = None,\nprogress: bool = False,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Given a model and training data, it uses conjugate gradient to calculate the\n    inverse of the Hessian Vector Product. More precisely, it finds x such that \\(Hx =\n    b\\), with \\(H\\) being the model hessian. For more info, see\n    [Wikipedia](https://en.wikipedia.org/wiki/Conjugate_gradient_method).\n    Args:\n        model: A model wrapped in the TwiceDifferentiable interface.\n        training_data: A DataLoader containing the training data.\n        b: A vector or matrix, the right hand side of the equation \\(Hx = b\\).\n        hessian_perturbation: Regularization of the hessian.\n        x0: Initial guess for hvp. If None, defaults to b.\n        rtol: Maximum relative tolerance of result.\n        atol: Absolute tolerance of result.\n        maxiter: Maximum number of iterations. If None, defaults to 10*len(b).\n        progress: If True, display progress bars.\n    Returns:\n        Instance of [InverseHvpResult][pydvl.influence.twice_differentiable.InverseHvpResult],\n            having a matrix of shape [NxP] with each line being a solution of \\(Ax=b\\),\n            and a dictionary containing information about the convergence of CG,\n            one entry for each line of the matrix.\n    \"\"\"\ntotal_grad_xy = 0\ntotal_points = 0\nfor x, y in maybe_progress(training_data, progress, desc=\"Batch Train Gradients\"):\ngrad_xy = model.grad(x, y, create_graph=True)\ntotal_grad_xy += grad_xy * len(x)\ntotal_points += len(x)\nbackprop_on = model.parameters\nreg_hvp = lambda v: model.mvp(\ntotal_grad_xy / total_points, v, backprop_on\n) + hessian_perturbation * v.type(torch.float64)\nbatch_cg = torch.zeros_like(b)\ninfo = {}\nfor idx, bi in enumerate(maybe_progress(b, progress, desc=\"Conjugate gradient\")):\nbatch_result, batch_info = solve_cg(\nreg_hvp, bi, x0=x0, rtol=rtol, atol=atol, maxiter=maxiter\n)\nbatch_cg[idx] = batch_result\ninfo[f\"batch_{idx}\"] = batch_info\nreturn InverseHvpResult(x=batch_cg, info=info)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.solve_cg","title":"<code>solve_cg(hvp, b, *, x0=None, rtol=1e-07, atol=1e-07, maxiter=None)</code>","text":"<p>Conjugate gradient solver for the Hessian vector product.</p> PARAMETER  DESCRIPTION <code>hvp</code> <p>A callable Hvp, operating with tensors of size N.</p> <p> TYPE: <code>Callable[[Tensor], Tensor]</code> </p> <code>b</code> <p>A vector or matrix, the right hand side of the equation \\(Hx = b\\).</p> <p> TYPE: <code>Tensor</code> </p> <code>x0</code> <p>Initial guess for hvp.</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>rtol</code> <p>Maximum relative tolerance of result.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-07</code> </p> <code>atol</code> <p>Absolute tolerance of result.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-07</code> </p> <code>maxiter</code> <p>Maximum number of iterations. If None, defaults to 10*len(b).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>InverseHvpResult</code> <p>Instance of InverseHvpResult, with a vector x, solution of \\(Ax=b\\), and a dictionary containing information about the convergence of CG.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>def solve_cg(\nhvp: Callable[[torch.Tensor], torch.Tensor],\nb: torch.Tensor,\n*,\nx0: Optional[torch.Tensor] = None,\nrtol: float = 1e-7,\natol: float = 1e-7,\nmaxiter: Optional[int] = None,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Conjugate gradient solver for the Hessian vector product.\n    Args:\n        hvp: A callable Hvp, operating with tensors of size N.\n        b: A vector or matrix, the right hand side of the equation \\(Hx = b\\).\n        x0: Initial guess for hvp.\n        rtol: Maximum relative tolerance of result.\n        atol: Absolute tolerance of result.\n        maxiter: Maximum number of iterations. If None, defaults to 10*len(b).\n    Returns:\n        Instance of [InverseHvpResult][pydvl.influence.twice_differentiable.InverseHvpResult],\n            with a vector x, solution of \\(Ax=b\\), and a dictionary containing\n            information about the convergence of CG.\n    \"\"\"\nif x0 is None:\nx0 = torch.clone(b)\nif maxiter is None:\nmaxiter = len(b) * 10\ny_norm = torch.sum(torch.matmul(b, b)).item()\nstopping_val = max([rtol**2 * y_norm, atol**2])\nx = x0\np = r = (b - hvp(x)).squeeze().type(torch.float64)\ngamma = torch.sum(torch.matmul(r, r)).item()\noptimal = False\nfor k in range(maxiter):\nif gamma &lt; stopping_val:\noptimal = True\nbreak\nAp = hvp(p).squeeze()\nalpha = gamma / torch.sum(torch.matmul(p, Ap)).item()\nx += alpha * p\nr -= alpha * Ap\ngamma_ = torch.sum(torch.matmul(r, r)).item()\nbeta = gamma_ / gamma\ngamma = gamma_\np = r + beta * p\ninfo = {\"niter\": k, \"optimal\": optimal, \"gamma\": gamma}\nreturn InverseHvpResult(x=x, info=info)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.solve_lissa","title":"<code>solve_lissa(model, training_data, b, hessian_perturbation=0.0, *, maxiter=1000, dampen=0.0, scale=10.0, h0=None, rtol=0.0001, progress=False)</code>","text":"<p>Uses LISSA, Linear time Stochastic Second-Order Algorithm, to iteratively approximate the inverse Hessian. More precisely, it finds x s.t. \\(Hx = b\\), with \\(H\\) being the model's second derivative wrt. the parameters. This is done with the update</p> \\[H^{-1}_{j+1} b = b + (I - d) \\ H - \\frac{H^{-1}_j b}{s},\\] <p>where \\(I\\) is the identity matrix, \\(d\\) is a dampening term and \\(s\\) a scaling factor that are applied to help convergence. For details, see (Koh and Liang, 2017)<sup>1</sup> and the original paper (Agarwal et. al.)<sup>2</sup>.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A model wrapped in the TwiceDifferentiable interface.</p> <p> TYPE: <code>TorchTwiceDifferentiable</code> </p> <code>training_data</code> <p>A DataLoader containing the training data.</p> <p> TYPE: <code>DataLoader</code> </p> <code>b</code> <p>A vector or matrix, the right hand side of the equation \\(Hx = b\\).</p> <p> TYPE: <code>Tensor</code> </p> <code>hessian_perturbation</code> <p>Regularization of the hessian.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>maxiter</code> <p>Maximum number of iterations.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1000</code> </p> <code>dampen</code> <p>Dampening factor, defaults to 0 for no dampening.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>scale</code> <p>Scaling factor, defaults to 10.</p> <p> TYPE: <code>float</code> DEFAULT: <code>10.0</code> </p> <code>h0</code> <p>Initial guess for hvp.</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>rtol</code> <p>tolerance to use for early stopping</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0001</code> </p> <code>progress</code> <p>If True, display progress bars.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>InverseHvpResult</code> <p>Instance of InverseHvpResult, with a matrix of shape [NxP] with each line being a solution of \\(Ax=b\\), and a dictionary containing information about the accuracy of the solution.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@InversionRegistry.register(TorchTwiceDifferentiable, InversionMethod.Lissa)\ndef solve_lissa(\nmodel: TorchTwiceDifferentiable,\ntraining_data: DataLoader,\nb: torch.Tensor,\nhessian_perturbation: float = 0.0,\n*,\nmaxiter: int = 1000,\ndampen: float = 0.0,\nscale: float = 10.0,\nh0: Optional[torch.Tensor] = None,\nrtol: float = 1e-4,\nprogress: bool = False,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Uses LISSA, Linear time Stochastic Second-Order Algorithm, to iteratively\n    approximate the inverse Hessian. More precisely, it finds x s.t. \\(Hx = b\\),\n    with \\(H\\) being the model's second derivative wrt. the parameters.\n    This is done with the update\n    \\[H^{-1}_{j+1} b = b + (I - d) \\ H - \\frac{H^{-1}_j b}{s},\\]\n    where \\(I\\) is the identity matrix, \\(d\\) is a dampening term and \\(s\\) a scaling\n    factor that are applied to help convergence. For details, see\n    (Koh and Liang, 2017)&lt;sup&gt;&lt;a href=\"#koh_liang_2017\"&gt;1&lt;/a&gt;&lt;/sup&gt; and the original paper\n    (Agarwal et. al.)&lt;sup&gt;&lt;a href=\"#agarwal_secondorder_2017\"&gt;2&lt;/a&gt;&lt;/sup&gt;.\n    Args:\n        model: A model wrapped in the TwiceDifferentiable interface.\n        training_data: A DataLoader containing the training data.\n        b: A vector or matrix, the right hand side of the equation \\(Hx = b\\).\n        hessian_perturbation: Regularization of the hessian.\n        maxiter: Maximum number of iterations.\n        dampen: Dampening factor, defaults to 0 for no dampening.\n        scale: Scaling factor, defaults to 10.\n        h0: Initial guess for hvp.\n        rtol: tolerance to use for early stopping\n        progress: If True, display progress bars.\n    Returns:\n        Instance of [InverseHvpResult][pydvl.influence.twice_differentiable.InverseHvpResult], with a matrix of shape [NxP] with each line being a solution of \\(Ax=b\\),\n            and a dictionary containing information about the accuracy of the solution.\n    \"\"\"\nif h0 is None:\nh_estimate = torch.clone(b)\nelse:\nh_estimate = h0\nshuffled_training_data = DataLoader(\ntraining_data.dataset, training_data.batch_size, shuffle=True\n)\ndef lissa_step(\nh: torch.Tensor, reg_hvp: Callable[[torch.Tensor], torch.Tensor]\n) -&gt; torch.Tensor:\n\"\"\"Given an estimate of the hessian inverse and the regularised hessian\n        vector product, it computes the next estimate.\n        Args:\n            h: An estimate of the hessian inverse.\n            reg_hvp: Regularised hessian vector product.\n        Returns:\n            The next estimate of the hessian inverse.\n        \"\"\"\nreturn b + (1 - dampen) * h - reg_hvp(h) / scale\nfor _ in maybe_progress(range(maxiter), progress, desc=\"Lissa\"):\nx, y = next(iter(shuffled_training_data))\ngrad_xy = model.grad(x, y, create_graph=True)\nreg_hvp = (\nlambda v: model.mvp(grad_xy, v, model.parameters) + hessian_perturbation * v\n)\nresidual = lissa_step(h_estimate, reg_hvp) - h_estimate\nh_estimate += residual\nif torch.isnan(h_estimate).any():\nraise RuntimeError(\"NaNs in h_estimate. Increase scale or dampening.\")\nmax_residual = torch.max(torch.abs(residual / h_estimate))\nif max_residual &lt; rtol:\nbreak\nmean_residual = torch.mean(torch.abs(residual / h_estimate))\nlogger.info(\nf\"Terminated Lissa with {max_residual*100:.2f} % max residual.\"\nf\" Mean residual: {mean_residual*100:.5f} %\"\n)\ninfo = {\n\"max_perc_residual\": max_residual * 100,\n\"mean_perc_residual\": mean_residual * 100,\n}\nreturn InverseHvpResult(x=h_estimate / scale, info=info)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.solve_arnoldi","title":"<code>solve_arnoldi(model, training_data, b, hessian_perturbation=0.0, *, rank_estimate=10, krylov_dimension=None, low_rank_representation=None, tol=1e-06, max_iter=None, eigen_computation_on_gpu=False)</code>","text":"<p>Solves the linear system Hx = b, where H is the Hessian of the model's loss function and b is the given right-hand side vector. It employs the implicitly restarted Arnoldi method for computing a partial eigen decomposition, which is used fo the inversion i.e.</p> \\[x = V D^{-1} V^T b\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) <code>rank_estimate</code> eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model instance that is twice differentiable, wrapped into TorchTwiceDifferential. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>TorchTwiceDifferentiable</code> </p> <code>training_data</code> <p>A DataLoader instance that provides the model's training data. Used in calculating the Hessian-vector products.</p> <p> TYPE: <code>DataLoader</code> </p> <code>b</code> <p>The right-hand side vector in the system Hx = b.</p> <p> TYPE: <code>Tensor</code> </p> <code>hessian_perturbation</code> <p>Optional regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>rank_estimate</code> <p>The number of eigenvalues and corresponding eigenvectors to compute. Represents the desired rank of the Hessian approximation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>krylov_dimension</code> <p>The number of Krylov vectors to use for the Lanczos method. Defaults to min(model's number of parameters, max(2 times rank_estimate + 1, 20)).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>low_rank_representation</code> <p>An instance of LowRankProductRepresentation containing a previously computed low-rank representation of the Hessian. If provided, all other parameters are ignored; otherwise, a new low-rank representation is computed using provided parameters.</p> <p> TYPE: <code>Optional[LowRankProductRepresentation]</code> DEFAULT: <code>None</code> </p> <code>tol</code> <p>The stopping criteria for the Lanczos algorithm. Ignored if <code>low_rank_representation</code> is provided.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-06</code> </p> <code>max_iter</code> <p>The maximum number of iterations for the Lanczos method. Ignored if <code>low_rank_representation</code> is provided.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>eigen_computation_on_gpu</code> <p>If True, tries to execute the eigen pair approximation on the model's device via a cupy implementation. Ensure the model size or rank_estimate is appropriate for device memory. If False, the eigen pair approximation is executed on the CPU by the scipy wrapper to ARPACK.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>InverseHvpResult</code> <p>Instance of InverseHvpResult, having the solution vector x that satisfies the system \\(Ax = b\\), where \\(A\\) is a low-rank approximation of the Hessian \\(H\\) of the model's loss function, and an instance of LowRankProductRepresentation, which represents the approximation of H.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@InversionRegistry.register(TorchTwiceDifferentiable, InversionMethod.Arnoldi)\ndef solve_arnoldi(\nmodel: TorchTwiceDifferentiable,\ntraining_data: DataLoader,\nb: torch.Tensor,\nhessian_perturbation: float = 0.0,\n*,\nrank_estimate: int = 10,\nkrylov_dimension: Optional[int] = None,\nlow_rank_representation: Optional[LowRankProductRepresentation] = None,\ntol: float = 1e-6,\nmax_iter: Optional[int] = None,\neigen_computation_on_gpu: bool = False,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Solves the linear system Hx = b, where H is the Hessian of the model's loss function and b is the given\n    right-hand side vector.\n    It employs the [implicitly restarted Arnoldi method](https://en.wikipedia.org/wiki/Arnoldi_iteration) for\n    computing a partial eigen decomposition, which is used fo the inversion i.e.\n    \\[x = V D^{-1} V^T b\\]\n    where \\(D\\) is a diagonal matrix with the top (in absolute value) `rank_estimate` eigenvalues of the Hessian\n    and \\(V\\) contains the corresponding eigenvectors.\n    Args:\n        model: A PyTorch model instance that is twice differentiable, wrapped into\n            [TorchTwiceDifferential][pydvl.influence.torch.torch_differentiable.TorchTwiceDifferentiable].\n            The Hessian will be calculated with respect to this model's parameters.\n        training_data: A DataLoader instance that provides the model's training data.\n            Used in calculating the Hessian-vector products.\n        b: The right-hand side vector in the system Hx = b.\n        hessian_perturbation: Optional regularization parameter added to the Hessian-vector\n            product for numerical stability.\n        rank_estimate: The number of eigenvalues and corresponding eigenvectors to compute.\n            Represents the desired rank of the Hessian approximation.\n        krylov_dimension: The number of Krylov vectors to use for the Lanczos method.\n            Defaults to min(model's number of parameters, max(2 times rank_estimate + 1, 20)).\n        low_rank_representation: An instance of\n            [LowRankProductRepresentation][pydvl.influence.torch.torch_differentiable.LowRankProductRepresentation]\n            containing a previously computed low-rank representation of the Hessian. If provided, all other parameters\n            are ignored; otherwise, a new low-rank representation is computed\n            using provided parameters.\n        tol: The stopping criteria for the Lanczos algorithm.\n            Ignored if `low_rank_representation` is provided.\n        max_iter: The maximum number of iterations for the Lanczos method.\n            Ignored if `low_rank_representation` is provided.\n        eigen_computation_on_gpu: If True, tries to execute the eigen pair approximation on the model's device\n            via a cupy implementation. Ensure the model size or rank_estimate is appropriate for device memory.\n            If False, the eigen pair approximation is executed on the CPU by the scipy wrapper to ARPACK.\n    Returns:\n        Instance of [InverseHvpResult][pydvl.influence.torch.torch_differentiable.InverseHvpResult],\n            having the solution vector x that satisfies the system \\(Ax = b\\),\n            where \\(A\\) is a low-rank approximation of the Hessian \\(H\\) of the model's loss function, and an instance\n            of [LowRankProductRepresentation][pydvl.influence.torch.torch_differentiable.LowRankProductRepresentation],\n            which represents the approximation of H.\n    \"\"\"\nb_device = b.device if hasattr(b, \"device\") else torch.device(\"cpu\")\nif low_rank_representation is None:\nif b_device.type == \"cuda\" and not eigen_computation_on_gpu:\nraise ValueError(\n\"Using 'eigen_computation_on_gpu=False' while 'b' is on a 'cuda' device is not supported. \"\n\"To address this, consider the following options:\\n\"\n\" - Set eigen_computation_on_gpu=True if your model and data are small enough \"\n\"and if 'cupy' is available in your environment.\\n\"\n\" - Move 'b' to the CPU with b.to('cpu').\\n\"\n\" - Precompute a low rank representation and move it to the 'b' device using:\\n\"\n\"     low_rank_representation = model_hessian_low_rank(model, training_data, ..., \"\n\"eigen_computation_on_gpu=False).to(b.device)\"\n)\nlow_rank_representation = model_hessian_low_rank(\nmodel,\ntraining_data,\nhessian_perturbation=hessian_perturbation,\nrank_estimate=rank_estimate,\nkrylov_dimension=krylov_dimension,\ntol=tol,\nmax_iter=max_iter,\neigen_computation_on_gpu=eigen_computation_on_gpu,\n)\nelse:\nif b_device.type != low_rank_representation.device.type:\nraise RuntimeError(\nf\"The devices for 'b' and 'low_rank_representation' do not match.\\n\"\nf\" - 'b' is on device: {b_device}\\n\"\nf\" - 'low_rank_representation' is on device: {low_rank_representation.device}\\n\"\nf\"\\nTo resolve this, consider moving 'low_rank_representation' to '{b_device}' by using:\\n\"\nf\"low_rank_representation = low_rank_representation.to(b.device)\"\n)\nlogger.info(\"Using provided low rank representation, ignoring other parameters\")\nresult = low_rank_representation.projections @ (\ntorch.diag_embed(1.0 / low_rank_representation.eigen_vals)\n@ (low_rank_representation.projections.t() @ b.t())\n)\nreturn InverseHvpResult(\nx=result.t(),\ninfo={\n\"eigenvalues\": low_rank_representation.eigen_vals,\n\"eigenvectors\": low_rank_representation.projections,\n},\n)\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/","title":"Util","text":""},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.TorchTensorContainerType","title":"<code>TorchTensorContainerType = TypeVar('TorchTensorContainerType', torch.Tensor, Tuple[torch.Tensor, ...], Dict[str, torch.Tensor])</code>  <code>module-attribute</code>","text":"<p>Type variable for a PyTorch tensor or a container thereof.</p>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.to_model_device","title":"<code>to_model_device(x, model)</code>","text":"<p>Returns the tensor <code>x</code> moved to the device of the <code>model</code>, if device of model is set</p> PARAMETER  DESCRIPTION <code>x</code> <p>The tensor to be moved to the device of the model.</p> <p> TYPE: <code>Tensor</code> </p> <code>model</code> <p>The model whose device will be used to move the tensor.</p> <p> TYPE: <code>Module</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>The tensor <code>x</code> moved to the device of the <code>model</code>, if device of model is set.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def to_model_device(x: torch.Tensor, model: torch.nn.Module) -&gt; torch.Tensor:\n\"\"\"\n    Returns the tensor `x` moved to the device of the `model`, if device of model is set\n    Args:\n        x: The tensor to be moved to the device of the model.\n        model: The model whose device will be used to move the tensor.\n    Returns:\n        The tensor `x` moved to the device of the `model`, if device of model is set.\n    \"\"\"\nif hasattr(model, \"device\"):\nreturn x.to(model.device)\nreturn x\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.flatten_tensors_to_vector","title":"<code>flatten_tensors_to_vector(tensors)</code>","text":"<p>Flatten multiple tensors into a single 1D tensor (vector).</p> <p>This function takes an iterable of tensors and reshapes each of them into a 1D tensor. These reshaped tensors are then concatenated together into a single 1D tensor in the order they were given.</p> PARAMETER  DESCRIPTION <code>tensors</code> <p>An iterable of tensors to be reshaped and concatenated.</p> <p> TYPE: <code>Iterable[Tensor]</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A 1D tensor that is the concatenation of all the reshaped input tensors.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def flatten_tensors_to_vector(tensors: Iterable[torch.Tensor]) -&gt; torch.Tensor:\n\"\"\"\n    Flatten multiple tensors into a single 1D tensor (vector).\n    This function takes an iterable of tensors and reshapes each of them into a 1D tensor.\n    These reshaped tensors are then concatenated together into a single 1D tensor in the order they were given.\n    Args:\n        tensors: An iterable of tensors to be reshaped and concatenated.\n    Returns:\n        A 1D tensor that is the concatenation of all the reshaped input tensors.\n    \"\"\"\nreturn torch.cat([t.contiguous().view(-1) for t in tensors])\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.reshape_vector_to_tensors","title":"<code>reshape_vector_to_tensors(input_vector, target_shapes)</code>","text":"<p>Reshape a 1D tensor into multiple tensors with specified shapes.</p> <p>This function takes a 1D tensor (input_vector) and reshapes it into a series of tensors with shapes given by 'target_shapes'. The reshaped tensors are returned as a tuple in the same order as their corresponding shapes.</p> <p>Note: The total number of elements in 'input_vector' must be equal to the sum of the products of the shapes in 'target_shapes'.</p> PARAMETER  DESCRIPTION <code>input_vector</code> <p>The 1D tensor to be reshaped. Must be 1D.</p> <p> TYPE: <code>Tensor</code> </p> <code>target_shapes</code> <p>An iterable of tuples. Each tuple defines the shape of a tensor to be reshaped from the 'input_vector'.</p> <p> TYPE: <code>Iterable[Tuple[int, ...]]</code> </p> RETURNS DESCRIPTION <code>Tuple[Tensor, ...]</code> <p>A tuple of reshaped tensors.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If 'input_vector' is not a 1D tensor or if the total number of elements in 'input_vector' does not match the sum of the products of the shapes in 'target_shapes'.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def reshape_vector_to_tensors(\ninput_vector: torch.Tensor, target_shapes: Iterable[Tuple[int, ...]]\n) -&gt; Tuple[torch.Tensor, ...]:\n\"\"\"\n    Reshape a 1D tensor into multiple tensors with specified shapes.\n    This function takes a 1D tensor (input_vector) and reshapes it into a series of tensors with shapes given by 'target_shapes'.\n    The reshaped tensors are returned as a tuple in the same order as their corresponding shapes.\n    Note: The total number of elements in 'input_vector' must be equal to the sum of the products of the shapes in 'target_shapes'.\n    Args:\n        input_vector: The 1D tensor to be reshaped. Must be 1D.\n        target_shapes: An iterable of tuples. Each tuple defines the shape of a tensor to be reshaped from the 'input_vector'.\n    Returns:\n        A tuple of reshaped tensors.\n    Raises:\n        ValueError: If 'input_vector' is not a 1D tensor or if the total number of elements in 'input_vector' does not match the sum of the products of the shapes in 'target_shapes'.\n    \"\"\"\nif input_vector.dim() != 1:\nraise ValueError(\"Input vector must be a 1D tensor\")\ntotal_elements = sum(math.prod(shape) for shape in target_shapes)\nif total_elements != input_vector.shape[0]:\nraise ValueError(\nf\"The total elements in shapes {total_elements} does not match the vector length {input_vector.shape[0]}\"\n)\ntensors = []\nstart = 0\nfor shape in target_shapes:\nsize = math.prod(shape)  # compute the total size of the tensor with this shape\ntensors.append(\ninput_vector[start : start + size].view(shape)\n)  # slice the vector and reshape it\nstart += size\nreturn tuple(tensors)\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.align_structure","title":"<code>align_structure(source, target)</code>","text":"<p>This function transforms <code>target</code> to have the same structure as <code>source</code>, i.e., it should be a dictionary with the same keys as <code>source</code> and each corresponding value in <code>target</code> should have the same shape as the value in <code>source</code>.</p> PARAMETER  DESCRIPTION <code>source</code> <p>The reference dictionary containing PyTorch tensors.</p> <p> TYPE: <code>Dict[str, Tensor]</code> </p> <code>target</code> <p>The input to be harmonized. It can be a dictionary, tuple, or tensor.</p> <p> TYPE: <code>TorchTensorContainerType</code> </p> RETURNS DESCRIPTION <code>Dict[str, Tensor]</code> <p>The harmonized version of <code>target</code>.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If <code>target</code> cannot be harmonized to match <code>source</code>.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def align_structure(\nsource: Dict[str, torch.Tensor],\ntarget: TorchTensorContainerType,\n) -&gt; Dict[str, torch.Tensor]:\n\"\"\"\n    This function transforms `target` to have the same structure as `source`, i.e.,\n    it should be a dictionary with the same keys as `source` and each corresponding\n    value in `target` should have the same shape as the value in `source`.\n    Args:\n        source: The reference dictionary containing PyTorch tensors.\n        target: The input to be harmonized. It can be a dictionary, tuple, or tensor.\n    Returns:\n        The harmonized version of `target`.\n    Raises:\n        ValueError: If `target` cannot be harmonized to match `source`.\n    \"\"\"\ntangent_dict: Dict[str, torch.Tensor]\nif isinstance(target, dict):\nif list(target.keys()) != list(source.keys()):\nraise ValueError(\"The keys in 'target' do not match the keys in 'source'.\")\nif [v.shape for v in target.values()] != [v.shape for v in source.values()]:\nraise ValueError(\n\"The shapes of the values in 'target' do not match the shapes of the values in 'source'.\"\n)\ntangent_dict = target\nelif isinstance(target, tuple) or isinstance(target, list):\nif [v.shape for v in target] != [v.shape for v in source.values()]:\nraise ValueError(\n\"'target' is a tuple/list but its elements' shapes do not match the shapes \"\n\"of the values in 'source'.\"\n)\ntangent_dict = dict(zip(source.keys(), target))\nelif isinstance(target, torch.Tensor):\ntry:\ntangent_dict = dict(\nzip(\nsource.keys(),\nreshape_vector_to_tensors(\ntarget, [p.shape for p in source.values()]\n),\n)\n)\nexcept Exception as e:\nraise ValueError(\nf\"'target' is a tensor but cannot be reshaped to match 'source'. Original error: {e}\"\n)\nelse:\nraise ValueError(f\"'target' is of type {type(target)} which is not supported.\")\nreturn tangent_dict\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.as_tensor","title":"<code>as_tensor(a, warn=True, **kwargs)</code>","text":"<p>Converts an array into a torch tensor.</p> PARAMETER  DESCRIPTION <code>a</code> <p>Array to convert to tensor.</p> <p> TYPE: <code>Any</code> </p> <code>warn</code> <p>If True, warns that <code>a</code> will be converted.</p> <p> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A torch tensor converted from the input array.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def as_tensor(a: Any, warn=True, **kwargs) -&gt; torch.Tensor:\n\"\"\"\n    Converts an array into a torch tensor.\n    Args:\n        a: Array to convert to tensor.\n        warn: If True, warns that `a` will be converted.\n    Returns:\n        A torch tensor converted from the input array.\n    \"\"\"\nif warn and not isinstance(a, torch.Tensor):\nlogger.warning(\"Converting tensor to type torch.Tensor.\")\nreturn torch.as_tensor(a, **kwargs)\n</code></pre>"},{"location":"api/pydvl/parallel/","title":"Parallel","text":"<p>This module provides a common interface to parallelization backends. The list of supported backends is here. Backends can be selected with the <code>backend</code> argument of an instance of ParallelConfig, as seen in the examples below.</p> <p>We use executors to submit tasks in parallel. The basic high-level pattern is</p> <pre><code>from pydvl.parallel import init_executor, ParallelConfig\nconfig = ParallelConfig(backend=\"ray\")\nwith init_executor(max_workers=1, config=config) as executor:\nfuture = executor.submit(lambda x: x + 1, 1)\nresult = future.result()\nassert result == 2\n</code></pre> <p>Running a map-reduce job is also easy:</p> <pre><code>from pydvl.parallel import init_executor, ParallelConfig\nconfig = ParallelConfig(backend=\"joblib\")\nwith init_executor(config=config) as executor:\nresults = list(executor.map(lambda x: x + 1, range(5)))\nassert results == [1, 2, 3, 4, 5]\n</code></pre> <p>There is an alternative map-reduce implementation MapReduceJob which internally uses joblib's higher level API with <code>Parallel()</code></p>"},{"location":"api/pydvl/parallel/backend/","title":"Backend","text":""},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.CancellationPolicy","title":"<code>CancellationPolicy</code>","text":"<p>             Bases: <code>Flag</code></p> <p>Policy to use when cancelling futures after exiting an Executor.</p> <p>Note</p> <p>Not all backends support all policies.</p> ATTRIBUTE DESCRIPTION <code>NONE</code> <p>Do not cancel any futures.</p> <p> </p> <code>PENDING</code> <p>Cancel all pending futures, but not running ones.</p> <p> </p> <code>RUNNING</code> <p>Cancel all running futures, but not pending ones.</p> <p> </p> <code>ALL</code> <p>Cancel all pending and running futures.</p> <p> </p>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.BaseParallelBackend","title":"<code>BaseParallelBackend</code>","text":"<p>Abstract base class for all parallel backends.</p>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.BaseParallelBackend.executor","title":"<code>executor(max_workers=None, config=ParallelConfig(), cancel_futures=CancellationPolicy.PENDING)</code>  <code>abstractmethod</code> <code>classmethod</code>","text":"<p>Returns an executor for the parallel backend.</p> Source code in <code>src/pydvl/parallel/backend.py</code> <pre><code>@classmethod\n@abstractmethod\ndef executor(\ncls,\nmax_workers: int | None = None,\nconfig: ParallelConfig = ParallelConfig(),\ncancel_futures: CancellationPolicy = CancellationPolicy.PENDING,\n) -&gt; Executor:\n\"\"\"Returns an executor for the parallel backend.\"\"\"\n...\n</code></pre>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.init_parallel_backend","title":"<code>init_parallel_backend(config)</code>","text":"<p>Initializes the parallel backend and returns an instance of it.</p> <p>The following example creates a parallel backend instance with the default configuration, which is a local joblib backend.</p> Example <pre><code>config = ParallelConfig()\nparallel_backend = init_parallel_backend(config)\n</code></pre> <p>To create a parallel backend instance with a different backend, e.g. ray, you can pass the backend name as a string to the constructor of ParallelConfig.</p> Example <pre><code>config = ParallelConfig(backend=\"ray\")\nparallel_backend = init_parallel_backend(config)\n</code></pre> PARAMETER  DESCRIPTION <code>config</code> <p>instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> </p> Source code in <code>src/pydvl/parallel/backend.py</code> <pre><code>def init_parallel_backend(config: ParallelConfig) -&gt; BaseParallelBackend:\n\"\"\"Initializes the parallel backend and returns an instance of it.\n    The following example creates a parallel backend instance with the default\n    configuration, which is a local joblib backend.\n    ??? Example\n        ``` python\n        config = ParallelConfig()\n        parallel_backend = init_parallel_backend(config)\n        ```\n    To create a parallel backend instance with a different backend, e.g. ray,\n    you can pass the backend name as a string to the constructor of\n    [ParallelConfig][pydvl.utils.config.ParallelConfig].\n    ??? Example\n        ```python\n        config = ParallelConfig(backend=\"ray\")\n        parallel_backend = init_parallel_backend(config)\n        ```\n    Args:\n        config: instance of [ParallelConfig][pydvl.utils.config.ParallelConfig]\n            with cluster address, number of cpus, etc.\n    \"\"\"\ntry:\nparallel_backend_cls = BaseParallelBackend.BACKENDS[config.backend]\nexcept KeyError:\nraise NotImplementedError(f\"Unexpected parallel backend {config.backend}\")\nreturn parallel_backend_cls.create(config)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.available_cpus","title":"<code>available_cpus()</code>","text":"<p>Platform-independent count of available cores.</p> <p>FIXME: do we really need this or is <code>os.cpu_count</code> enough? Is this portable?</p> RETURNS DESCRIPTION <code>int</code> <p>Number of cores, or 1 if it is not possible to determine.</p> Source code in <code>src/pydvl/parallel/backend.py</code> <pre><code>def available_cpus() -&gt; int:\n\"\"\"Platform-independent count of available cores.\n    FIXME: do we really need this or is `os.cpu_count` enough? Is this portable?\n    Returns:\n        Number of cores, or 1 if it is not possible to determine.\n    \"\"\"\nfrom platform import system\nif system() != \"Linux\":\nreturn os.cpu_count() or 1\nreturn len(os.sched_getaffinity(0))  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.effective_n_jobs","title":"<code>effective_n_jobs(n_jobs, config=ParallelConfig())</code>","text":"<p>Returns the effective number of jobs.</p> <p>This number may vary depending on the parallel backend and the resources available.</p> PARAMETER  DESCRIPTION <code>n_jobs</code> <p>the number of jobs requested. If -1, the number of available CPUs is returned.</p> <p> TYPE: <code>int</code> </p> <code>config</code> <p>instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> RETURNS DESCRIPTION <code>int</code> <p>The effective number of jobs, guaranteed to be &gt;= 1.</p> RAISES DESCRIPTION <code>RuntimeError</code> <p>if the effective number of jobs returned by the backend is &lt; 1.</p> Source code in <code>src/pydvl/parallel/backend.py</code> <pre><code>def effective_n_jobs(n_jobs: int, config: ParallelConfig = ParallelConfig()) -&gt; int:\n\"\"\"Returns the effective number of jobs.\n    This number may vary depending on the parallel backend and the resources\n    available.\n    Args:\n        n_jobs: the number of jobs requested. If -1, the number of available\n            CPUs is returned.\n        config: instance of [ParallelConfig][pydvl.utils.config.ParallelConfig] with\n            cluster address, number of cpus, etc.\n    Returns:\n        The effective number of jobs, guaranteed to be &gt;= 1.\n    Raises:\n        RuntimeError: if the effective number of jobs returned by the backend\n            is &lt; 1.\n    \"\"\"\nparallel_backend = init_parallel_backend(config)\nif (eff_n_jobs := parallel_backend.effective_n_jobs(n_jobs)) &lt; 1:\nraise RuntimeError(\nf\"Invalid number of jobs {eff_n_jobs} obtained from parallel backend {config.backend}\"\n)\nreturn eff_n_jobs\n</code></pre>"},{"location":"api/pydvl/parallel/config/","title":"Config","text":""},{"location":"api/pydvl/parallel/config/#pydvl.parallel.config.ParallelConfig","title":"<code>ParallelConfig</code>  <code>dataclass</code>","text":"<p>Configuration for parallel computation backend.</p> PARAMETER  DESCRIPTION <code>backend</code> <p>Type of backend to use. Defaults to 'joblib'</p> <p> TYPE: <code>Literal['joblib', 'ray']</code> DEFAULT: <code>'joblib'</code> </p> <code>address</code> <p>Address of existing remote or local cluster to use.</p> <p> TYPE: <code>Optional[Union[str, Tuple[str, int]]]</code> DEFAULT: <code>None</code> </p> <code>n_cpus_local</code> <p>Number of CPUs to use when creating a local ray cluster. This has no effect when using an existing ray cluster.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>logging_level</code> <p>Logging level for the parallel backend's worker.</p> <p> TYPE: <code>int</code> DEFAULT: <code>WARNING</code> </p> <code>wait_timeout</code> <p>Timeout in seconds for waiting on futures.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p>"},{"location":"api/pydvl/parallel/map_reduce/","title":"Map reduce","text":"<p>This module contains a wrapper around joblib's <code>Parallel()</code> class that makes it easy to run map-reduce jobs.</p> <p>Deprecation</p> <p>This interface might be deprecated or changed in a future release before 1.0</p>"},{"location":"api/pydvl/parallel/map_reduce/#pydvl.parallel.map_reduce.MapReduceJob","title":"<code>MapReduceJob(inputs, map_func, reduce_func=identity, map_kwargs=None, reduce_kwargs=None, config=ParallelConfig(), *, n_jobs=-1, timeout=None)</code>","text":"<p>             Bases: <code>Generic[T, R]</code></p> <p>Takes an embarrassingly parallel fun and runs it in <code>n_jobs</code> parallel jobs, splitting the data evenly into a number of chunks equal to the number of jobs.</p> <p>Typing information for objects of this class requires the type of the inputs that are split for <code>map_func</code> and the type of its output.</p> PARAMETER  DESCRIPTION <code>inputs</code> <p>The input that will be split and passed to <code>map_func</code>. if it's not a sequence object. It will be repeat <code>n_jobs</code> number of times.</p> <p> TYPE: <code>Union[Collection[T], T]</code> </p> <code>map_func</code> <p>Function that will be applied to the input chunks in each job.</p> <p> TYPE: <code>MapFunction[R]</code> </p> <code>reduce_func</code> <p>Function that will be applied to the results of <code>map_func</code> to reduce them.</p> <p> TYPE: <code>ReduceFunction[R]</code> DEFAULT: <code>identity</code> </p> <code>map_kwargs</code> <p>Keyword arguments that will be passed to <code>map_func</code> in each job. Alternatively, one can use functools.partial.</p> <p> TYPE: <code>Optional[Dict]</code> DEFAULT: <code>None</code> </p> <code>reduce_kwargs</code> <p>Keyword arguments that will be passed to <code>reduce_func</code> in each job. Alternatively, one can use functools.partial.</p> <p> TYPE: <code>Optional[Dict]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>Instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to run. Does not accept 0</p> <p> TYPE: <code>int</code> DEFAULT: <code>-1</code> </p> Example <p>A simple usage example with 2 jobs:</p> <pre><code>&gt;&gt;&gt; from pydvl.parallel import MapReduceJob\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; map_reduce_job: MapReduceJob[np.ndarray, np.ndarray] = MapReduceJob(\n...     np.arange(5),\n...     map_func=np.sum,\n...     reduce_func=np.sum,\n...     n_jobs=2,\n... )\n&gt;&gt;&gt; map_reduce_job()\n10\n</code></pre> <p>When passed a single object as input, it will be repeated for each job: <pre><code>&gt;&gt;&gt; from pydvl.parallel import MapReduceJob\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; map_reduce_job: MapReduceJob[int, np.ndarray] = MapReduceJob(\n...     5,\n...     map_func=lambda x: np.array([x]),\n...     reduce_func=np.sum,\n...     n_jobs=2,\n... )\n&gt;&gt;&gt; map_reduce_job()\n10\n</code></pre></p> Source code in <code>src/pydvl/parallel/map_reduce.py</code> <pre><code>def __init__(\nself,\ninputs: Union[Collection[T], T],\nmap_func: MapFunction[R],\nreduce_func: ReduceFunction[R] = identity,\nmap_kwargs: Optional[Dict] = None,\nreduce_kwargs: Optional[Dict] = None,\nconfig: ParallelConfig = ParallelConfig(),\n*,\nn_jobs: int = -1,\ntimeout: Optional[float] = None,\n):\nself.config = config\nparallel_backend = init_parallel_backend(self.config)\nself.parallel_backend = parallel_backend\nself.timeout = timeout\n# This uses the setter defined below\nself.n_jobs = n_jobs\nself.inputs_ = inputs\nself.map_kwargs = map_kwargs if map_kwargs is not None else dict()\nself.reduce_kwargs = reduce_kwargs if reduce_kwargs is not None else dict()\nself._map_func = reduce(maybe_add_argument, [\"job_id\", \"seed\"], map_func)\nself._reduce_func = reduce_func\n</code></pre>"},{"location":"api/pydvl/parallel/map_reduce/#pydvl.parallel.map_reduce.MapReduceJob.n_jobs","title":"<code>n_jobs: int</code>  <code>property</code> <code>writable</code>","text":"<p>Effective number of jobs according to the used ParallelBackend instance.</p>"},{"location":"api/pydvl/parallel/map_reduce/#pydvl.parallel.map_reduce.MapReduceJob.__call__","title":"<code>__call__(seed=None)</code>","text":"<p>Runs the map-reduce job.</p> PARAMETER  DESCRIPTION <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Union[Seed, SeedSequence]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>R</code> <p>The result of the reduce function.</p> Source code in <code>src/pydvl/parallel/map_reduce.py</code> <pre><code>def __call__(\nself,\nseed: Optional[Union[Seed, SeedSequence]] = None,\n) -&gt; R:\n\"\"\"\n    Runs the map-reduce job.\n    Args:\n        seed: Either an instance of a numpy random number generator or a seed for\n            it.\n    Returns:\n         The result of the reduce function.\n    \"\"\"\nif self.config.backend == \"joblib\":\nbackend = \"loky\"\nelse:\nbackend = self.config.backend\n# In joblib the levels are reversed.\n# 0 means no logging and 50 means log everything to stdout\nverbose = 50 - self.config.logging_level\nseed_seq = ensure_seed_sequence(seed)\nwith Parallel(backend=backend, n_jobs=self.n_jobs, verbose=verbose) as parallel:\nchunks = self._chunkify(self.inputs_, n_chunks=self.n_jobs)\nmap_results: List[R] = parallel(\ndelayed(self._map_func)(\nnext_chunk, job_id=j, seed=seed, **self.map_kwargs\n)\nfor j, (next_chunk, seed) in enumerate(\nzip(chunks, seed_seq.spawn(len(chunks)))\n)\n)\nreduce_results: R = self._reduce_func(map_results, **self.reduce_kwargs)\nreturn reduce_results\n</code></pre>"},{"location":"api/pydvl/parallel/backends/","title":"Backends","text":""},{"location":"api/pydvl/parallel/backends/joblib/","title":"Joblib","text":""},{"location":"api/pydvl/parallel/backends/joblib/#pydvl.parallel.backends.joblib.JoblibParallelBackend","title":"<code>JoblibParallelBackend(config)</code>","text":"<p>             Bases: <code>BaseParallelBackend</code></p> <p>Class used to wrap joblib to make it transparent to algorithms.</p> <p>It shouldn't be initialized directly. You should instead call init_parallel_backend().</p> PARAMETER  DESCRIPTION <code>config</code> <p>instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> </p> Source code in <code>src/pydvl/parallel/backends/joblib.py</code> <pre><code>def __init__(self, config: ParallelConfig):\nself.config = {\n\"logging_level\": config.logging_level,\n\"n_jobs\": config.n_cpus_local,\n}\n</code></pre>"},{"location":"api/pydvl/parallel/backends/joblib/#pydvl.parallel.backends.joblib.JoblibParallelBackend.wrap","title":"<code>wrap(fun, **kwargs)</code>","text":"<p>Wraps a function as a joblib delayed.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>the function to wrap</p> <p> TYPE: <code>Callable</code> </p> RETURNS DESCRIPTION <code>Callable</code> <p>The delayed function.</p> Source code in <code>src/pydvl/parallel/backends/joblib.py</code> <pre><code>def wrap(self, fun: Callable, **kwargs) -&gt; Callable:\n\"\"\"Wraps a function as a joblib delayed.\n    Args:\n        fun: the function to wrap\n    Returns:\n        The delayed function.\n    \"\"\"\nreturn delayed(fun)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/backends/ray/","title":"Ray","text":""},{"location":"api/pydvl/parallel/backends/ray/#pydvl.parallel.backends.ray.RayParallelBackend","title":"<code>RayParallelBackend(config)</code>","text":"<p>             Bases: <code>BaseParallelBackend</code></p> <p>Class used to wrap ray to make it transparent to algorithms.</p> <p>It shouldn't be initialized directly. You should instead call init_parallel_backend().</p> PARAMETER  DESCRIPTION <code>config</code> <p>instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> </p> Source code in <code>src/pydvl/parallel/backends/ray.py</code> <pre><code>def __init__(self, config: ParallelConfig):\nself.config = {\"address\": config.address, \"logging_level\": config.logging_level}\nif self.config[\"address\"] is None:\nself.config[\"num_cpus\"] = config.n_cpus_local\nif not ray.is_initialized():\nray.init(**self.config)\n# Register ray joblib backend\nregister_ray()\n</code></pre>"},{"location":"api/pydvl/parallel/backends/ray/#pydvl.parallel.backends.ray.RayParallelBackend.wrap","title":"<code>wrap(fun, **kwargs)</code>","text":"<p>Wraps a function as a ray remote.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>the function to wrap</p> <p> TYPE: <code>Callable</code> </p> <code>kwargs</code> <p>keyword arguments to pass to @ray.remote</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Callable</code> <p>The <code>.remote</code> method of the ray <code>RemoteFunction</code>.</p> Source code in <code>src/pydvl/parallel/backends/ray.py</code> <pre><code>def wrap(self, fun: Callable, **kwargs) -&gt; Callable:\n\"\"\"Wraps a function as a ray remote.\n    Args:\n        fun: the function to wrap\n        kwargs: keyword arguments to pass to @ray.remote\n    Returns:\n        The `.remote` method of the ray `RemoteFunction`.\n    \"\"\"\nif len(kwargs) &gt; 0:\nreturn ray.remote(**kwargs)(fun).remote  # type: ignore\nreturn ray.remote(fun).remote  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/futures/","title":"Futures","text":""},{"location":"api/pydvl/parallel/futures/#pydvl.parallel.futures.init_executor","title":"<code>init_executor(max_workers=None, config=ParallelConfig(), **kwargs)</code>","text":"<p>Initializes a futures executor for the given parallel configuration.</p> PARAMETER  DESCRIPTION <code>max_workers</code> <p>Maximum number of concurrent tasks.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>kwargs</code> <p>Other optional parameter that will be passed to the executor.</p> <p> DEFAULT: <code>{}</code> </p> Examples <p><pre><code>from pydvl.parallel import init_executor, ParallelConfig\nconfig = ParallelConfig(backend=\"ray\")\nwith init_executor(max_workers=1, config=config) as executor:\nfuture = executor.submit(lambda x: x + 1, 1)\nresult = future.result()\nassert result == 2\n</code></pre> <pre><code>from pydvl.parallel.futures import init_executor\nwith init_executor() as executor:\nresults = list(executor.map(lambda x: x + 1, range(5)))\nassert results == [1, 2, 3, 4, 5]\n</code></pre></p> Source code in <code>src/pydvl/parallel/futures/__init__.py</code> <pre><code>@contextmanager\ndef init_executor(\nmax_workers: Optional[int] = None,\nconfig: ParallelConfig = ParallelConfig(),\n**kwargs,\n) -&gt; Generator[Executor, None, None]:\n\"\"\"Initializes a futures executor for the given parallel configuration.\n    Args:\n        max_workers: Maximum number of concurrent tasks.\n        config: instance of [ParallelConfig][pydvl.utils.config.ParallelConfig]\n            with cluster address, number of cpus, etc.\n        kwargs: Other optional parameter that will be passed to the executor.\n    ??? Examples\n        ``` python\n        from pydvl.parallel import init_executor, ParallelConfig\n        config = ParallelConfig(backend=\"ray\")\n        with init_executor(max_workers=1, config=config) as executor:\n            future = executor.submit(lambda x: x + 1, 1)\n            result = future.result()\n        assert result == 2\n        ```\n        ``` python\n        from pydvl.parallel.futures import init_executor\n        with init_executor() as executor:\n            results = list(executor.map(lambda x: x + 1, range(5)))\n        assert results == [1, 2, 3, 4, 5]\n        ```\n    \"\"\"\ntry:\ncls = BaseParallelBackend.BACKENDS[config.backend]\nwith cls.executor(max_workers=max_workers, config=config, **kwargs) as e:\nyield e\nexcept KeyError:\nraise NotImplementedError(f\"Unexpected parallel backend {config.backend}\")\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/","title":"Ray","text":""},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor","title":"<code>RayExecutor(max_workers=None, *, config=ParallelConfig(), cancel_futures=CancellationPolicy.ALL)</code>","text":"<p>             Bases: <code>Executor</code></p> <p>Asynchronous executor using Ray that implements the concurrent.futures API.</p> <p>It shouldn't be initialized directly. You should instead call init_executor().</p> PARAMETER  DESCRIPTION <code>max_workers</code> <p>Maximum number of concurrent tasks. Each task can request itself any number of vCPUs. You must ensure the product of this value and the n_cpus_per_job parameter passed to submit() does not exceed available cluster resources. If set to <code>None</code>, it will default to the total number of vCPUs in the ray cluster.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>cancel_futures</code> <p>Select which futures will be cancelled when exiting this context manager. <code>Pending</code> is the default, which will cancel all pending futures, but not running ones, as done by concurrent.futures.ProcessPoolExecutor. Additionally, <code>All</code> cancels all pending and running futures, and <code>None</code> doesn't cancel any. See CancellationPolicy</p> <p> TYPE: <code>CancellationPolicy</code> DEFAULT: <code>ALL</code> </p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>@deprecated(\ntarget=True,\ndeprecated_in=\"0.7.0\",\nremove_in=\"0.8.0\",\nargs_mapping={\"cancel_futures_on_exit\": \"cancel_futures\"},\n)\ndef __init__(\nself,\nmax_workers: Optional[int] = None,\n*,\nconfig: ParallelConfig = ParallelConfig(),\ncancel_futures: CancellationPolicy = CancellationPolicy.ALL,\n):\nif config.backend != \"ray\":\nraise ValueError(\nf\"Parallel backend must be set to 'ray' and not '{config.backend}'\"\n)\nif max_workers is not None:\nif max_workers &lt;= 0:\nraise ValueError(\"max_workers must be greater than 0\")\nmax_workers = max_workers\nif isinstance(cancel_futures, CancellationPolicy):\nself._cancel_futures = cancel_futures\nelse:\nself._cancel_futures = (\nCancellationPolicy.PENDING\nif cancel_futures\nelse CancellationPolicy.NONE\n)\nself.config = {\"address\": config.address, \"logging_level\": config.logging_level}\nif config.address is None:\nself.config[\"num_cpus\"] = config.n_cpus_local\nif not ray.is_initialized():\nray.init(**self.config)\nself._max_workers = max_workers\nif self._max_workers is None:\nself._max_workers = int(ray._private.state.cluster_resources()[\"CPU\"])\nself._shutdown = False\nself._shutdown_lock = threading.Lock()\nself._queue_lock = threading.Lock()\nself._work_queue: \"queue.Queue[Optional[_WorkItem]]\" = queue.Queue(\nmaxsize=self._max_workers\n)\nself._pending_queue: \"queue.SimpleQueue[Optional[_WorkItem]]\" = (\nqueue.SimpleQueue()\n)\n# Work Item Manager Thread\nself._work_item_manager_thread: Optional[_WorkItemManagerThread] = None\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor.submit","title":"<code>submit(fn, *args, **kwargs)</code>","text":"<p>Submits a callable to be executed with the given arguments.</p> <p>Schedules the callable to be executed as fn(*args, **kwargs) and returns a Future instance representing the execution of the callable.</p> PARAMETER  DESCRIPTION <code>fn</code> <p>Callable.</p> <p> TYPE: <code>Callable[..., T]</code> </p> <code>args</code> <p>Positional arguments that will be passed to <code>fn</code>.</p> <p> DEFAULT: <code>()</code> </p> <code>kwargs</code> <p>Keyword arguments that will be passed to <code>fn</code>. It can also optionally contain options for the ray remote function as a dictionary as the keyword argument <code>remote_function_options</code>.</p> <p> DEFAULT: <code>{}</code> </p> <p>Returns:     A Future representing the given call.</p> RAISES DESCRIPTION <code>RuntimeError</code> <p>If a task is submitted after the executor has been shut down.</p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>def submit(self, fn: Callable[..., T], *args, **kwargs) -&gt; \"Future[T]\":\nr\"\"\"Submits a callable to be executed with the given arguments.\n    Schedules the callable to be executed as fn(\\*args, \\**kwargs)\n    and returns a Future instance representing the execution of the callable.\n    Args:\n        fn: Callable.\n        args: Positional arguments that will be passed to `fn`.\n        kwargs: Keyword arguments that will be passed to `fn`.\n            It can also optionally contain options for the ray remote function\n            as a dictionary as the keyword argument `remote_function_options`.\n    Returns:\n        A Future representing the given call.\n    Raises:\n        RuntimeError: If a task is submitted after the executor has been shut down.\n    \"\"\"\nwith self._shutdown_lock:\nlogger.debug(\"executor acquired shutdown lock\")\nif self._shutdown:\nraise RuntimeError(\"cannot schedule new futures after shutdown\")\nlogging.debug(\"Creating future and putting work item in work queue\")\nfuture: \"Future[T]\" = Future()\nremote_function_options = kwargs.pop(\"remote_function_options\", None)\nw = _WorkItem(\nfuture,\nfn,\nargs,\nkwargs,\nremote_function_options=remote_function_options,\n)\nself._put_work_item_in_queue(w)\n# We delay starting the thread until the first call to submit\nself._start_work_item_manager_thread()\nreturn future\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor.shutdown","title":"<code>shutdown(wait=True, *, cancel_futures=None)</code>","text":"<p>Clean up the resources associated with the Executor.</p> <p>This method tries to mimic the behaviour of Executor.shutdown while allowing one more value for <code>cancel_futures</code> which instructs it to use the CancellationPolicy defined upon construction.</p> PARAMETER  DESCRIPTION <code>wait</code> <p>Whether to wait for pending futures to finish.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>cancel_futures</code> <p>Overrides the executor's default policy for cancelling futures on exit. If <code>True</code>, all pending futures are cancelled, and if <code>False</code>, no futures are cancelled. If <code>None</code> (default), the executor's policy set at initialization is used.</p> <p> TYPE: <code>Optional[bool]</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>def shutdown(\nself, wait: bool = True, *, cancel_futures: Optional[bool] = None\n) -&gt; None:\n\"\"\"Clean up the resources associated with the Executor.\n    This method tries to mimic the behaviour of\n    [Executor.shutdown][concurrent.futures.Executor.shutdown]\n    while allowing one more value for ``cancel_futures`` which instructs it\n    to use the [CancellationPolicy][pydvl.parallel.backend.CancellationPolicy]\n    defined upon construction.\n    Args:\n        wait: Whether to wait for pending futures to finish.\n        cancel_futures: Overrides the executor's default policy for\n            cancelling futures on exit. If ``True``, all pending futures are\n            cancelled, and if ``False``, no futures are cancelled. If ``None``\n            (default), the executor's policy set at initialization is used.\n    \"\"\"\nlogger.debug(\"executor shutting down\")\nwith self._shutdown_lock:\nlogger.debug(\"executor acquired shutdown lock\")\nself._shutdown = True\nself._cancel_futures = {\nNone: self._cancel_futures,\nTrue: CancellationPolicy.PENDING,\nFalse: CancellationPolicy.NONE,\n}[cancel_futures]\nif wait:\nlogger.debug(\"executor waiting for futures to finish\")\nif self._work_item_manager_thread is not None:\n# Putting None in the queue to signal\n# to work item manager thread that we are shutting down\nself._put_work_item_in_queue(None)\nlogger.debug(\n\"executor waiting for work item manager thread to terminate\"\n)\nself._work_item_manager_thread.join()\n# To reduce the risk of opening too many files, remove references to\n# objects that use file descriptors.\nself._work_item_manager_thread = None\ndel self._work_queue\ndel self._pending_queue\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor.__exit__","title":"<code>__exit__(exc_type, exc_val, exc_tb)</code>","text":"<p>Exit the runtime context related to the RayExecutor object.</p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>def __exit__(self, exc_type, exc_val, exc_tb):\n\"\"\"Exit the runtime context related to the RayExecutor object.\"\"\"\nself.shutdown()\nreturn False\n</code></pre>"},{"location":"api/pydvl/reporting/","title":"Reporting","text":""},{"location":"api/pydvl/reporting/plots/","title":"Plots","text":""},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.shaded_mean_std","title":"<code>shaded_mean_std(data, abscissa=None, num_std=1.0, mean_color='dodgerblue', shade_color='lightblue', title=None, xlabel=None, ylabel=None, ax=None, **kwargs)</code>","text":"<p>The usual mean \\(\\pm\\) std deviation plot to aggregate runs of experiments.</p> PARAMETER  DESCRIPTION <code>data</code> <p>axis 0 is to be aggregated on (e.g. runs) and axis 1 is the data for each run.</p> <p> TYPE: <code>ndarray</code> </p> <code>abscissa</code> <p>values for the x-axis. Leave empty to use increasing integers.</p> <p> TYPE: <code>Optional[Sequence[Any]]</code> DEFAULT: <code>None</code> </p> <code>num_std</code> <p>number of standard deviations to shade around the mean.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>mean_color</code> <p>color for the mean</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'dodgerblue'</code> </p> <code>shade_color</code> <p>color for the shaded region</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'lightblue'</code> </p> <code>title</code> <p>Title text. To use mathematics, use LaTeX notation.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>xlabel</code> <p>Text for the horizontal axis.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ylabel</code> <p>Text for the vertical axis</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ax</code> <p>If passed, axes object into which to insert the figure. Otherwise, a new figure is created and returned</p> <p> TYPE: <code>Optional[Axes]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>these are forwarded to the ax.plot() call for the mean.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Axes</code> <p>The axes used (or created)</p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def shaded_mean_std(\ndata: np.ndarray,\nabscissa: Optional[Sequence[Any]] = None,\nnum_std: float = 1.0,\nmean_color: Optional[str] = \"dodgerblue\",\nshade_color: Optional[str] = \"lightblue\",\ntitle: Optional[str] = None,\nxlabel: Optional[str] = None,\nylabel: Optional[str] = None,\nax: Optional[Axes] = None,\n**kwargs,\n) -&gt; Axes:\n\"\"\"The usual mean \\(\\pm\\) std deviation plot to aggregate runs of experiments.\n    Args:\n        data: axis 0 is to be aggregated on (e.g. runs) and axis 1 is the\n            data for each run.\n        abscissa: values for the x-axis. Leave empty to use increasing integers.\n        num_std: number of standard deviations to shade around the mean.\n        mean_color: color for the mean\n        shade_color: color for the shaded region\n        title: Title text. To use mathematics, use LaTeX notation.\n        xlabel: Text for the horizontal axis.\n        ylabel: Text for the vertical axis\n        ax: If passed, axes object into which to insert the figure. Otherwise,\n            a new figure is created and returned\n        kwargs: these are forwarded to the ax.plot() call for the mean.\n    Returns:\n        The axes used (or created)\n    \"\"\"\nassert len(data.shape) == 2\nmean = data.mean(axis=0)\nstd = num_std * data.std(axis=0)\nif ax is None:\nfig, ax = plt.subplots()\nif abscissa is None:\nabscissa = list(range(data.shape[1]))\nax.fill_between(abscissa, mean - std, mean + std, alpha=0.3, color=shade_color)\nax.plot(abscissa, mean, color=mean_color, **kwargs)\nax.set_title(title)\nax.set_xlabel(xlabel)\nax.set_ylabel(ylabel)\nreturn ax\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.spearman_correlation","title":"<code>spearman_correlation(vv, num_values, pvalue)</code>","text":"<p>Simple matrix plots with spearman correlation for each pair in vv.</p> PARAMETER  DESCRIPTION <code>vv</code> <p>list of OrderedDicts with index: value. Spearman correlation is computed for the keys.</p> <p> TYPE: <code>List[OrderedDict]</code> </p> <code>num_values</code> <p>Use only these many values from the data (from the start of the OrderedDicts)</p> <p> TYPE: <code>int</code> </p> <code>pvalue</code> <p>correlation coefficients for which the p-value is below the threshold <code>pvalue/len(vv)</code> will be discarded.</p> <p> TYPE: <code>float</code> </p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def spearman_correlation(vv: List[OrderedDict], num_values: int, pvalue: float):\n\"\"\"Simple matrix plots with spearman correlation for each pair in vv.\n    Args:\n        vv: list of OrderedDicts with index: value. Spearman correlation\n            is computed for the keys.\n        num_values: Use only these many values from the data (from the start\n            of the OrderedDicts)\n        pvalue: correlation coefficients for which the p-value is below the\n            threshold `pvalue/len(vv)` will be discarded.\n    \"\"\"\nr: np.ndarray = np.ndarray((len(vv), len(vv)))\np: np.ndarray = np.ndarray((len(vv), len(vv)))\nfor i, a in enumerate(vv):\nfor j, b in enumerate(vv):\nfrom scipy.stats._stats_py import SpearmanrResult\nspearman: SpearmanrResult = sp.stats.spearmanr(\nlist(a.keys())[:num_values], list(b.keys())[:num_values]\n)\nr[i][j] = (\nspearman.correlation if spearman.pvalue &lt; pvalue / len(vv) else np.nan\n)  # Bonferroni correction\np[i][j] = spearman.pvalue\nfig, axs = plt.subplots(1, 2, figsize=(16, 7))\nplot1 = axs[0].matshow(r, vmin=-1, vmax=1)\naxs[0].set_title(f\"Spearman correlation (top {num_values} values)\")\naxs[0].set_xlabel(\"Runs\")\naxs[0].set_ylabel(\"Runs\")\nfig.colorbar(plot1, ax=axs[0])\nplot2 = axs[1].matshow(p, vmin=0, vmax=1)\naxs[1].set_title(\"p-value\")\naxs[1].set_xlabel(\"Runs\")\naxs[1].set_ylabel(\"Runs\")\nfig.colorbar(plot2, ax=axs[1])\nreturn fig\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.plot_shapley","title":"<code>plot_shapley(df, *, level=0.05, ax=None, title=None, xlabel=None, ylabel=None)</code>","text":"<p>Plots the shapley values, as returned from compute_shapley_values, with error bars corresponding to an \\(\\alpha\\)-level confidence interval.</p> PARAMETER  DESCRIPTION <code>df</code> <p>dataframe with the shapley values</p> <p> TYPE: <code>DataFrame</code> </p> <code>level</code> <p>confidence level for the error bars</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.05</code> </p> <code>ax</code> <p>axes to plot on or None if a new subplots should be created</p> <p> TYPE: <code>Optional[Axes]</code> DEFAULT: <code>None</code> </p> <code>title</code> <p>string, title of the plot</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>xlabel</code> <p>string, x label of the plot</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ylabel</code> <p>string, y label of the plot</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Axes</code> <p>The axes created or used</p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def plot_shapley(\ndf: pd.DataFrame,\n*,\nlevel: float = 0.05,\nax: Optional[plt.Axes] = None,\ntitle: Optional[str] = None,\nxlabel: Optional[str] = None,\nylabel: Optional[str] = None,\n) -&gt; plt.Axes:\nr\"\"\"Plots the shapley values, as returned from\n    [compute_shapley_values][pydvl.value.shapley.common.compute_shapley_values], with error bars\n    corresponding to an $\\alpha$-level confidence interval.\n    Args:\n        df: dataframe with the shapley values\n        level: confidence level for the error bars\n        ax: axes to plot on or None if a new subplots should be created\n        title: string, title of the plot\n        xlabel: string, x label of the plot\n        ylabel: string, y label of the plot\n    Returns:\n        The axes created or used\n    \"\"\"\nif ax is None:\n_, ax = plt.subplots()\nyerr = norm.ppf(1 - level / 2) * df[\"data_value_stderr\"]\nax.errorbar(x=df.index, y=df[\"data_value\"], yerr=yerr, fmt=\"o\", capsize=6)\nax.set_xlabel(xlabel)\nax.set_ylabel(ylabel)\nax.set_title(title)\nplt.xticks(rotation=60)\nreturn ax\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.plot_influence_distribution_by_label","title":"<code>plot_influence_distribution_by_label(influences, labels, title_extra='')</code>","text":"<p>Plots the histogram of the influence that all samples in the training set have over a single sample index, separated by labels.</p> PARAMETER  DESCRIPTION <code>influences</code> <p>array of influences (training samples x test samples)</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>labels</code> <p>labels for the training set.</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>title_extra</code> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def plot_influence_distribution_by_label(\ninfluences: NDArray[np.float_], labels: NDArray[np.float_], title_extra: str = \"\"\n):\n\"\"\"Plots the histogram of the influence that all samples in the training set\n    have over a single sample index, separated by labels.\n    Args:\n       influences: array of influences (training samples x test samples)\n       labels: labels for the training set.\n       title_extra:\n    \"\"\"\n_, ax = plt.subplots()\nunique_labels = np.unique(labels)\nfor label in unique_labels:\nax.hist(influences[labels == label], label=label, alpha=0.7)\nax.set_xlabel(\"Influence values\")\nax.set_ylabel(\"Number of samples\")\nax.set_title(f\"Distribution of influences \" + title_extra)\nax.legend()\nplt.show()\n</code></pre>"},{"location":"api/pydvl/reporting/scores/","title":"Scores","text":""},{"location":"api/pydvl/reporting/scores/#pydvl.reporting.scores.compute_removal_score","title":"<code>compute_removal_score(u, values, percentages, *, remove_best=False, progress=False)</code>","text":"<p>Fits model and computes score on the test set after incrementally removing a percentage of data points from the training set, based on their values.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>values</code> <p>Data values of data instances in the training set.</p> <p> TYPE: <code>ValuationResult</code> </p> <code>percentages</code> <p>Sequence of removal percentages.</p> <p> TYPE: <code>Union[NDArray[float_], Iterable[float]]</code> </p> <code>remove_best</code> <p>If True, removes data points in order of decreasing valuation.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>progress</code> <p>If True, display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Dict[float, float]</code> <p>Dictionary that maps the percentages to their respective scores.</p> Source code in <code>src/pydvl/reporting/scores.py</code> <pre><code>def compute_removal_score(\nu: Utility,\nvalues: ValuationResult,\npercentages: Union[NDArray[np.float_], Iterable[float]],\n*,\nremove_best: bool = False,\nprogress: bool = False,\n) -&gt; Dict[float, float]:\nr\"\"\"Fits model and computes score on the test set after incrementally removing\n    a percentage of data points from the training set, based on their values.\n    Args:\n        u: Utility object with model, data, and scoring function.\n        values: Data values of data instances in the training set.\n        percentages: Sequence of removal percentages.\n        remove_best: If True, removes data points in order of decreasing valuation.\n        progress: If True, display a progress bar.\n    Returns:\n        Dictionary that maps the percentages to their respective scores.\n    \"\"\"\n# Sanity checks\nif np.any([x &gt;= 1.0 or x &lt; 0.0 for x in percentages]):\nraise ValueError(\"All percentages should be in the range [0.0, 1.0)\")\nif len(values) != len(u.data.indices):\nraise ValueError(\nf\"The number of values, {len(values) }, should be equal to the number of data indices, {len(u.data.indices)}\"\n)\nscores = {}\n# We sort in descending order if we want to remove the best values\nvalues.sort(reverse=remove_best)\nfor pct in maybe_progress(percentages, display=progress, desc=\"Removal Scores\"):\nn_removal = int(pct * len(u.data))\nindices = values.indices[n_removal:]\nscore = u(indices)\nscores[pct] = score\nreturn scores\n</code></pre>"},{"location":"api/pydvl/utils/","title":"Utils","text":""},{"location":"api/pydvl/utils/caching/","title":"Caching","text":"<p>Distributed caching of functions.</p> <p>pyDVL uses memcached to cache utility values, through pymemcache. This allows sharing evaluations across processes and nodes in a cluster. You can run memcached as a service, locally or remotely, see Setting up the cache</p> <p>Warning</p> <p>Function evaluations are cached with a key based on the function's signature and code. This can lead to undesired cache hits, see Cache reuse.</p> <p>Remember not to reuse utility objects for different datasets.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--configuration","title":"Configuration","text":"<p>Memoization is disabled by default but can be enabled easily, see Setting up the cache. When enabled, it will be added to any callable used to construct a Utility (done with the decorator @memcached). Depending on the nature of the utility you might want to enable the computation of a running average of function values, see Usage with stochastic functions. You can see all configuration options under MemcachedConfig.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--default-configuration","title":"Default configuration","text":"<pre><code>default_config = dict(\nserver=('localhost', 11211),\nconnect_timeout=1.0,\ntimeout=0.1,\n# IMPORTANT! Disable small packet consolidation:\nno_delay=True,\nserde=serde.PickleSerde(pickle_version=PICKLE_VERSION)\n)\n</code></pre>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--usage-with-stochastic-functions","title":"Usage with stochastic functions","text":"<p>In addition to standard memoization, the decorator memcached() can compute running average and standard error of repeated evaluations for the same input. This can be useful for stochastic functions with high variance (e.g. model training for small sample sizes), but drastically reduces the speed benefits of memoization.</p> <p>This behaviour can be activated with the argument <code>allow_repeated_evaluations</code> to memcached().</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--cache-reuse","title":"Cache reuse","text":"<p>When working directly with memcached(),  it is essential to only cache pure functions. If they have any kind of state, either internal or external (e.g. a closure over some data that may change), then the cache will fail to notice this and the same value will be returned.</p> <p>When a function is wrapped with memcached() for memoization, its signature (input and output names) and code are used as a key for the cache. Alternatively you can pass a custom value to be used as key with</p> <pre><code>cached_fun = memcached(**asdict(cache_options))(fun, signature=custom_signature)\n</code></pre> <p>If you are running experiments with the same Utility but different datasets, this will lead to evaluations of the utility on new data returning old values because utilities only use sample indices as arguments (so there is no way to tell the difference between '1' for dataset A and '1' for dataset 2 from the point of view of the cache). One solution is to empty the cache between runs, but the preferred one is to use a different Utility object for each dataset.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--unexpected-cache-misses","title":"Unexpected cache misses","text":"<p>Because all arguments to a function are used as part of the key for the cache, sometimes one must exclude some of them. For example, If a function is going to run across multiple processes and some reporting arguments are added (like a <code>job_id</code> for logging purposes), these will be part of the signature and make the functions distinct to the eyes of the cache. This can be avoided with the use of ignore_args in the configuration.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching.CacheStats","title":"<code>CacheStats</code>  <code>dataclass</code>","text":"<p>Statistics gathered by cached functions.</p> ATTRIBUTE DESCRIPTION <code>sets</code> <p>number of times a value was set in the cache</p> <p> TYPE: <code>int</code> </p> <code>misses</code> <p>number of times a value was not found in the cache</p> <p> TYPE: <code>int</code> </p> <code>hits</code> <p>number of times a value was found in the cache</p> <p> TYPE: <code>int</code> </p> <code>timeouts</code> <p>number of times a timeout occurred</p> <p> TYPE: <code>int</code> </p> <code>errors</code> <p>number of times an error occurred</p> <p> TYPE: <code>int</code> </p> <code>reconnects</code> <p>number of times the client reconnected to the server</p> <p> TYPE: <code>int</code> </p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching.serialize","title":"<code>serialize(x)</code>","text":"<p>Serialize an object to bytes. Args:     x: object to serialize.</p> RETURNS DESCRIPTION <code>bytes</code> <p>serialized object.</p> Source code in <code>src/pydvl/utils/caching.py</code> <pre><code>def serialize(x: Any) -&gt; bytes:\n\"\"\"Serialize an object to bytes.\n    Args:\n        x: object to serialize.\n    Returns:\n        serialized object.\n    \"\"\"\npickled_output = BytesIO()\npickler = Pickler(pickled_output, PICKLE_VERSION)\npickler.dump(x)\nreturn pickled_output.getvalue()\n</code></pre>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching.memcached","title":"<code>memcached(client_config=None, time_threshold=0.3, allow_repeated_evaluations=False, rtol_stderr=0.1, min_repetitions=3, ignore_args=None)</code>","text":"<p>Transparent, distributed memoization of function calls.</p> <p>Given a function and its signature, memcached uses a distributed cache that, for each set of inputs, keeps track of the average returned value, with variance and number of times it was calculated.</p> <p>If the function is deterministic, i.e. same input corresponds to the same exact output, set <code>allow_repeated_evaluations</code> to <code>False</code>. If instead the function is stochastic (like the training of a model depending on random initializations), memcached() allows to set a minimum number of evaluations to compute a running average, and a tolerance after which the function will not be called anymore. In other words, the function will be recomputed until the value has stabilized with a standard error smaller than <code>rtol_stderr * running average</code>.</p> <p>Warning</p> <p>Do not cache functions with state! See Cache reuse</p> Example <pre><code>cached_fun = memcached(**asdict(cache_options))(heavy_computation)\n</code></pre> PARAMETER  DESCRIPTION <code>client_config</code> <p>configuration for pymemcache's Client. Will be merged on top of the default configuration (see below).</p> <p> TYPE: <code>Optional[MemcachedClientConfig]</code> DEFAULT: <code>None</code> </p> <code>time_threshold</code> <p>computations taking less time than this many seconds are not cached.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.3</code> </p> <code>allow_repeated_evaluations</code> <p>If <code>True</code>, repeated calls to a function with the same arguments will be allowed and outputs averaged until the running standard deviation of the mean stabilises below <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>rtol_stderr</code> <p>relative tolerance for repeated evaluations. More precisely, memcached() will stop evaluating the function once the standard deviation of the mean is smaller than <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.1</code> </p> <code>min_repetitions</code> <p>minimum number of times that a function evaluation on the same arguments is repeated before returning cached values. Useful for stochastic functions only. If the model training is very noisy, set this number to higher values to reduce variance.</p> <p> TYPE: <code>int</code> DEFAULT: <code>3</code> </p> <code>ignore_args</code> <p>Do not take these keyword arguments into account when hashing the wrapped function for usage as key in memcached. This allows sharing the cache among different jobs for the same experiment run if the callable happens to have \"nuisance\" parameters like <code>job_id</code> which do not affect the result of the computation.</p> <p> TYPE: <code>Optional[Iterable[str]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Callable[[Callable[..., T], bytes | None], Callable[..., T]]</code> <p>A wrapped function</p> Source code in <code>src/pydvl/utils/caching.py</code> <pre><code>def memcached(\nclient_config: Optional[MemcachedClientConfig] = None,\ntime_threshold: float = 0.3,\nallow_repeated_evaluations: bool = False,\nrtol_stderr: float = 0.1,\nmin_repetitions: int = 3,\nignore_args: Optional[Iterable[str]] = None,\n) -&gt; Callable[[Callable[..., T], bytes | None], Callable[..., T]]:\n\"\"\"\n    Transparent, distributed memoization of function calls.\n    Given a function and its signature, memcached uses a distributed cache\n    that, for each set of inputs, keeps track of the average returned value,\n    with variance and number of times it was calculated.\n    If the function is deterministic, i.e. same input corresponds to the same\n    exact output, set `allow_repeated_evaluations` to `False`. If instead the\n    function is stochastic (like the training of a model depending on random\n    initializations), memcached() allows to set a minimum number of evaluations\n    to compute a running average, and a tolerance after which the function will\n    not be called anymore. In other words, the function will be recomputed\n    until the value has stabilized with a standard error smaller than\n    `rtol_stderr * running average`.\n    !!! Warning\n        Do not cache functions with state! See [Cache reuse](cache-reuse)\n    ??? Example\n        ```python\n        cached_fun = memcached(**asdict(cache_options))(heavy_computation)\n        ```\n    Args:\n        client_config: configuration for pymemcache's\n            [Client][pymemcache.client.base.Client].\n            Will be merged on top of the default configuration (see below).\n        time_threshold: computations taking less time than this many seconds are\n            not cached.\n        allow_repeated_evaluations: If `True`, repeated calls to a function\n            with the same arguments will be allowed and outputs averaged until the\n            running standard deviation of the mean stabilises below\n            `rtol_stderr * mean`.\n        rtol_stderr: relative tolerance for repeated evaluations. More precisely,\n            [memcached()][pydvl.utils.caching.memcached] will stop evaluating the function once the\n            standard deviation of the mean is smaller than `rtol_stderr * mean`.\n        min_repetitions: minimum number of times that a function evaluation\n            on the same arguments is repeated before returning cached values. Useful\n            for stochastic functions only. If the model training is very noisy, set\n            this number to higher values to reduce variance.\n        ignore_args: Do not take these keyword arguments into account when\n            hashing the wrapped function for usage as key in memcached. This allows\n            sharing the cache among different jobs for the same experiment run if\n            the callable happens to have \"nuisance\" parameters like `job_id` which\n            do not affect the result of the computation.\n    Returns:\n        A wrapped function\n    \"\"\"\nif ignore_args is None:\nignore_args = []\n# Do I really need this?\ndef connect(config: MemcachedClientConfig):\n\"\"\"First tries to establish a connection, then tries setting and\n        getting a value.\"\"\"\ntry:\nclient = RetryingClient(\nClient(**asdict(config)),\nattempts=3,\nretry_delay=0.1,\nretry_for=[MemcacheUnexpectedCloseError],\n)\ntemp_key = str(uuid.uuid4())\nclient.set(temp_key, 7)\nassert client.get(temp_key) == 7\nclient.delete(temp_key, 0)\nreturn client\nexcept ConnectionRefusedError as e:\nlogger.error(  # type: ignore\nf\"@memcached: Timeout connecting \"\nf\"to {config.server} after \"\nf\"{config.connect_timeout} seconds: {str(e)}. Did you start memcached?\"\n)\nraise e\nexcept AssertionError as e:\nlogger.error(  # type: ignore\nf\"@memcached: Failure saving dummy value \"\nf\"to {config.server}: {str(e)}\"\n)\ndef wrapper(fun: Callable[..., T], signature: Optional[bytes] = None):\nif signature is None:\nsignature = serialize((fun.__code__.co_code, fun.__code__.co_consts))\n@wraps(fun, updated=[])  # don't try to use update() for a class\nclass Wrapped:\nconfig: MemcachedClientConfig\nstats: CacheStats\nclient: RetryingClient\ndef __init__(self, config: MemcachedClientConfig):\nself.config = config\nself.stats = CacheStats()\nself.client = connect(self.config)\nself._signature = signature\ndef __call__(self, *args, **kwargs) -&gt; T:\nkey_kwargs = {k: v for k, v in kwargs.items() if k not in ignore_args}  # type: ignore\narg_signature: bytes = serialize((args, list(key_kwargs.items())))\nkey = blake2b(self._signature + arg_signature).hexdigest().encode(\"ASCII\")  # type: ignore\nresult_dict: Dict[str, float] = self.get_key_value(key)\nif result_dict is None:\nresult_dict = {}\nstart = time()\nvalue = fun(*args, **kwargs)\nend = time()\nresult_dict[\"value\"] = value\nif end - start &gt;= time_threshold or allow_repeated_evaluations:\nresult_dict[\"count\"] = 1\nresult_dict[\"variance\"] = 0\nself.client.set(key, result_dict, noreply=True)\nself.stats.sets += 1\nself.stats.misses += 1\nelif allow_repeated_evaluations:\nself.stats.hits += 1\nvalue = result_dict[\"value\"]\ncount = result_dict[\"count\"]\nvariance = result_dict[\"variance\"]\nerror_on_average = (variance / count) ** (1 / 2)\nif (\nerror_on_average &gt; rtol_stderr * value\nor count &lt;= min_repetitions\n):\nnew_value = fun(*args, **kwargs)\nnew_avg, new_var = running_moments(\nvalue, variance, int(count), cast(float, new_value)\n)\nresult_dict[\"value\"] = new_avg\nresult_dict[\"count\"] = count + 1\nresult_dict[\"variance\"] = new_var\nself.client.set(key, result_dict, noreply=True)\nself.stats.sets += 1\nelse:\nself.stats.hits += 1\nreturn result_dict[\"value\"]  # type: ignore\ndef __getstate__(self):\n\"\"\"Enables pickling after a socket has been opened to the\n                memcached server, by removing the client from the stored\n                data.\"\"\"\nodict = self.__dict__.copy()\ndel odict[\"client\"]\nreturn odict\ndef __setstate__(self, d: dict):\n\"\"\"Restores a client connection after loading from a pickle.\"\"\"\nself.config = d[\"config\"]\nself.stats = d[\"stats\"]\nself.client = Client(**asdict(self.config))\nself._signature = signature\ndef get_key_value(self, key: bytes):\nresult = None\ntry:\nresult = self.client.get(key)\nexcept socket.timeout as e:\nself.stats.timeouts += 1\nwarnings.warn(f\"{type(self).__name__}: {str(e)}\", RuntimeWarning)\nexcept OSError as e:\nself.stats.errors += 1\nwarnings.warn(f\"{type(self).__name__}: {str(e)}\", RuntimeWarning)\nexcept AttributeError as e:\n# FIXME: this depends on _recv() failing on invalid sockets\n# See pymemcache.base.py,\nself.stats.reconnects += 1\nwarnings.warn(f\"{type(self).__name__}: {str(e)}\", RuntimeWarning)\nself.client = connect(self.config)\nreturn result\nWrapped.__doc__ = (\nf\"A wrapper around {fun.__name__}() with remote caching enabled.\\n\"\n+ (Wrapped.__doc__ or \"\")\n)\nWrapped.__name__ = f\"memcached_{fun.__name__}\"\npath = list(reversed(fun.__qualname__.split(\".\")))\npatched = [f\"memcached_{path[0]}\"] + path[1:]\nWrapped.__qualname__ = \".\".join(reversed(patched))\n# TODO: pick from some config file or something\nreturn Wrapped(client_config or MemcachedClientConfig())\nreturn wrapper\n</code></pre>"},{"location":"api/pydvl/utils/config/","title":"Config","text":""},{"location":"api/pydvl/utils/config/#pydvl.utils.config.ParallelConfig","title":"<code>ParallelConfig</code>  <code>dataclass</code>","text":"<p>Configuration for parallel computation backend.</p> PARAMETER  DESCRIPTION <code>backend</code> <p>Type of backend to use. Defaults to 'joblib'</p> <p> TYPE: <code>Literal['joblib', 'ray']</code> DEFAULT: <code>'joblib'</code> </p> <code>address</code> <p>Address of existing remote or local cluster to use.</p> <p> TYPE: <code>Optional[Union[str, Tuple[str, int]]]</code> DEFAULT: <code>None</code> </p> <code>n_cpus_local</code> <p>Number of CPUs to use when creating a local ray cluster. This has no effect when using an existing ray cluster.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>logging_level</code> <p>Logging level for the parallel backend's worker.</p> <p> TYPE: <code>int</code> DEFAULT: <code>WARNING</code> </p> <code>wait_timeout</code> <p>Timeout in seconds for waiting on futures.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p>"},{"location":"api/pydvl/utils/config/#pydvl.utils.config.MemcachedClientConfig","title":"<code>MemcachedClientConfig</code>  <code>dataclass</code>","text":"<p>Configuration of the memcached client.</p> PARAMETER  DESCRIPTION <code>server</code> <p>A tuple of (IP|domain name, port).</p> <p> TYPE: <code>Tuple[str, int]</code> DEFAULT: <code>('localhost', 11211)</code> </p> <code>connect_timeout</code> <p>How many seconds to wait before raising <code>ConnectionRefusedError</code> on failure to connect.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>timeout</code> <p>seconds to wait for send or recv calls on the socket connected to memcached.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>no_delay</code> <p>set the <code>TCP_NODELAY</code> flag, which may help with performance in some cases.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>serde</code> <p>a serializer / deserializer (\"serde\"). The default <code>PickleSerde</code> should work in most cases. See pymemcached's documentation for details.</p> <p> TYPE: <code>PickleSerde</code> DEFAULT: <code>PickleSerde(pickle_version=PICKLE_VERSION)</code> </p>"},{"location":"api/pydvl/utils/config/#pydvl.utils.config.MemcachedConfig","title":"<code>MemcachedConfig</code>  <code>dataclass</code>","text":"<p>Configuration for memcached(), providing memoization of function calls.</p> <p>Instances of this class are typically used as arguments for the construction of a Utility.</p> PARAMETER  DESCRIPTION <code>client_config</code> <p>Configuration for the connection to the memcached server.</p> <p> TYPE: <code>MemcachedClientConfig</code> DEFAULT: <code>field(default_factory=MemcachedClientConfig)</code> </p> <code>time_threshold</code> <p>computations taking less time than this many seconds are not cached.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.3</code> </p> <code>allow_repeated_evaluations</code> <p>If <code>True</code>, repeated calls to a function with the same arguments will be allowed and outputs averaged until the running standard deviation of the mean stabilises below <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>rtol_stderr</code> <p>relative tolerance for repeated evaluations. More precisely, memcached() will stop evaluating the function once the standard deviation of the mean is smaller than <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.1</code> </p> <code>min_repetitions</code> <p>minimum number of times that a function evaluation on the same arguments is repeated before returning cached values. Useful for stochastic functions only. If the model training is very noisy, set this number to higher values to reduce variance.</p> <p> TYPE: <code>int</code> DEFAULT: <code>3</code> </p> <code>ignore_args</code> <p>Do not take these keyword arguments into account when hashing the wrapped function for usage as key in memcached.</p> <p> TYPE: <code>Optional[Iterable[str]]</code> DEFAULT: <code>None</code> </p>"},{"location":"api/pydvl/utils/dataset/","title":"Dataset","text":"<p>This module contains convenience classes to handle data and groups thereof.</p> <p>Shapley and Least Core value computations require evaluation of a scoring function (the utility). This is typically the performance of the model on a test set (as an approximation to its true expected performance). It is therefore convenient to keep both the training data and the test data together to be passed around to methods in shapley and least_core. This is done with Dataset.</p> <p>This abstraction layer also seamlessly grouping data points together if one is interested in computing their value as a group, see GroupedDataset.</p> <p>Objects of both types are used to construct a Utility object.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset","title":"<code>Dataset(x_train, y_train, x_test, y_test, feature_names=None, target_names=None, data_names=None, description=None, is_multi_output=False)</code>","text":"<p>A convenience class to handle datasets.</p> <p>It holds a dataset, split into training and test data, together with several labels on feature names, data point names and a description.</p> PARAMETER  DESCRIPTION <code>x_train</code> <p>training data</p> <p> TYPE: <code>Union[ndarray, DataFrame]</code> </p> <code>y_train</code> <p>labels for training data</p> <p> TYPE: <code>Union[ndarray, DataFrame]</code> </p> <code>x_test</code> <p>test data</p> <p> TYPE: <code>Union[ndarray, DataFrame]</code> </p> <code>y_test</code> <p>labels for test data</p> <p> TYPE: <code>Union[ndarray, DataFrame]</code> </p> <code>feature_names</code> <p>name of the features of input data</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>target_names</code> <p>names of the features of target data</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>data_names</code> <p>names assigned to data points. For example, if the dataset is a time series, each entry can be a timestamp which can be referenced directly instead of using a row number.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>description</code> <p>A textual description of the dataset.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>is_multi_output</code> <p>set to <code>False</code> if labels are scalars, or to <code>True</code> if they are vectors of dimension &gt; 1.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def __init__(\nself,\nx_train: Union[np.ndarray, pd.DataFrame],\ny_train: Union[np.ndarray, pd.DataFrame],\nx_test: Union[np.ndarray, pd.DataFrame],\ny_test: Union[np.ndarray, pd.DataFrame],\nfeature_names: Optional[Sequence[str]] = None,\ntarget_names: Optional[Sequence[str]] = None,\ndata_names: Optional[Sequence[str]] = None,\ndescription: Optional[str] = None,\n# FIXME: use same parameter name as in check_X_y()\nis_multi_output: bool = False,\n):\n\"\"\"Constructs a Dataset from data and labels.\n    Args:\n        x_train: training data\n        y_train: labels for training data\n        x_test: test data\n        y_test: labels for test data\n        feature_names: name of the features of input data\n        target_names: names of the features of target data\n        data_names: names assigned to data points.\n            For example, if the dataset is a time series, each entry can be a\n            timestamp which can be referenced directly instead of using a row\n            number.\n        description: A textual description of the dataset.\n        is_multi_output: set to `False` if labels are scalars, or to\n            `True` if they are vectors of dimension &gt; 1.\n    \"\"\"\nself.x_train, self.y_train = check_X_y(\nx_train, y_train, multi_output=is_multi_output\n)\nself.x_test, self.y_test = check_X_y(\nx_test, y_test, multi_output=is_multi_output\n)\nif x_train.shape[-1] != x_test.shape[-1]:\nraise ValueError(\nf\"Mismatching number of features: \"\nf\"{x_train.shape[-1]} and {x_test.shape[-1]}\"\n)\nif x_train.shape[0] != y_train.shape[0]:\nraise ValueError(\nf\"Mismatching number of samples: \"\nf\"{x_train.shape[-1]} and {x_test.shape[-1]}\"\n)\nif x_test.shape[0] != y_test.shape[0]:\nraise ValueError(\nf\"Mismatching number of samples: \"\nf\"{x_test.shape[-1]} and {y_test.shape[-1]}\"\n)\ndef make_names(s: str, a: np.ndarray) -&gt; List[str]:\nn = a.shape[1] if len(a.shape) &gt; 1 else 1\nreturn [f\"{s}{i:0{1 + int(np.log10(n))}d}\" for i in range(1, n + 1)]\nself.feature_names = feature_names\nself.target_names = target_names\nif self.feature_names is None:\nif isinstance(x_train, pd.DataFrame):\nself.feature_names = x_train.columns.tolist()\nelse:\nself.feature_names = make_names(\"x\", x_train)\nif self.target_names is None:\nif isinstance(y_train, pd.DataFrame):\nself.target_names = y_train.columns.tolist()\nelse:\nself.target_names = make_names(\"y\", y_train)\nif len(self.x_train.shape) &gt; 1:\nif (\nlen(self.feature_names) != self.x_train.shape[-1]\nor len(self.feature_names) != self.x_test.shape[-1]\n):\nraise ValueError(\"Mismatching number of features and names\")\nif len(self.y_train.shape) &gt; 1:\nif (\nlen(self.target_names) != self.y_train.shape[-1]\nor len(self.target_names) != self.y_test.shape[-1]\n):\nraise ValueError(\"Mismatching number of targets and names\")\nself.description = description or \"No description\"\nself._indices = np.arange(len(self.x_train))\nself._data_names = data_names if data_names is not None else self._indices\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.indices","title":"<code>indices</code>  <code>property</code>","text":"<p>Index of positions in data.x_train.</p> <p>Contiguous integers from 0 to len(Dataset).</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.data_names","title":"<code>data_names</code>  <code>property</code>","text":"<p>Names of each individual datapoint.</p> <p>Used for reporting Shapley values.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.dim","title":"<code>dim</code>  <code>property</code>","text":"<p>Returns the number of dimensions of a sample.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.get_training_data","title":"<code>get_training_data(indices=None)</code>","text":"<p>Given a set of indices, returns the training data that refer to those indices.</p> <p>This is used mainly by Utility to retrieve subsets of the data from indices. It is typically not needed in algorithms.</p> PARAMETER  DESCRIPTION <code>indices</code> <p>Optional indices that will be used to select points from the training data. If <code>None</code>, the entire training data will be returned.</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>If <code>indices</code> is not <code>None</code>, the selected x and y arrays from the training data. Otherwise, the entire dataset.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_training_data(\nself, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n\"\"\"Given a set of indices, returns the training data that refer to those\n    indices.\n    This is used mainly by [Utility][pydvl.utils.utility.Utility] to retrieve\n    subsets of the data from indices. It is typically **not needed in\n    algorithms**.\n    Args:\n        indices: Optional indices that will be used to select points from\n            the training data. If `None`, the entire training data will be\n            returned.\n    Returns:\n        If `indices` is not `None`, the selected x and y arrays from the\n            training data. Otherwise, the entire dataset.\n    \"\"\"\nif indices is None:\nreturn self.x_train, self.y_train\nx = self.x_train[indices]\ny = self.y_train[indices]\nreturn x, y\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.get_test_data","title":"<code>get_test_data(indices=None)</code>","text":"<p>Returns the entire test set regardless of the passed indices.</p> <p>The passed indices will not be used because for data valuation we generally want to score the trained model on the entire test data.</p> <p>Additionally, the way this method is used in the Utility class, the passed indices will be those of the training data and would not work on the test data.</p> <p>There may be cases where it is desired to use parts of the test data. In those cases, it is recommended to inherit from Dataset and override get_test_data().</p> <p>For example, the following snippet shows how one could go about mapping the training data indices into test data indices inside get_test_data():</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; class DatasetWithTestDataIndices(Dataset):\n...    def get_test_data(self, indices=None):\n...        if indices is None:\n...            return self.x_test, self.y_test\n...        fraction = len(list(indices)) / len(self)\n...        mapped_indices = len(self.x_test) / len(self) * np.asarray(indices)\n...        mapped_indices = np.unique(mapped_indices.astype(int))\n...        return self.x_test[mapped_indices], self.y_test[mapped_indices]\n...\n&gt;&gt;&gt; X = np.random.rand(100, 10)\n&gt;&gt;&gt; y = np.random.randint(0, 2, 100)\n&gt;&gt;&gt; dataset = DatasetWithTestDataIndices.from_arrays(X, y)\n&gt;&gt;&gt; indices = np.random.choice(dataset.indices, 30, replace=False)\n&gt;&gt;&gt; _ = dataset.get_training_data(indices)\n&gt;&gt;&gt; _ = dataset.get_test_data(indices)\n</code></pre> PARAMETER  DESCRIPTION <code>indices</code> <p>Optional indices into the test data. This argument is unused left for compatibility with get_training_data().</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>The entire test data.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_test_data(\nself, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n\"\"\"Returns the entire test set regardless of the passed indices.\n    The passed indices will not be used because for data valuation\n    we generally want to score the trained model on the entire test data.\n    Additionally, the way this method is used in the\n    [Utility][pydvl.utils.utility.Utility] class, the passed indices will\n    be those of the training data and would not work on the test data.\n    There may be cases where it is desired to use parts of the test data.\n    In those cases, it is recommended to inherit from\n    [Dataset][pydvl.utils.dataset.Dataset] and override\n    [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data].\n    For example, the following snippet shows how one could go about\n    mapping the training data indices into test data indices\n    inside [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data]:\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; import numpy as np\n        &gt;&gt;&gt; class DatasetWithTestDataIndices(Dataset):\n        ...    def get_test_data(self, indices=None):\n        ...        if indices is None:\n        ...            return self.x_test, self.y_test\n        ...        fraction = len(list(indices)) / len(self)\n        ...        mapped_indices = len(self.x_test) / len(self) * np.asarray(indices)\n        ...        mapped_indices = np.unique(mapped_indices.astype(int))\n        ...        return self.x_test[mapped_indices], self.y_test[mapped_indices]\n        ...\n        &gt;&gt;&gt; X = np.random.rand(100, 10)\n        &gt;&gt;&gt; y = np.random.randint(0, 2, 100)\n        &gt;&gt;&gt; dataset = DatasetWithTestDataIndices.from_arrays(X, y)\n        &gt;&gt;&gt; indices = np.random.choice(dataset.indices, 30, replace=False)\n        &gt;&gt;&gt; _ = dataset.get_training_data(indices)\n        &gt;&gt;&gt; _ = dataset.get_test_data(indices)\n        ```\n    Args:\n        indices: Optional indices into the test data. This argument is\n            unused left for compatibility with\n            [get_training_data()][pydvl.utils.dataset.Dataset.get_training_data].\n    Returns:\n        The entire test data.\n    \"\"\"\nreturn self.x_test, self.y_test\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.from_sklearn","title":"<code>from_sklearn(data, train_size=0.8, random_state=None, stratify_by_target=False, **kwargs)</code>  <code>classmethod</code>","text":"<p>Constructs a Dataset object from a sklearn.utils.Bunch, as returned by the <code>load_*</code> functions in scikit-learn toy datasets.</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; from sklearn.datasets import load_boston\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_boston())\n</code></pre> PARAMETER  DESCRIPTION <code>data</code> <p>scikit-learn Bunch object. The following attributes are supported:</p> <ul> <li><code>data</code>: covariates.</li> <li><code>target</code>: target variables (labels).</li> <li><code>feature_names</code> (optional): the feature names.</li> <li><code>target_names</code> (optional): the target names.</li> <li><code>DESCR</code> (optional): a description.</li> </ul> <p> TYPE: <code>Bunch</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code></p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the target variable as labels. Read more in scikit-learn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor. Use this to pass e.g. <code>is_multi_output</code>.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Object with the sklearn dataset</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_sklearn(\ncls,\ndata: Bunch,\ntrain_size: float = 0.8,\nrandom_state: Optional[int] = None,\nstratify_by_target: bool = False,\n**kwargs,\n) -&gt; \"Dataset\":\n\"\"\"Constructs a [Dataset][pydvl.utils.Dataset] object from a\n    [sklearn.utils.Bunch][sklearn.utils.Bunch], as returned by the `load_*`\n    functions in [scikit-learn toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html).\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; from sklearn.datasets import load_boston\n        &gt;&gt;&gt; dataset = Dataset.from_sklearn(load_boston())\n        ```\n    Args:\n        data: scikit-learn Bunch object. The following attributes are supported:\n            - `data`: covariates.\n            - `target`: target variables (labels).\n            - `feature_names` (**optional**): the feature names.\n            - `target_names` (**optional**): the target names.\n            - `DESCR` (**optional**): a description.\n        train_size: size of the training dataset. Used in `train_test_split`\n        random_state: seed for train / test split\n        stratify_by_target: If `True`, data is split in a stratified\n            fashion, using the target variable as labels. Read more in\n            [scikit-learn's user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `is_multi_output`.\n    Returns:\n        Object with the sklearn dataset\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\nx_train, x_test, y_train, y_test = train_test_split(\ndata.data,\ndata.target,\ntrain_size=train_size,\nrandom_state=random_state,\nstratify=data.target if stratify_by_target else None,\n)\nreturn cls(\nx_train,\ny_train,\nx_test,\ny_test,\nfeature_names=data.get(\"feature_names\"),\ntarget_names=data.get(\"target_names\"),\ndescription=data.get(\"DESCR\"),\n**kwargs,\n)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.from_arrays","title":"<code>from_arrays(X, y, train_size=0.8, random_state=None, stratify_by_target=False, **kwargs)</code>  <code>classmethod</code>","text":"<p>Constructs a Dataset object from X and y numpy arrays  as returned by the <code>make_*</code> functions in sklearn generated datasets.</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; from sklearn.datasets import make_regression\n&gt;&gt;&gt; X, y = make_regression()\n&gt;&gt;&gt; dataset = Dataset.from_arrays(X, y)\n</code></pre> PARAMETER  DESCRIPTION <code>X</code> <p>numpy array of shape (n_samples, n_features)</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>numpy array of shape (n_samples,)</p> <p> TYPE: <code>NDArray</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code></p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the y variable as labels. Read more in sklearn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor. Use this to pass e.g. <code>feature_names</code> or <code>target_names</code>.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Object with the passed X and y arrays split across training and test sets.</p> <p>New in version 0.4.0</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_arrays(\ncls,\nX: NDArray,\ny: NDArray,\ntrain_size: float = 0.8,\nrandom_state: Optional[int] = None,\nstratify_by_target: bool = False,\n**kwargs,\n) -&gt; \"Dataset\":\n\"\"\"Constructs a [Dataset][pydvl.utils.Dataset] object from X and y numpy arrays  as\n    returned by the `make_*` functions in [sklearn generated datasets](https://scikit-learn.org/stable/datasets/sample_generators.html).\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; from sklearn.datasets import make_regression\n        &gt;&gt;&gt; X, y = make_regression()\n        &gt;&gt;&gt; dataset = Dataset.from_arrays(X, y)\n        ```\n    Args:\n        X: numpy array of shape (n_samples, n_features)\n        y: numpy array of shape (n_samples,)\n        train_size: size of the training dataset. Used in `train_test_split`\n        random_state: seed for train / test split\n        stratify_by_target: If `True`, data is split in a stratified fashion,\n            using the y variable as labels. Read more in [sklearn's user\n            guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `feature_names`\n            or `target_names`.\n    Returns:\n        Object with the passed X and y arrays split across training and test sets.\n    !!! tip \"New in version 0.4.0\"\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\nx_train, x_test, y_train, y_test = train_test_split(\nX,\ny,\ntrain_size=train_size,\nrandom_state=random_state,\nstratify=y if stratify_by_target else None,\n)\nreturn cls(x_train, y_train, x_test, y_test, **kwargs)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset","title":"<code>GroupedDataset(x_train, y_train, x_test, y_test, data_groups, feature_names=None, target_names=None, group_names=None, description=None, **kwargs)</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Used for calculating Shapley values of subsets of the data considered as logical units. For instance, one can group by value of a categorical feature, by bin into which a continuous feature falls, or by label.</p> PARAMETER  DESCRIPTION <code>x_train</code> <p>training data</p> <p> TYPE: <code>ndarray</code> </p> <code>y_train</code> <p>labels of training data</p> <p> TYPE: <code>ndarray</code> </p> <code>x_test</code> <p>test data</p> <p> TYPE: <code>ndarray</code> </p> <code>y_test</code> <p>labels of test data</p> <p> TYPE: <code>ndarray</code> </p> <code>data_groups</code> <p>Iterable of the same length as <code>x_train</code> containing a group label for each training data point. The label can be of any type, e.g. <code>str</code> or <code>int</code>. Data points with the same label will then be grouped by this object and considered as one for effects of valuation.</p> <p> TYPE: <code>Sequence</code> </p> <code>feature_names</code> <p>names of the covariates' features.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>target_names</code> <p>names of the labels or targets y</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>group_names</code> <p>names of the groups. If not provided, the labels from <code>data_groups</code> will be used.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>description</code> <p>A textual description of the dataset</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor.</p> <p> DEFAULT: <code>{}</code> </p> <p>Changed in version 0.6.0</p> <p>Added <code>group_names</code> and forwarding of <code>kwargs</code></p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def __init__(\nself,\nx_train: np.ndarray,\ny_train: np.ndarray,\nx_test: np.ndarray,\ny_test: np.ndarray,\ndata_groups: Sequence,\nfeature_names: Optional[Sequence[str]] = None,\ntarget_names: Optional[Sequence[str]] = None,\ngroup_names: Optional[Sequence[str]] = None,\ndescription: Optional[str] = None,\n**kwargs,\n):\n\"\"\"Class for grouping datasets.\n    Used for calculating Shapley values of subsets of the data considered\n    as logical units. For instance, one can group by value of a categorical\n    feature, by bin into which a continuous feature falls, or by label.\n    Args:\n        x_train: training data\n        y_train: labels of training data\n        x_test: test data\n        y_test: labels of test data\n        data_groups: Iterable of the same length as `x_train` containing\n            a group label for each training data point. The label can be of any\n            type, e.g. `str` or `int`. Data points with the same label will\n            then be grouped by this object and considered as one for effects of\n            valuation.\n        feature_names: names of the covariates' features.\n        target_names: names of the labels or targets y\n        group_names: names of the groups. If not provided, the labels\n            from `data_groups` will be used.\n        description: A textual description of the dataset\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor.\n    !!! tip \"Changed in version 0.6.0\"\n    Added `group_names` and forwarding of `kwargs`\n    \"\"\"\nsuper().__init__(\nx_train=x_train,\ny_train=y_train,\nx_test=x_test,\ny_test=y_test,\nfeature_names=feature_names,\ntarget_names=target_names,\ndescription=description,\n**kwargs,\n)\nif len(data_groups) != len(x_train):\nraise ValueError(\nf\"data_groups and x_train must have the same length.\"\nf\"Instead got {len(data_groups)=} and {len(x_train)=}\"\n)\nself.groups: OrderedDict[Any, List[int]] = OrderedDict(\n{k: [] for k in set(data_groups)}\n)\nfor idx, group in enumerate(data_groups):\nself.groups[group].append(idx)\nself.group_items = list(self.groups.items())\nself._indices = np.arange(len(self.groups.keys()))\nself._data_names = (\ngroup_names if group_names is not None else list(self.groups.keys())\n)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.indices","title":"<code>indices</code>  <code>property</code>","text":"<p>Indices of the groups.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.data_names","title":"<code>data_names</code>  <code>property</code>","text":"<p>Names of the groups.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.get_training_data","title":"<code>get_training_data(indices=None)</code>","text":"<p>Returns the data and labels of all samples in the given groups.</p> PARAMETER  DESCRIPTION <code>indices</code> <p>group indices whose elements to return. If <code>None</code>, all data from all groups are returned.</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>Tuple of training data x and labels y.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_training_data(\nself, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n\"\"\"Returns the data and labels of all samples in the given groups.\n    Args:\n        indices: group indices whose elements to return. If `None`,\n            all data from all groups are returned.\n    Returns:\n        Tuple of training data x and labels y.\n    \"\"\"\nif indices is None:\nindices = self.indices\ndata_indices = [\nidx for group_id in indices for idx in self.group_items[group_id][1]\n]\nreturn super().get_training_data(data_indices)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.from_sklearn","title":"<code>from_sklearn(data, train_size=0.8, random_state=None, stratify_by_target=False, data_groups=None, **kwargs)</code>  <code>classmethod</code>","text":"<p>Constructs a GroupedDataset object from a sklearn.utils.Bunch as returned by the <code>load_*</code> functions in scikit-learn toy datasets and groups it.</p> Example <pre><code>&gt;&gt;&gt; from sklearn.datasets import load_iris\n&gt;&gt;&gt; from pydvl.utils import GroupedDataset\n&gt;&gt;&gt; iris = load_iris()\n&gt;&gt;&gt; data_groups = iris.data[:, 0] // 0.5\n&gt;&gt;&gt; dataset = GroupedDataset.from_sklearn(iris, data_groups=data_groups)\n</code></pre> PARAMETER  DESCRIPTION <code>data</code> <p>scikit-learn Bunch object. The following attributes are supported:</p> <ul> <li><code>data</code>: covariates.</li> <li><code>target</code>: target variables (labels).</li> <li><code>feature_names</code> (optional): the feature names.</li> <li><code>target_names</code> (optional): the target names.</li> <li><code>DESCR</code> (optional): a description.</li> </ul> <p> TYPE: <code>Bunch</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the target variable as labels. Read more in sklearn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>data_groups</code> <p>an array holding the group index or name for each data point. The length of this array must be equal to the number of data points in the dataset.</p> <p> TYPE: <code>Optional[Sequence]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>GroupedDataset</code> <p>Dataset with the selected sklearn data</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_sklearn(\ncls,\ndata: Bunch,\ntrain_size: float = 0.8,\nrandom_state: Optional[int] = None,\nstratify_by_target: bool = False,\ndata_groups: Optional[Sequence] = None,\n**kwargs,\n) -&gt; \"GroupedDataset\":\n\"\"\"Constructs a [GroupedDataset][pydvl.utils.GroupedDataset] object from a\n    [sklearn.utils.Bunch][sklearn.utils.Bunch] as returned by the `load_*` functions in\n    [scikit-learn toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html) and groups\n    it.\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from sklearn.datasets import load_iris\n        &gt;&gt;&gt; from pydvl.utils import GroupedDataset\n        &gt;&gt;&gt; iris = load_iris()\n        &gt;&gt;&gt; data_groups = iris.data[:, 0] // 0.5\n        &gt;&gt;&gt; dataset = GroupedDataset.from_sklearn(iris, data_groups=data_groups)\n        ```\n    Args:\n        data: scikit-learn Bunch object. The following attributes are supported:\n            - `data`: covariates.\n            - `target`: target variables (labels).\n            - `feature_names` (**optional**): the feature names.\n            - `target_names` (**optional**): the target names.\n            - `DESCR` (**optional**): a description.\n        train_size: size of the training dataset. Used in `train_test_split`.\n        random_state: seed for train / test split.\n        stratify_by_target: If `True`, data is split in a stratified\n            fashion, using the target variable as labels. Read more in\n            [sklearn's user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        data_groups: an array holding the group index or name for each\n            data point. The length of this array must be equal to the number of\n            data points in the dataset.\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor.\n    Returns:\n        Dataset with the selected sklearn data\n    \"\"\"\nif data_groups is None:\nraise ValueError(\n\"data_groups must be provided when constructing a GroupedDataset\"\n)\nx_train, x_test, y_train, y_test, data_groups_train, _ = train_test_split(\ndata.data,\ndata.target,\ndata_groups,\ntrain_size=train_size,\nrandom_state=random_state,\nstratify=data.target if stratify_by_target else None,\n)\ndataset = Dataset(\nx_train=x_train, y_train=y_train, x_test=x_test, y_test=y_test, **kwargs\n)\nreturn cls.from_dataset(dataset, data_groups_train)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.from_arrays","title":"<code>from_arrays(X, y, train_size=0.8, random_state=None, stratify_by_target=False, data_groups=None, **kwargs)</code>  <code>classmethod</code>","text":"<p>Constructs a GroupedDataset object from X and y numpy arrays as returned by the <code>make_*</code> functions in scikit-learn generated datasets.</p> Example <pre><code>&gt;&gt;&gt; from sklearn.datasets import make_classification\n&gt;&gt;&gt; from pydvl.utils import GroupedDataset\n&gt;&gt;&gt; X, y = make_classification(\n...     n_samples=100,\n...     n_features=4,\n...     n_informative=2,\n...     n_redundant=0,\n...     random_state=0,\n...     shuffle=False\n... )\n&gt;&gt;&gt; data_groups = X[:, 0] // 0.5\n&gt;&gt;&gt; dataset = GroupedDataset.from_arrays(X, y, data_groups=data_groups)\n</code></pre> PARAMETER  DESCRIPTION <code>X</code> <p>array of shape (n_samples, n_features)</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>array of shape (n_samples,)</p> <p> TYPE: <code>NDArray</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the y variable as labels. Read more in sklearn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>data_groups</code> <p>an array holding the group index or name for each data point. The length of this array must be equal to the number of data points in the dataset.</p> <p> TYPE: <code>Optional[Sequence]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments that will be passed to the Dataset constructor.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Dataset with the passed X and y arrays split across training and test sets.</p> <p>New in version 0.4.0</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_arrays(\ncls,\nX: NDArray,\ny: NDArray,\ntrain_size: float = 0.8,\nrandom_state: Optional[int] = None,\nstratify_by_target: bool = False,\ndata_groups: Optional[Sequence] = None,\n**kwargs,\n) -&gt; \"Dataset\":\n\"\"\"Constructs a [GroupedDataset][pydvl.utils.GroupedDataset] object from X and y numpy arrays\n    as returned by the `make_*` functions in\n    [scikit-learn generated datasets](https://scikit-learn.org/stable/datasets/sample_generators.html).\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from sklearn.datasets import make_classification\n        &gt;&gt;&gt; from pydvl.utils import GroupedDataset\n        &gt;&gt;&gt; X, y = make_classification(\n        ...     n_samples=100,\n        ...     n_features=4,\n        ...     n_informative=2,\n        ...     n_redundant=0,\n        ...     random_state=0,\n        ...     shuffle=False\n        ... )\n        &gt;&gt;&gt; data_groups = X[:, 0] // 0.5\n        &gt;&gt;&gt; dataset = GroupedDataset.from_arrays(X, y, data_groups=data_groups)\n        ```\n    Args:\n        X: array of shape (n_samples, n_features)\n        y: array of shape (n_samples,)\n        train_size: size of the training dataset. Used in `train_test_split`.\n        random_state: seed for train / test split.\n        stratify_by_target: If `True`, data is split in a stratified\n            fashion, using the y variable as labels. Read more in\n            [sklearn's user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        data_groups: an array holding the group index or name for each data\n            point. The length of this array must be equal to the number of\n            data points in the dataset.\n        kwargs: Additional keyword arguments that will be passed to the\n            [Dataset][pydvl.utils.Dataset] constructor.\n    Returns:\n        Dataset with the passed X and y arrays split across training and\n            test sets.\n    !!! tip \"New in version 0.4.0\"\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\nif data_groups is None:\nraise ValueError(\n\"data_groups must be provided when constructing a GroupedDataset\"\n)\nx_train, x_test, y_train, y_test, data_groups_train, _ = train_test_split(\nX,\ny,\ndata_groups,\ntrain_size=train_size,\nrandom_state=random_state,\nstratify=y if stratify_by_target else None,\n)\ndataset = Dataset(\nx_train=x_train, y_train=y_train, x_test=x_test, y_test=y_test, **kwargs\n)\nreturn cls.from_dataset(dataset, data_groups_train)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.from_dataset","title":"<code>from_dataset(dataset, data_groups)</code>  <code>classmethod</code>","text":"<p>Creates a GroupedDataset object from the data a Dataset object and a mapping of data groups.</p> Example <pre><code>&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pydvl.utils import Dataset, GroupedDataset\n&gt;&gt;&gt; dataset = Dataset.from_arrays(\n...     X=np.asarray([[1, 2], [3, 4], [5, 6], [7, 8]]),\n...     y=np.asarray([0, 1, 0, 1]),\n... )\n&gt;&gt;&gt; dataset = GroupedDataset.from_dataset(dataset, data_groups=[0, 0, 1, 1])\n</code></pre> PARAMETER  DESCRIPTION <code>dataset</code> <p>The original data.</p> <p> TYPE: <code>Dataset</code> </p> <code>data_groups</code> <p>An array holding the group index or name for each data point. The length of this array must be equal to the number of data points in the dataset.</p> <p> TYPE: <code>Sequence[Any]</code> </p> RETURNS DESCRIPTION <code>GroupedDataset</code> <p>A GroupedDataset with the initial Dataset grouped by data_groups.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_dataset(\ncls, dataset: Dataset, data_groups: Sequence[Any]\n) -&gt; \"GroupedDataset\":\n\"\"\"Creates a [GroupedDataset][pydvl.utils.GroupedDataset] object from the data a\n    [Dataset][pydvl.utils.Dataset] object and a mapping of data groups.\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; import numpy as np\n        &gt;&gt;&gt; from pydvl.utils import Dataset, GroupedDataset\n        &gt;&gt;&gt; dataset = Dataset.from_arrays(\n        ...     X=np.asarray([[1, 2], [3, 4], [5, 6], [7, 8]]),\n        ...     y=np.asarray([0, 1, 0, 1]),\n        ... )\n        &gt;&gt;&gt; dataset = GroupedDataset.from_dataset(dataset, data_groups=[0, 0, 1, 1])\n        ```\n    Args:\n        dataset: The original data.\n        data_groups: An array holding the group index or name for each data\n            point. The length of this array must be equal to the number of\n            data points in the dataset.\n    Returns:\n        A [GroupedDataset][pydvl.utils.GroupedDataset] with the initial\n            [Dataset][pydvl.utils.Dataset] grouped by data_groups.\n    \"\"\"\nreturn cls(\nx_train=dataset.x_train,\ny_train=dataset.y_train,\nx_test=dataset.x_test,\ny_test=dataset.y_test,\ndata_groups=data_groups,\nfeature_names=dataset.feature_names,\ntarget_names=dataset.target_names,\ndescription=dataset.description,\n)\n</code></pre>"},{"location":"api/pydvl/utils/functional/","title":"Functional","text":"<p>Supporting utilities for manipulating arguments of functions.</p>"},{"location":"api/pydvl/utils/functional/#pydvl.utils.functional.free_arguments","title":"<code>free_arguments(fun)</code>","text":"<p>Computes the set of free arguments for a function or functools.partial object.</p> <p>All arguments of a function are considered free unless they are set by a partial. For example, if <code>f = partial(g, a=1)</code>, then <code>a</code> is not a free argument of <code>f</code>.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>A callable or a [partial object][].</p> <p> TYPE: <code>Union[Callable, partial]</code> </p> RETURNS DESCRIPTION <code>Set[str]</code> <p>The set of free arguments of <code>fun</code>.</p> <p>New in version 0.7.0</p> Source code in <code>src/pydvl/utils/functional.py</code> <pre><code>def free_arguments(fun: Union[Callable, partial]) -&gt; Set[str]:\n\"\"\"Computes the set of free arguments for a function or\n    [functools.partial][] object.\n    All arguments of a function are considered free unless they are set by a\n    partial. For example, if `f = partial(g, a=1)`, then `a` is not a free\n    argument of `f`.\n    Args:\n        fun: A callable or a [partial object][].\n    Returns:\n        The set of free arguments of `fun`.\n    !!! tip \"New in version 0.7.0\"\n    \"\"\"\nargs_set_by_partial: Set[str] = set()\ndef _rec_unroll_partial_function_args(g: Union[Callable, partial]) -&gt; Callable:\n\"\"\"Stores arguments and recursively call itself if `g` is a\n        [functools.partial][] object. In the end, returns the initially wrapped\n        function.\n        This handles the construct `partial(_accept_additional_argument, *args,\n        **kwargs)` that is used by `maybe_add_argument`.\n        Args:\n            g: A partial or a function to unroll.\n        Returns:\n            Initial wrapped function.\n        \"\"\"\nnonlocal args_set_by_partial\nif isinstance(g, partial) and g.func == _accept_additional_argument:\narg = g.keywords[\"arg\"]\nif arg in args_set_by_partial:\nargs_set_by_partial.remove(arg)\nreturn _rec_unroll_partial_function_args(g.keywords[\"fun\"])\nelif isinstance(g, partial):\nargs_set_by_partial.update(g.keywords.keys())\nargs_set_by_partial.update(g.args)\nreturn _rec_unroll_partial_function_args(g.func)\nelse:\nreturn g\nwrapped_fn = _rec_unroll_partial_function_args(fun)\nsig = inspect.signature(wrapped_fn)\nreturn args_set_by_partial | set(sig.parameters.keys())\n</code></pre>"},{"location":"api/pydvl/utils/functional/#pydvl.utils.functional.maybe_add_argument","title":"<code>maybe_add_argument(fun, new_arg)</code>","text":"<p>Wraps a function to accept the given keyword parameter if it doesn't already.</p> <p>If <code>fun</code> already takes a keyword parameter of name <code>new_arg</code>, then it is returned as is. Otherwise, a wrapper is returned which merely ignores the argument.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>The function to wrap</p> <p> TYPE: <code>Callable</code> </p> <code>new_arg</code> <p>The name of the argument that the new function will accept (and ignore).</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Callable</code> <p>A new function accepting one more keyword argument.</p> <p>Changed in version 0.7.0</p> <p>Ability to work with partials.</p> Source code in <code>src/pydvl/utils/functional.py</code> <pre><code>def maybe_add_argument(fun: Callable, new_arg: str) -&gt; Callable:\n\"\"\"Wraps a function to accept the given keyword parameter if it doesn't\n    already.\n    If `fun` already takes a keyword parameter of name `new_arg`, then it is\n    returned as is. Otherwise, a wrapper is returned which merely ignores the\n    argument.\n    Args:\n        fun: The function to wrap\n        new_arg: The name of the argument that the new function will accept\n            (and ignore).\n    Returns:\n        A new function accepting one more keyword argument.\n    !!! tip \"Changed in version 0.7.0\"\n        Ability to work with partials.\n    \"\"\"\nif new_arg in free_arguments(fun):\nreturn fun\nreturn partial(_accept_additional_argument, fun=fun, arg=new_arg)\n</code></pre>"},{"location":"api/pydvl/utils/numeric/","title":"Numeric","text":"<p>This module contains routines for numerical computations used across the library.</p>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.powerset","title":"<code>powerset(s)</code>","text":"<p>Returns an iterator for the power set of the argument.</p> <p>Subsets are generated in sequence by growing size. See  random_powerset() for random  sampling.</p> Example <pre><code>&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pydvl.utils.numeric import powerset\n&gt;&gt;&gt; list(powerset(np.array((1,2))))\n[(), (1,), (2,), (1, 2)]\n</code></pre> PARAMETER  DESCRIPTION <code>s</code> <p>The set to use</p> <p> TYPE: <code>NDArray[T]</code> </p> RETURNS DESCRIPTION <code>Iterator[Collection[T]]</code> <p>An iterator over all subsets of the set of indices <code>s</code>.</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def powerset(s: NDArray[T]) -&gt; Iterator[Collection[T]]:\n\"\"\"Returns an iterator for the power set of the argument.\n     Subsets are generated in sequence by growing size. See\n     [random_powerset()][pydvl.utils.numeric.random_powerset] for random\n     sampling.\n    ??? Example\n        ``` pycon\n        &gt;&gt;&gt; import numpy as np\n        &gt;&gt;&gt; from pydvl.utils.numeric import powerset\n        &gt;&gt;&gt; list(powerset(np.array((1,2))))\n        [(), (1,), (2,), (1, 2)]\n        ```\n    Args:\n         s: The set to use\n    Returns:\n        An iterator over all subsets of the set of indices `s`.\n    \"\"\"\nreturn chain.from_iterable(combinations(s, r) for r in range(len(s) + 1))\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.num_samples_permutation_hoeffding","title":"<code>num_samples_permutation_hoeffding(eps, delta, u_range)</code>","text":"<p>Lower bound on the number of samples required for MonteCarlo Shapley to obtain an (\u03b5,\u03b4)-approximation.</p> <p>That is: with probability 1-\u03b4, the estimated value for one data point will be \u03b5-close to the true quantity, if at least this many permutations are sampled.</p> PARAMETER  DESCRIPTION <code>eps</code> <p>\u03b5 &gt; 0</p> <p> TYPE: <code>float</code> </p> <code>delta</code> <p>0 &lt; \u03b4 &lt;= 1</p> <p> TYPE: <code>float</code> </p> <code>u_range</code> <p>Range of the Utility function</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>int</code> <p>Number of permutations required to guarantee \u03b5-correct Shapley values with probability 1-\u03b4</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def num_samples_permutation_hoeffding(eps: float, delta: float, u_range: float) -&gt; int:\n\"\"\"Lower bound on the number of samples required for MonteCarlo Shapley to\n    obtain an (\u03b5,\u03b4)-approximation.\n    That is: with probability 1-\u03b4, the estimated value for one data point will\n    be \u03b5-close to the true quantity, if at least this many permutations are\n    sampled.\n    Args:\n        eps: \u03b5 &gt; 0\n        delta: 0 &lt; \u03b4 &lt;= 1\n        u_range: Range of the [Utility][pydvl.utils.utility.Utility] function\n    Returns:\n        Number of _permutations_ required to guarantee \u03b5-correct Shapley\n            values with probability 1-\u03b4\n    \"\"\"\nreturn int(np.ceil(np.log(2 / delta) * 2 * u_range**2 / eps**2))\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_subset","title":"<code>random_subset(s, q=0.5, seed=None)</code>","text":"<p>Returns one subset at random from <code>s</code>.</p> PARAMETER  DESCRIPTION <code>s</code> <p>set to sample from</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>q</code> <p>Sampling probability for elements. The default 0.5 yields a uniform distribution over the power set of s.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.5</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>NDArray[T]</code> <p>The subset</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_subset(\ns: NDArray[T],\nq: float = 0.5,\nseed: Optional[Seed] = None,\n) -&gt; NDArray[T]:\n\"\"\"Returns one subset at random from ``s``.\n    Args:\n        s: set to sample from\n        q: Sampling probability for elements. The default 0.5 yields a\n            uniform distribution over the power set of s.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n    Returns:\n        The subset\n    \"\"\"\nrng = np.random.default_rng(seed)\nselection = rng.uniform(size=len(s)) &gt; q\nreturn s[selection]\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_powerset","title":"<code>random_powerset(s, n_samples=None, q=0.5, seed=None)</code>","text":"<p>Samples subsets from the power set of the argument, without pre-generating all subsets and in no order.</p> <p>See powerset if you wish to deterministically generate all subsets.</p> <p>To generate subsets, <code>len(s)</code> Bernoulli draws with probability <code>q</code> are drawn. The default value of <code>q = 0.5</code> provides a uniform distribution over the power set of <code>s</code>. Other choices can be used e.g. to implement owen_sampling_shapley.</p> PARAMETER  DESCRIPTION <code>s</code> <p>set to sample from</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>n_samples</code> <p>if set, stop the generator after this many steps. Defaults to <code>np.iinfo(np.int32).max</code></p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>q</code> <p>Sampling probability for elements. The default 0.5 yields a uniform distribution over the power set of s.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.5</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Generator[NDArray[T], None, None]</code> <p>Samples from the power set of <code>s</code>.</p> RAISES DESCRIPTION <code>ValueError</code> <p>if the element sampling probability is not in [0,1]</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_powerset(\ns: NDArray[T],\nn_samples: Optional[int] = None,\nq: float = 0.5,\nseed: Optional[Seed] = None,\n) -&gt; Generator[NDArray[T], None, None]:\n\"\"\"Samples subsets from the power set of the argument, without\n    pre-generating all subsets and in no order.\n    See [powerset][pydvl.utils.numeric.powerset] if you wish to deterministically generate all subsets.\n    To generate subsets, `len(s)` Bernoulli draws with probability `q` are\n    drawn. The default value of `q = 0.5` provides a uniform distribution over\n    the power set of `s`. Other choices can be used e.g. to implement\n    [owen_sampling_shapley][pydvl.value.shapley.owen.owen_sampling_shapley].\n    Args:\n        s: set to sample from\n        n_samples: if set, stop the generator after this many steps.\n            Defaults to `np.iinfo(np.int32).max`\n        q: Sampling probability for elements. The default 0.5 yields a\n            uniform distribution over the power set of s.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n    Returns:\n        Samples from the power set of `s`.\n    Raises:\n        ValueError: if the element sampling probability is not in [0,1]\n    \"\"\"\nif q &lt; 0 or q &gt; 1:\nraise ValueError(\"Element sampling probability must be in [0,1]\")\nrng = np.random.default_rng(seed)\ntotal = 1\nif n_samples is None:\nn_samples = np.iinfo(np.int32).max\nwhile total &lt;= n_samples:\nyield random_subset(s, q, seed=rng)\ntotal += 1\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_subset_of_size","title":"<code>random_subset_of_size(s, size, seed=None)</code>","text":"<p>Samples a random subset of given size uniformly from the powerset of <code>s</code>.</p> PARAMETER  DESCRIPTION <code>s</code> <p>Set to sample from</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>size</code> <p>Size of the subset to generate</p> <p> TYPE: <code>int</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>NDArray[T]</code> <p>The subset</p> <p>Raises     ValueError: If size &gt; len(s)</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_subset_of_size(\ns: NDArray[T],\nsize: int,\nseed: Optional[Seed] = None,\n) -&gt; NDArray[T]:\n\"\"\"Samples a random subset of given size uniformly from the powerset\n    of `s`.\n    Args:\n        s: Set to sample from\n        size: Size of the subset to generate\n        seed: Either an instance of a numpy random number generator or a seed for it.\n    Returns:\n        The subset\n    Raises\n        ValueError: If size &gt; len(s)\n    \"\"\"\nif size &gt; len(s):\nraise ValueError(\"Cannot sample subset larger than set\")\nrng = np.random.default_rng(seed)\nreturn rng.choice(s, size=size, replace=False)\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_matrix_with_condition_number","title":"<code>random_matrix_with_condition_number(n, condition_number, seed=None)</code>","text":"<p>Constructs a square matrix with a given condition number.</p> <p>Taken from: https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py</p> <p>Also see: https://math.stackexchange.com/questions/1351616/condition-number-of-ata.</p> PARAMETER  DESCRIPTION <code>n</code> <p>size of the matrix</p> <p> TYPE: <code>int</code> </p> <code>condition_number</code> <p>duh</p> <p> TYPE: <code>float</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>NDArray</code> <p>An (n,n) matrix with the requested condition number.</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_matrix_with_condition_number(\nn: int, condition_number: float, seed: Optional[Seed] = None\n) -&gt; NDArray:\n\"\"\"Constructs a square matrix with a given condition number.\n    Taken from:\n    [https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py](\n    https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py)\n    Also see:\n    [https://math.stackexchange.com/questions/1351616/condition-number-of-ata](\n    https://math.stackexchange.com/questions/1351616/condition-number-of-ata).\n    Args:\n        n: size of the matrix\n        condition_number: duh\n        seed: Either an instance of a numpy random number generator or a seed for it.\n    Returns:\n        An (n,n) matrix with the requested condition number.\n    \"\"\"\nif n &lt; 2:\nraise ValueError(\"Matrix size must be at least 2\")\nif condition_number &lt;= 1:\nraise ValueError(\"Condition number must be greater than 1\")\nrng = np.random.default_rng(seed)\nlog_condition_number = np.log(condition_number)\nexp_vec = np.arange(\n-log_condition_number / 4.0,\nlog_condition_number * (n + 1) / (4 * (n - 1)),\nlog_condition_number / (2.0 * (n - 1)),\n)\nexp_vec = exp_vec[:n]\ns: np.ndarray = np.exp(exp_vec)\nS = np.diag(s)\nU, _ = np.linalg.qr((rng.uniform(size=(n, n)) - 5.0) * 200)\nV, _ = np.linalg.qr((rng.uniform(size=(n, n)) - 5.0) * 200)\nP: np.ndarray = U.dot(S).dot(V.T)\nP = P.dot(P.T)\nreturn P\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.running_moments","title":"<code>running_moments(previous_avg, previous_variance, count, new_value)</code>","text":"<p>Uses Welford's algorithm to calculate the running average and variance of  a set of numbers.</p> <p>See Welford's algorithm in wikipedia</p> <p>Warning</p> <p>This is not really using Welford's correction for numerical stability for the variance. (FIXME)</p> <p>Todo</p> <p>This could be generalised to arbitrary moments. See this paper</p> PARAMETER  DESCRIPTION <code>previous_avg</code> <p>average value at previous step</p> <p> TYPE: <code>float | NDArray[float_]</code> </p> <code>previous_variance</code> <p>variance at previous step</p> <p> TYPE: <code>float | NDArray[float_]</code> </p> <code>count</code> <p>number of points seen so far</p> <p> TYPE: <code>int</code> </p> <code>new_value</code> <p>new value in the series of numbers</p> <p> TYPE: <code>float | NDArray[float_]</code> </p> RETURNS DESCRIPTION <code>Tuple[float | NDArray[float_], float | NDArray[float_]]</code> <p>new_average, new_variance, calculated with the new count</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def running_moments(\nprevious_avg: float | NDArray[np.float_],\nprevious_variance: float | NDArray[np.float_],\ncount: int,\nnew_value: float | NDArray[np.float_],\n) -&gt; Tuple[float | NDArray[np.float_], float | NDArray[np.float_]]:\n\"\"\"Uses Welford's algorithm to calculate the running average and variance of\n     a set of numbers.\n    See [Welford's algorithm in wikipedia](https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm)\n    !!! Warning\n        This is not really using Welford's correction for numerical stability\n        for the variance. (FIXME)\n    !!! Todo\n        This could be generalised to arbitrary moments. See [this paper](https://www.osti.gov/biblio/1028931)\n    Args:\n        previous_avg: average value at previous step\n        previous_variance: variance at previous step\n        count: number of points seen so far\n        new_value: new value in the series of numbers\n    Returns:\n        new_average, new_variance, calculated with the new count\n    \"\"\"\n# broadcasted operations seem not to be supported by mypy, so we ignore the type\nnew_average = (new_value + count * previous_avg) / (count + 1)  # type: ignore\nnew_variance = previous_variance + (\n(new_value - previous_avg) * (new_value - new_average) - previous_variance\n) / (count + 1)\nreturn new_average, new_variance\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.top_k_value_accuracy","title":"<code>top_k_value_accuracy(y_true, y_pred, k=3)</code>","text":"<p>Computes the top-k accuracy for the estimated values by comparing indices of the highest k values.</p> PARAMETER  DESCRIPTION <code>y_true</code> <p>Exact/true value</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>y_pred</code> <p>Predicted/estimated value</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>k</code> <p>Number of the highest values taken into account</p> <p> TYPE: <code>int</code> DEFAULT: <code>3</code> </p> RETURNS DESCRIPTION <code>float</code> <p>Accuracy</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def top_k_value_accuracy(\ny_true: NDArray[np.float_], y_pred: NDArray[np.float_], k: int = 3\n) -&gt; float:\n\"\"\"Computes the top-k accuracy for the estimated values by comparing indices\n    of the highest k values.\n    Args:\n        y_true: Exact/true value\n        y_pred: Predicted/estimated value\n        k: Number of the highest values taken into account\n    Returns:\n        Accuracy\n    \"\"\"\ntop_k_exact_values = np.argsort(y_true)[-k:]\ntop_k_pred_values = np.argsort(y_pred)[-k:]\ntop_k_accuracy = len(np.intersect1d(top_k_exact_values, top_k_pred_values)) / k\nreturn top_k_accuracy\n</code></pre>"},{"location":"api/pydvl/utils/parallel/","title":"Parallel","text":""},{"location":"api/pydvl/utils/parallel/#pydvl.utils.parallel--this-module-is-deprecated","title":"This module is deprecated","text":"<p>Redirects</p> <p>Imports from this module will be redirected to pydvl.parallel only until v0.9.0. Please update your imports.</p>"},{"location":"api/pydvl/utils/progress/","title":"Progress","text":"<p>Warning</p> <p>This module is deprecated and will be removed in a future release. It implements a wrapper for the tqdm progress bar iterator for easy toggling, but this functionality is already provided by the <code>disable</code> argument of <code>tqdm</code>.</p>"},{"location":"api/pydvl/utils/progress/#pydvl.utils.progress.MockProgress","title":"<code>MockProgress(iterator)</code>","text":"<p>             Bases: <code>Iterator</code></p> <p>A Naive mock class to use with maybe_progress and tqdm. Mocked methods don't support return values. Mocked properties don't do anything</p> Source code in <code>src/pydvl/utils/progress.py</code> <pre><code>def __init__(self, iterator: Union[Iterator, Iterable]):\n# Since there is no _it in __dict__ at this point, doing here\n# self._it = iterator\n# results in a call to __getattr__() and the assignment fails, so we\n# use __dict__ instead\nself.__dict__[\"_it\"] = iterator\n</code></pre>"},{"location":"api/pydvl/utils/progress/#pydvl.utils.progress.maybe_progress","title":"<code>maybe_progress(it, display=False, **kwargs)</code>","text":"<p>Returns either a tqdm progress bar or a mock object which wraps the iterator as well, but ignores any accesses to methods or properties.</p> PARAMETER  DESCRIPTION <code>it</code> <p>the iterator to wrap</p> <p> TYPE: <code>Union[int, Iterable, Iterator]</code> </p> <code>display</code> <p>set to True to return a tqdm bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>kwargs</code> <p>Keyword arguments that will be forwarded to tqdm</p> <p> DEFAULT: <code>{}</code> </p> Source code in <code>src/pydvl/utils/progress.py</code> <pre><code>def maybe_progress(\nit: Union[int, Iterable, Iterator], display: bool = False, **kwargs\n) -&gt; Union[tqdm, MockProgress]:\n\"\"\"Returns either a tqdm progress bar or a mock object which wraps the\n    iterator as well, but ignores any accesses to methods or properties.\n    Args:\n        it: the iterator to wrap\n        display: set to True to return a tqdm bar\n        kwargs: Keyword arguments that will be forwarded to tqdm\n    \"\"\"\nif isinstance(it, int):\nit = range(it)  # type: ignore\nreturn tqdm(it, **kwargs) if display else MockProgress(it)\n</code></pre>"},{"location":"api/pydvl/utils/score/","title":"Score","text":"<p>This module provides a Scorer class that wraps scoring functions with additional information.</p> <p>Scorers can be constructed in the same way as in scikit-learn: either from  known strings or from a callable. Greater values must be better. If they are not, a negated version can be used, see scikit-learn's make_scorer().</p> <p>Scorer provides additional information about the scoring function, like its range and default values, which can be used by some data valuation methods (like group_testing_shapley()) to estimate the number of samples required for a certain quality of approximation.</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.squashed_r2","title":"<code>squashed_r2 = compose_score(Scorer('r2'), _sigmoid, (0, 1), 'squashed r2')</code>  <code>module-attribute</code>","text":"<p>A scorer that squashes the R\u00b2 score into the range [0, 1] using a sigmoid.</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.squashed_variance","title":"<code>squashed_variance = compose_score(Scorer('explained_variance'), _sigmoid, (0, 1), 'squashed explained variance')</code>  <code>module-attribute</code>","text":"<p>A scorer that squashes the explained variance score into the range [0, 1] using a sigmoid.</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.ScorerCallable","title":"<code>ScorerCallable</code>","text":"<p>             Bases: <code>Protocol</code></p> <p>Signature for a scorer</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.Scorer","title":"<code>Scorer(scoring, default=np.nan, range=(-np.inf, np.inf), name=None)</code>","text":"<p>A scoring callable that takes a model, data, and labels and returns a scalar.</p> PARAMETER  DESCRIPTION <code>scoring</code> <p>Either a string or callable that can be passed to get_scorer.</p> <p> TYPE: <code>Union[str, ScorerCallable]</code> </p> <code>default</code> <p>score to be used when a model cannot be fit, e.g. when too little data is passed, or errors arise.</p> <p> TYPE: <code>float</code> DEFAULT: <code>nan</code> </p> <code>range</code> <p>numerical range of the score function. Some Monte Carlo methods can use this to estimate the number of samples required for a certain quality of approximation. If not provided, it can be read from the <code>scoring</code> object if it provides it, for instance if it was constructed with compose_score().</p> <p> TYPE: <code>Tuple</code> DEFAULT: <code>(-inf, inf)</code> </p> <code>name</code> <p>The name of the scorer. If not provided, the name of the function passed will be used.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <p>New in version 0.5.0</p> Source code in <code>src/pydvl/utils/score.py</code> <pre><code>def __init__(\nself,\nscoring: Union[str, ScorerCallable],\ndefault: float = np.nan,\nrange: Tuple = (-np.inf, np.inf),\nname: Optional[str] = None,\n):\nif name is None and isinstance(scoring, str):\nname = scoring\nself._scorer = get_scorer(scoring)\nself.default = default\n# TODO: auto-fill from known scorers ?\nself.range = np.array(range)\nself._name = getattr(self._scorer, \"__name__\", name or \"scorer\")\n</code></pre>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.compose_score","title":"<code>compose_score(scorer, transformation, range, name)</code>","text":"<p>Composes a scoring function with an arbitrary scalar transformation.</p> <p>Useful to squash unbounded scores into ranges manageable by data valuation methods.</p> <p>Example:</p> <pre><code>sigmoid = lambda x: 1/(1+np.exp(-x))\ncompose_score(Scorer(\"r2\"), sigmoid, range=(0,1), name=\"squashed r2\")\n</code></pre> PARAMETER  DESCRIPTION <code>scorer</code> <p>The object to be composed.</p> <p> TYPE: <code>Scorer</code> </p> <code>transformation</code> <p>A scalar transformation</p> <p> TYPE: <code>Callable[[float], float]</code> </p> <code>range</code> <p>The range of the transformation. This will be used e.g. by Utility for the range of the composed.</p> <p> TYPE: <code>Tuple[float, float]</code> </p> <code>name</code> <p>A string representation for the composition, for <code>str()</code>.</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Scorer</code> <p>The composite Scorer.</p> Source code in <code>src/pydvl/utils/score.py</code> <pre><code>def compose_score(\nscorer: Scorer,\ntransformation: Callable[[float], float],\nrange: Tuple[float, float],\nname: str,\n) -&gt; Scorer:\n\"\"\"Composes a scoring function with an arbitrary scalar transformation.\n    Useful to squash unbounded scores into ranges manageable by data valuation\n    methods.\n    Example:\n    ```python\n    sigmoid = lambda x: 1/(1+np.exp(-x))\n    compose_score(Scorer(\"r2\"), sigmoid, range=(0,1), name=\"squashed r2\")\n    ```\n    Args:\n        scorer: The object to be composed.\n        transformation: A scalar transformation\n        range: The range of the transformation. This will be used e.g. by\n            [Utility][pydvl.utils.utility.Utility] for the range of the composed.\n        name: A string representation for the composition, for `str()`.\n    Returns:\n        The composite [Scorer][pydvl.utils.score.Scorer].\n    \"\"\"\nclass CompositeScorer(Scorer):\ndef __call__(self, model: SupervisedModel, X: NDArray, y: NDArray) -&gt; float:\nscore = self._scorer(model=model, X=X, y=y)\nreturn transformation(score)\nreturn CompositeScorer(scorer, range=range, name=name)\n</code></pre>"},{"location":"api/pydvl/utils/status/","title":"Status","text":""},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status","title":"<code>Status</code>","text":"<p>             Bases: <code>Enum</code></p> <p>Status of a computation.</p> <p>Statuses can be combined using bitwise or (<code>|</code>) and bitwise and (<code>&amp;</code>) to get the status of a combined computation. For example, if we have two computations, one that has converged and one that has failed, then the combined status is <code>Status.Converged | Status.Failed == Status.Converged</code>, but <code>Status.Converged &amp; Status.Failed == Status.Failed</code>.</p>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--or","title":"OR","text":"<p>The result of bitwise or-ing two valuation statuses with <code>|</code> is given by the following table:</p> P C F P P C P C C C C F P C F <p>where P = Pending, C = Converged, F = Failed.</p>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--and","title":"AND","text":"<p>The result of bitwise and-ing two valuation statuses with <code>&amp;</code> is given by the following table:</p> P C F P P P F C P C F F F F F <p>where P = Pending, C = Converged, F = Failed.</p>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--not","title":"NOT","text":"<p>The result of bitwise negation of a Status with <code>~</code> is <code>Failed</code> if the status is <code>Converged</code>, or <code>Converged</code> otherwise:</p> <pre><code>~P == C, ~C == F, ~F == C\n</code></pre>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--boolean-casting","title":"Boolean casting","text":"<p>A Status evaluates to <code>True</code> iff it's <code>Converged</code> or <code>Failed</code>:</p> <pre><code>bool(Status.Pending) == False\nbool(Status.Converged) == True\nbool(Status.Failed) == True\n</code></pre> <p>Warning</p> <p>These truth values are inconsistent with the usual boolean operations. In particular the XOR of two instances of <code>Status</code> is not the same as the XOR of their boolean values.</p>"},{"location":"api/pydvl/utils/types/","title":"Types","text":"<p>This module contains types, protocols, decorators and generic function transformations. Some of it probably belongs elsewhere.</p>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.SupervisedModel","title":"<code>SupervisedModel</code>","text":"<p>             Bases: <code>Protocol</code></p> <p>This is the minimal Protocol that valuation methods require from models in order to work.</p> <p>All that is needed are the standard sklearn methods <code>fit()</code>, <code>predict()</code> and <code>score()</code>.</p>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.NoPublicConstructor","title":"<code>NoPublicConstructor</code>","text":"<p>             Bases: <code>ABCMeta</code></p> <p>Metaclass that ensures a private constructor</p> <p>If a class uses this metaclass like this:</p> <pre><code>class SomeClass(metaclass=NoPublicConstructor):\n    pass\n</code></pre> <p>If you try to instantiate your class (<code>SomeClass()</code>), a <code>TypeError</code> will be thrown.</p> <p>Taken almost verbatim from: https://stackoverflow.com/a/64682734</p>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.ensure_seed_sequence","title":"<code>ensure_seed_sequence(seed=None)</code>","text":"<p>If the passed seed is a SeedSequence object then it is returned as is. If it is a Generator the internal protected seed sequence from the generator gets extracted. Otherwise, a new SeedSequence object is created from the passed (optional) seed.</p> PARAMETER  DESCRIPTION <code>seed</code> <p>Either an int, a Generator object a SeedSequence object or None.</p> <p> TYPE: <code>Optional[Union[Seed, SeedSequence]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>SeedSequence</code> <p>A SeedSequence object.</p> <p>New in version 0.7.0</p> Source code in <code>src/pydvl/utils/types.py</code> <pre><code>def ensure_seed_sequence(\nseed: Optional[Union[Seed, SeedSequence]] = None\n) -&gt; SeedSequence:\n\"\"\"\n    If the passed seed is a SeedSequence object then it is returned as is. If it is\n    a Generator the internal protected seed sequence from the generator gets extracted.\n    Otherwise, a new SeedSequence object is created from the passed (optional) seed.\n    Args:\n        seed: Either an int, a Generator object a SeedSequence object or None.\n    Returns:\n        A SeedSequence object.\n    !!! tip \"New in version 0.7.0\"\n    \"\"\"\nif isinstance(seed, SeedSequence):\nreturn seed\nelif isinstance(seed, Generator):\nreturn cast(SeedSequence, seed.bit_generator.seed_seq)  # type: ignore\nelse:\nreturn SeedSequence(seed)\n</code></pre>"},{"location":"api/pydvl/utils/utility/","title":"Utility","text":"<p>This module contains classes to manage and learn utility functions for the computation of values. Please see the documentation on Computing Data Values for more information.</p> <p>Utility holds information about model, data and scoring function (the latter being what one usually understands under utility in the general definition of Shapley value). It is automatically cached across machines.</p> <p>DataUtilityLearning adds support for learning the scoring function to avoid repeated re-training of the model to compute the score.</p> <p>This module also contains Utility classes for toy games that are used for testing and for demonstration purposes.</p>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility--references","title":"References","text":"<ol> <li> <p>Wang, T., Yang, Y. and Jia, R., 2021. Improving cooperative game theory-based data valuation via data utility learning. arXiv preprint arXiv:2107.06336.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.Utility","title":"<code>Utility(model, data, scorer=None, *, default_score=0.0, score_range=(-np.inf, np.inf), catch_errors=True, show_warnings=False, enable_cache=False, cache_options=None, clone_before_fit=True)</code>","text":"<p>Convenience wrapper with configurable memoization of the scoring function.</p> <p>An instance of <code>Utility</code> holds the triple of model, dataset and scoring function which determines the value of data points. This is used for the computation of all game-theoretic values like Shapley values and the Least Core.</p> <p>The Utility expect the model to fulfill the SupervisedModel interface i.e. to have <code>fit()</code>, <code>predict()</code>, and <code>score()</code> methods.</p> <p>When calling the utility, the model will be cloned if it is a Sci-Kit Learn model, otherwise a copy is created using <code>deepcopy()</code> from the builtin copy module.</p> <p>Since evaluating the scoring function requires retraining the model and that can be time-consuming, this class wraps it and caches the results of each execution. Caching is available both locally and across nodes, but must always be enabled for your project first, see Setting up the cache.</p> ATTRIBUTE DESCRIPTION <code>model</code> <p>The supervised model.</p> <p> TYPE: <code>SupervisedModel</code> </p> <code>data</code> <p>An object containing the split data.</p> <p> TYPE: <code>Dataset</code> </p> <code>scorer</code> <p>A scoring function. If None, the <code>score()</code> method of the model will be used. See score for ways to create and compose scorers, in particular how to set default values and ranges.</p> <p> TYPE: <code>Scorer</code> </p> PARAMETER  DESCRIPTION <code>model</code> <p>Any supervised model. Typical choices can be found in the [sci-kit learn documentation][https://scikit-learn.org/stable/supervised_learning.html].</p> <p> TYPE: <code>SupervisedModel</code> </p> <code>data</code> <p>Dataset or GroupedDataset instance.</p> <p> TYPE: <code>Dataset</code> </p> <code>scorer</code> <p>A scoring object. If None, the <code>score()</code> method of the model will be used. See score for ways to create and compose scorers, in particular how to set default values and ranges. For convenience, a string can be passed, which will be used to construct a Scorer.</p> <p> TYPE: <code>Optional[Union[str, Scorer]]</code> DEFAULT: <code>None</code> </p> <code>default_score</code> <p>As a convenience when no <code>scorer</code> object is passed (where a default value can be provided), this argument also allows to set the default score for models that have not been fit, e.g. when too little data is passed, or errors arise.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>score_range</code> <p>As with <code>default_score</code>, this is a convenience argument for when no <code>scorer</code> argument is provided, to set the numerical range of the score function. Some Monte Carlo methods can use this to estimate the number of samples required for a certain quality of approximation.</p> <p> TYPE: <code>Tuple[float, float]</code> DEFAULT: <code>(-inf, inf)</code> </p> <code>catch_errors</code> <p>set to <code>True</code> to catch the errors when <code>fit()</code> fails. This could happen in several steps of the pipeline, e.g. when too little training data is passed, which happens often during Shapley value calculations. When this happens, the <code>default_score</code> is returned as a score and computation continues.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>show_warnings</code> <p>Set to <code>False</code> to suppress warnings thrown by <code>fit()</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>enable_cache</code> <p>If <code>True</code>, use memcached for memoization.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>cache_options</code> <p>Optional configuration object for memcached.</p> <p> TYPE: <code>Optional[MemcachedConfig]</code> DEFAULT: <code>None</code> </p> <code>clone_before_fit</code> <p>If <code>True</code>, the model will be cloned before calling <code>fit()</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Utility, DataUtilityLearning, Dataset\n&gt;&gt;&gt; from sklearn.linear_model import LinearRegression, LogisticRegression\n&gt;&gt;&gt; from sklearn.datasets import load_iris\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_iris(), random_state=16)\n&gt;&gt;&gt; u = Utility(LogisticRegression(random_state=16), dataset)\n&gt;&gt;&gt; u(dataset.indices)\n0.9\n</code></pre> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __init__(\nself,\nmodel: SupervisedModel,\ndata: Dataset,\nscorer: Optional[Union[str, Scorer]] = None,\n*,\ndefault_score: float = 0.0,\nscore_range: Tuple[float, float] = (-np.inf, np.inf),\ncatch_errors: bool = True,\nshow_warnings: bool = False,\nenable_cache: bool = False,\ncache_options: Optional[MemcachedConfig] = None,\nclone_before_fit: bool = True,\n):\nself.model = self._clone_model(model)\nself.data = data\nif isinstance(scorer, str):\nscorer = Scorer(scorer, default=default_score, range=score_range)\nself.scorer = check_scoring(self.model, scorer)\nself.default_score = scorer.default if scorer is not None else default_score\n# TODO: auto-fill from known scorers ?\nself.score_range = scorer.range if scorer is not None else np.array(score_range)\nself.catch_errors = catch_errors\nself.show_warnings = show_warnings\nself.enable_cache = enable_cache\nself.cache_options: MemcachedConfig = cache_options or MemcachedConfig()\nself.clone_before_fit = clone_before_fit\nself._signature = serialize((hash(self.model), hash(data), hash(scorer)))\nself._initialize_utility_wrapper()\n</code></pre>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.Utility.signature","title":"<code>signature</code>  <code>property</code>","text":"<p>Signature used for caching model results.</p>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.Utility.cache_stats","title":"<code>cache_stats: Optional[CacheStats]</code>  <code>property</code>","text":"<p>Cache statistics are gathered when cache is enabled. See CacheStats for all fields returned.</p>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.Utility.__call__","title":"<code>__call__(indices)</code>","text":"PARAMETER  DESCRIPTION <code>indices</code> <p>a subset of valid indices for the <code>x_train</code> attribute of Dataset.</p> <p> TYPE: <code>Iterable[int]</code> </p> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __call__(self, indices: Iterable[int]) -&gt; float:\n\"\"\"\n    Args:\n        indices: a subset of valid indices for the\n            `x_train` attribute of [Dataset][pydvl.utils.dataset.Dataset].\n    \"\"\"\nutility: float = self._utility_wrapper(frozenset(indices))\nreturn utility\n</code></pre>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.DataUtilityLearning","title":"<code>DataUtilityLearning(u, training_budget, model)</code>","text":"<p>Implementation of Data Utility Learning (Wang et al., 2022)<sup>1</sup>.</p> <p>This object wraps a Utility and delegates calls to it, up until a given budget (number of iterations). Every tuple of input and output (a so-called utility sample) is stored. Once the budget is exhausted, <code>DataUtilityLearning</code> fits the given model to the utility samples. Subsequent calls will use the learned model to predict the utility instead of delegating.</p> PARAMETER  DESCRIPTION <code>u</code> <p>The Utility to learn.</p> <p> TYPE: <code>Utility</code> </p> <code>training_budget</code> <p>Number of utility samples to collect before fitting the given model.</p> <p> TYPE: <code>int</code> </p> <code>model</code> <p>A supervised regression model</p> <p> TYPE: <code>SupervisedModel</code> </p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Utility, DataUtilityLearning, Dataset\n&gt;&gt;&gt; from sklearn.linear_model import LinearRegression, LogisticRegression\n&gt;&gt;&gt; from sklearn.datasets import load_iris\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_iris())\n&gt;&gt;&gt; u = Utility(LogisticRegression(), dataset)\n&gt;&gt;&gt; wrapped_u = DataUtilityLearning(u, 3, LinearRegression())\n... # First 3 calls will be computed normally\n&gt;&gt;&gt; for i in range(3):\n...     _ = wrapped_u((i,))\n&gt;&gt;&gt; wrapped_u((1, 2, 3)) # Subsequent calls will be computed using the fit model for DUL\n0.0\n</code></pre> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __init__(\nself, u: Utility, training_budget: int, model: SupervisedModel\n) -&gt; None:\nself.utility = u\nself.training_budget = training_budget\nself.model = model\nself._current_iteration = 0\nself._is_model_fit = False\nself._utility_samples: Dict[FrozenSet, Tuple[NDArray[np.bool_], float]] = {}\n</code></pre>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.DataUtilityLearning.data","title":"<code>data: Dataset</code>  <code>property</code>","text":"<p>Returns the wrapped utility's Dataset.</p>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.MinerGameUtility","title":"<code>MinerGameUtility(n_miners, **kwargs)</code>","text":"<p>             Bases: <code>Utility</code></p> <p>Toy game utility that is used for testing and demonstration purposes.</p> <p>Consider a group of n miners, who have discovered large bars of gold.</p> <p>If two miners can carry one piece of gold, then the payoff of a coalition \\(S\\) is:</p> \\[{ v(S) = \\left\\{\\begin{array}{lll} \\mid S \\mid / 2 &amp; \\text{, if} &amp; \\mid S \\mid \\text{ is even} \\\\ ( \\mid S \\mid - 1)/2 &amp; \\text{, if} &amp; \\mid S \\mid \\text{ is odd} \\end{array}\\right. }\\] <p>If there are more than two miners and there is an even number of miners, then the core consists of the single payoff where each miner gets 1/2.</p> <p>If there is an odd number of miners, then the core is empty.</p> <p>Taken from Wikipedia</p> PARAMETER  DESCRIPTION <code>n_miners</code> <p>Number of miners that participate in the game.</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __init__(self, n_miners: int, **kwargs):\nif n_miners &lt;= 2:\nraise ValueError(f\"n_miners, {n_miners} should be &gt; 2\")\nself.n_miners = n_miners\nx = np.arange(n_miners)[..., np.newaxis]\n# The y values don't matter here\ny = np.zeros_like(x)\nself.data = Dataset(x_train=x, y_train=y, x_test=x, y_test=y)\n</code></pre>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.GlovesGameUtility","title":"<code>GlovesGameUtility(left, right, **kwargs)</code>","text":"<p>             Bases: <code>Utility</code></p> <p>Toy game utility that is used for testing and demonstration purposes.</p> <p>In this game, some players have a left glove and others a right glove. Single gloves have a worth of zero while pairs have a worth of 1.</p> <p>The payoff of a coalition \\(S\\) is:</p> \\[{ v(S) = \\min( \\mid S \\cap L \\mid, \\mid S \\cap R \\mid ) }\\] <p>Where \\(L\\), respectively \\(R\\), is the set of players with left gloves, respectively right gloves.</p> PARAMETER  DESCRIPTION <code>left</code> <p>Number of players with a left glove.</p> <p> TYPE: <code>int</code> </p> <code>right</code> <p>Number of player with a right glove.</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __init__(self, left: int, right: int, **kwargs):\nself.left = left\nself.right = right\nx = np.empty(left + right)[..., np.newaxis]\n# The y values don't matter here\ny = np.zeros_like(x)\nself.data = Dataset(x_train=x, y_train=y, x_test=x, y_test=y)\n</code></pre>"},{"location":"api/pydvl/value/","title":"Value","text":"<p>This module implements algorithms for the exact and approximate computation of values and semi-values.</p> <p>See Data valuation for an introduction to the concepts and methods implemented here.</p>"},{"location":"api/pydvl/value/result/","title":"Result","text":"<p>This module collects types and methods for the inspection of the results of valuation algorithms.</p> <p>The most important class is ValuationResult, which provides access to raw values, as well as convenient behaviour as a <code>Sequence</code> with extended indexing and updating abilities, and conversion to pandas DataFrames.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result--operating-on-results","title":"Operating on results","text":"<p>Results can be added together with the standard <code>+</code> operator. Because values are typically running averages of iterative algorithms, addition behaves like a weighted average of the two results, with the weights being the number of updates in each result: adding two results is the same as generating one result with the mean of the values of the two results as values. The variances are updated accordingly. See ValuationResult for details.</p> <p>Results can also be sorted by value, variance or number of updates, see sort(). The arrays of ValuationResult.values, ValuationResult.variances, ValuationResult.counts, ValuationResult.indices, ValuationResult.names are sorted in the same way.</p> <p>Indexing and slicing of results is supported and ValueItem objects are returned. These objects can be compared with the usual operators, which take only the ValueItem.value into account.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result--creating-result-objects","title":"Creating result objects","text":"<p>The most commonly used factory method is ValuationResult.zeros(), which creates a result object with all values, variances and counts set to zero. ValuationResult.empty() creates an empty result object, which can be used as a starting point for adding results together. Empty results are discarded when added to other results. Finally, ValuationResult.from_random() samples random values uniformly.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValueItem","title":"<code>ValueItem</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Generic[IndexT, NameT]</code></p> <p>The result of a value computation for one datum.</p> <p><code>ValueItems</code> can be compared with the usual operators, forming a total order. Comparisons take only the <code>value</code> into account.</p> <p>Todo</p> <p>Maybe have a mode of comparing similar to <code>np.isclose</code>, or taking the <code>variance</code> into account.</p> ATTRIBUTE DESCRIPTION <code>index</code> <p>Index of the sample with this value in the original Dataset</p> <p> TYPE: <code>IndexT</code> </p> <code>name</code> <p>Name of the sample if it was provided. Otherwise, <code>str(index)</code></p> <p> TYPE: <code>NameT</code> </p> <code>value</code> <p>The value</p> <p> TYPE: <code>float</code> </p> <code>variance</code> <p>Variance of the value if it was computed with an approximate method</p> <p> TYPE: <code>Optional[float]</code> </p> <code>count</code> <p>Number of updates for this value</p> <p> TYPE: <code>Optional[int]</code> </p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValueItem.stderr","title":"<code>stderr: Optional[float]</code>  <code>property</code>","text":"<p>Standard error of the value.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult","title":"<code>ValuationResult(*, values, variances=None, counts=None, indices=None, data_names=None, algorithm='', status=Status.Pending, sort=False, **extra_values)</code>","text":"<p>             Bases: <code>Sequence</code>, <code>Iterable[ValueItem[IndexT, NameT]]</code>, <code>Generic[IndexT, NameT]</code></p> <p>Objects of this class hold the results of valuation algorithms.</p> <p>These include indices in the original Dataset, any data names (e.g. group names in GroupedDataset), the values themselves, and variance of the computation in the case of Monte Carlo methods. <code>ValuationResults</code> can be iterated over like any <code>Sequence</code>: <code>iter(valuation_result)</code> returns a generator of ValueItem in the order in which the object is sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult--indexing","title":"Indexing","text":"<p>Indexing can be position-based, when accessing any of the attributes values, variances, counts and indices, as well as when iterating over the object, or using the item access operator, both getter and setter. The \"position\" is either the original sequence in which the data was passed to the constructor, or the sequence in which the object is sorted, see below.</p> <p>Alternatively, indexing can be data-based, i.e. using the indices in the original dataset. This is the case for the methods get() and update().</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult--sorting","title":"Sorting","text":"<p>Results can be sorted in-place with sort(), or alternatively using python's standard <code>sorted()</code> and <code>reversed()</code> Note that sorting values affects how iterators and the object itself as <code>Sequence</code> behave: <code>values[0]</code> returns a ValueItem with the highest or lowest ranking point if this object is sorted by descending or ascending value, respectively. If unsorted, <code>values[0]</code> returns the <code>ValueItem</code> at position 0, which has data index <code>indices[0]</code> in the Dataset.</p> <p>The same applies to direct indexing of the <code>ValuationResult</code>: the index is positional, according to the sorting. It does not refer to the \"data index\". To sort according to data index, use sort() with <code>key=\"index\"</code>.</p> <p>In order to access ValueItem objects by their data index, use get().</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult--operating-on-results","title":"Operating on results","text":"<p>Results can be added to each other with the <code>+</code> operator. Means and variances are correctly updated, using the <code>counts</code> attribute.</p> <p>Results can also be updated with new values using update(). Means and variances are updated accordingly using the Welford algorithm.</p> <p>Empty objects behave in a special way, see empty().</p> PARAMETER  DESCRIPTION <code>values</code> <p>An array of values. If omitted, defaults to an empty array or to an array of zeros if <code>indices</code> are given.</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>indices</code> <p>An optional array of indices in the original dataset. If omitted, defaults to <code>np.arange(len(values))</code>. Warning: It is common to pass the indices of a Dataset here. Attention must be paid in a parallel context to copy them to the local process. Just do <code>indices=np.copy(data.indices)</code>.</p> <p> TYPE: <code>Optional[NDArray[IndexT]]</code> DEFAULT: <code>None</code> </p> <code>variances</code> <p>An optional array of variances in the computation of each value.</p> <p> TYPE: <code>Optional[NDArray[float_]]</code> DEFAULT: <code>None</code> </p> <code>counts</code> <p>An optional array with the number of updates for each value. Defaults to an array of ones.</p> <p> TYPE: <code>Optional[NDArray[int_]]</code> DEFAULT: <code>None</code> </p> <code>data_names</code> <p>Names for the data points. Defaults to index numbers if not set.</p> <p> TYPE: <code>Optional[Sequence[NameT] | NDArray[NameT]]</code> DEFAULT: <code>None</code> </p> <code>algorithm</code> <p>The method used.</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> <code>status</code> <p>The end status of the algorithm.</p> <p> TYPE: <code>Status</code> DEFAULT: <code>Pending</code> </p> <code>sort</code> <p>Whether to sort the indices by ascending value. See above how this affects usage as an iterable or sequence.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>extra_values</code> <p>Additional values that can be passed as keyword arguments. This can contain, for example, the least core value.</p> <p> DEFAULT: <code>{}</code> </p> RAISES DESCRIPTION <code>ValueError</code> <p>If input arrays have mismatching lengths.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __init__(\nself,\n*,\nvalues: NDArray[np.float_],\nvariances: Optional[NDArray[np.float_]] = None,\ncounts: Optional[NDArray[np.int_]] = None,\nindices: Optional[NDArray[IndexT]] = None,\ndata_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\nalgorithm: str = \"\",\nstatus: Status = Status.Pending,\nsort: bool = False,\n**extra_values,\n):\nif variances is not None and len(variances) != len(values):\nraise ValueError(\"Lengths of values and variances do not match\")\nif data_names is not None and len(data_names) != len(values):\nraise ValueError(\"Lengths of values and data_names do not match\")\nif indices is not None and len(indices) != len(values):\nraise ValueError(\"Lengths of values and indices do not match\")\nself._algorithm = algorithm\nself._status = Status(status)  # Just in case we are given a string\nself._values = values\nself._variances = np.zeros_like(values) if variances is None else variances\nself._counts = np.ones_like(values) if counts is None else counts\nself._sort_order = None\nself._extra_values = extra_values or {}\n# Yuk...\nif data_names is None:\nif indices is not None:\nself._names = np.copy(indices)\nelse:\nself._names = np.arange(len(self._values), dtype=np.int_)\nelif not isinstance(data_names, np.ndarray):\nself._names = np.array(data_names)\nelse:\nself._names = data_names.copy()\nif len(np.unique(self._names)) != len(self._names):\nraise ValueError(\"Data names must be unique\")\nif indices is None:\nindices = np.arange(len(self._values), dtype=np.int_)\nself._indices = indices\nself._positions = {idx: pos for pos, idx in enumerate(indices)}\nself._sort_positions: NDArray[np.int_] = np.arange(\nlen(self._values), dtype=np.int_\n)\nif sort:\nself.sort()\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.values","title":"<code>values: NDArray[np.float_]</code>  <code>property</code>","text":"<p>The values, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.variances","title":"<code>variances: NDArray[np.float_]</code>  <code>property</code>","text":"<p>The variances, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.stderr","title":"<code>stderr: NDArray[np.float_]</code>  <code>property</code>","text":"<p>The raw standard errors, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.counts","title":"<code>counts: NDArray[np.int_]</code>  <code>property</code>","text":"<p>The raw counts, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.indices","title":"<code>indices: NDArray[IndexT]</code>  <code>property</code>","text":"<p>The indices for the values, possibly sorted.</p> <p>If the object is unsorted, then these are the same as declared at construction or <code>np.arange(len(values))</code> if none were passed.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.names","title":"<code>names: NDArray[NameT]</code>  <code>property</code>","text":"<p>The names for the values, possibly sorted. If the object is unsorted, then these are the same as declared at construction or <code>np.arange(len(values))</code> if none were passed.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.sort","title":"<code>sort(reverse=False, key='value')</code>","text":"<p>Sorts the indices in place by <code>key</code>.</p> <p>Once sorted, iteration over the results, and indexing of all the properties ValuationResult.values, ValuationResult.variances, ValuationResult.counts, ValuationResult.indices and ValuationResult.names will follow the same order.</p> PARAMETER  DESCRIPTION <code>reverse</code> <p>Whether to sort in descending order by value.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>key</code> <p>The key to sort by. Defaults to ValueItem.value.</p> <p> TYPE: <code>Literal['value', 'variance', 'index', 'name']</code> DEFAULT: <code>'value'</code> </p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def sort(\nself,\nreverse: bool = False,\n# Need a \"Comparable\" type here\nkey: Literal[\"value\", \"variance\", \"index\", \"name\"] = \"value\",\n) -&gt; None:\n\"\"\"Sorts the indices in place by `key`.\n    Once sorted, iteration over the results, and indexing of all the\n    properties\n    [ValuationResult.values][pydvl.value.result.ValuationResult.values],\n    [ValuationResult.variances][pydvl.value.result.ValuationResult.variances],\n    [ValuationResult.counts][pydvl.value.result.ValuationResult.counts],\n    [ValuationResult.indices][pydvl.value.result.ValuationResult.indices]\n    and [ValuationResult.names][pydvl.value.result.ValuationResult.names]\n    will follow the same order.\n    Args:\n        reverse: Whether to sort in descending order by value.\n        key: The key to sort by. Defaults to\n            [ValueItem.value][pydvl.value.result.ValueItem].\n    \"\"\"\nkeymap = {\n\"index\": \"_indices\",\n\"value\": \"_values\",\n\"variance\": \"_variances\",\n\"name\": \"_names\",\n}\nself._sort_positions = np.argsort(getattr(self, keymap[key]))\nif reverse:\nself._sort_positions = self._sort_positions[::-1]\nself._sort_order = reverse\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.__getattr__","title":"<code>__getattr__(attr)</code>","text":"<p>Allows access to extra values as if they were properties of the instance.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __getattr__(self, attr: str) -&gt; Any:\n\"\"\"Allows access to extra values as if they were properties of the instance.\"\"\"\n# This is here to avoid a RecursionError when copying or pickling the object\nif attr == \"_extra_values\":\nraise AttributeError()\ntry:\nreturn self._extra_values[attr]\nexcept KeyError as e:\nraise AttributeError(\nf\"{self.__class__.__name__} object has no attribute {attr}\"\n) from e\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.__iter__","title":"<code>__iter__()</code>","text":"<p>Iterate over the results returning ValueItem objects. To sort in place before iteration, use sort().</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __iter__(self) -&gt; Iterator[ValueItem[IndexT, NameT]]:\n\"\"\"Iterate over the results returning [ValueItem][pydvl.value.result.ValueItem] objects.\n    To sort in place before iteration, use [sort()][pydvl.value.result.ValuationResult.sort].\n    \"\"\"\nfor pos in self._sort_positions:\nyield ValueItem(\nself._indices[pos],\nself._names[pos],\nself._values[pos],\nself._variances[pos],\nself._counts[pos],\n)\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.__add__","title":"<code>__add__(other)</code>","text":"<p>Adds two ValuationResults.</p> <p>The values must have been computed with the same algorithm. An exception to this is if one argument has empty values, in which case the other argument is returned.</p> <p>Warning</p> <p>Abusing this will introduce numerical errors.</p> <p>Means and standard errors are correctly handled. Statuses are added with bit-wise <code>&amp;</code>, see Status. <code>data_names</code> are taken from the left summand, or if unavailable from the right one. The <code>algorithm</code> string is carried over if both terms have the same one or concatenated.</p> <p>It is possible to add ValuationResults of different lengths, and with different or overlapping indices. The result will have the union of indices, and the values.</p> <p>Warning</p> <p>FIXME: Arbitrary <code>extra_values</code> aren't handled.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __add__(self, other: \"ValuationResult\") -&gt; \"ValuationResult\":\n\"\"\"Adds two ValuationResults.\n    The values must have been computed with the same algorithm. An exception\n    to this is if one argument has empty values, in which case the other\n    argument is returned.\n    !!! Warning\n        Abusing this will introduce numerical errors.\n    Means and standard errors are correctly handled. Statuses are added with\n    bit-wise `&amp;`, see [Status][pydvl.value.result.Status].\n    `data_names` are taken from the left summand, or if unavailable from\n    the right one. The `algorithm` string is carried over if both terms\n    have the same one or concatenated.\n    It is possible to add ValuationResults of different lengths, and with\n    different or overlapping indices. The result will have the union of\n    indices, and the values.\n    !!! Warning\n        FIXME: Arbitrary `extra_values` aren't handled.\n    \"\"\"\n# empty results\nif len(self.values) == 0:\nreturn other\nif len(other.values) == 0:\nreturn self\nself._check_compatible(other)\nindices = np.union1d(self._indices, other._indices).astype(self._indices.dtype)\nthis_pos = np.searchsorted(indices, self._indices)\nother_pos = np.searchsorted(indices, other._indices)\nn: NDArray[np.int_] = np.zeros_like(indices, dtype=int)\nm: NDArray[np.int_] = np.zeros_like(indices, dtype=int)\nxn: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\nxm: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\nvn: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\nvm: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\nn[this_pos] = self._counts\nxn[this_pos] = self._values\nvn[this_pos] = self._variances\nm[other_pos] = other._counts\nxm[other_pos] = other._values\nvm[other_pos] = other._variances\n# Sample mean of n+m samples from two means of n and m samples\nxnm = (n * xn + m * xm) / (n + m)\n# Sample variance of n+m samples from two sample variances of n and m samples\nvnm = (n * (vn + xn**2) + m * (vm + xm**2)) / (n + m) - xnm**2\nif np.any(vnm &lt; 0):\nif np.any(vnm &lt; -1e-6):\nlogger.warning(\n\"Numerical error in variance computation. \"\nf\"Negative sample variances clipped to 0 in {vnm}\"\n)\nvnm[np.where(vnm &lt; 0)] = 0\n# Merging of names:\n# If an index has the same name in both results, it must be the same.\n# If an index has a name in one result but not the other, the name is\n# taken from the result with the name.\nif self._names.dtype != other._names.dtype:\nif np.can_cast(other._names.dtype, self._names.dtype, casting=\"safe\"):\nother._names = other._names.astype(self._names.dtype)\nlogger.warning(\nf\"Casting ValuationResult.names from {other._names.dtype} to {self._names.dtype}\"\n)\nelse:\nraise TypeError(\nf\"Cannot cast ValuationResult.names from \"\nf\"{other._names.dtype} to {self._names.dtype}\"\n)\nboth_pos = np.intersect1d(this_pos, other_pos)\nif len(both_pos) &gt; 0:\nthis_names: NDArray = np.empty_like(indices, dtype=object)\nother_names: NDArray = np.empty_like(indices, dtype=object)\nthis_names[this_pos] = self._names\nother_names[other_pos] = other._names\nthis_shared_names = np.take(this_names, both_pos)\nother_shared_names = np.take(other_names, both_pos)\nif np.any(this_shared_names != other_shared_names):\nraise ValueError(f\"Mismatching names in ValuationResults\")\nnames = np.empty_like(indices, dtype=self._names.dtype)\nnames[this_pos] = self._names\nnames[other_pos] = other._names\nreturn ValuationResult(\nalgorithm=self.algorithm or other.algorithm or \"\",\nstatus=self.status &amp; other.status,\nindices=indices,\nvalues=xnm,\nvariances=vnm,\ncounts=n + m,\ndata_names=names,\n# FIXME: What to do with extra_values? This is not commutative:\n# extra_values=self._extra_values.update(other._extra_values),\n)\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.update","title":"<code>update(idx, new_value)</code>","text":"<p>Updates the result in place with a new value, using running mean and variance.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Data index of the value to update.</p> <p> TYPE: <code>int</code> </p> <code>new_value</code> <p>New value to add to the result.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>'ValuationResult'</code> <p>A reference to the same, modified result.</p> RAISES DESCRIPTION <code>IndexError</code> <p>If the index is not found.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def update(self, idx: int, new_value: float) -&gt; \"ValuationResult\":\n\"\"\"Updates the result in place with a new value, using running mean\n    and variance.\n    Args:\n        idx: Data index of the value to update.\n        new_value: New value to add to the result.\n    Returns:\n        A reference to the same, modified result.\n    Raises:\n        IndexError: If the index is not found.\n    \"\"\"\ntry:\npos = self._positions[idx]\nexcept KeyError:\nraise IndexError(f\"Index {idx} not found in ValuationResult\")\nval, var = running_moments(\nself._values[pos], self._variances[pos], self._counts[pos], new_value\n)\nself[pos] = ValueItem(\nindex=cast(IndexT, idx),\nname=self._names[pos],\nvalue=val,\nvariance=var,\ncount=self._counts[pos] + 1,\n)\nreturn self\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.get","title":"<code>get(idx)</code>","text":"<p>Retrieves a ValueItem by data index, as opposed to sort index, like the indexing operator.</p> RAISES DESCRIPTION <code>IndexError</code> <p>If the index is not found.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def get(self, idx: Integral) -&gt; ValueItem:\n\"\"\"Retrieves a ValueItem by data index, as opposed to sort index, like\n    the indexing operator.\n    Raises:\n         IndexError: If the index is not found.\n    \"\"\"\ntry:\npos = self._positions[idx]\nexcept KeyError:\nraise IndexError(f\"Index {idx} not found in ValuationResult\")\nreturn ValueItem(\nself._indices[pos],\nself._names[pos],\nself._values[pos],\nself._variances[pos],\nself._counts[pos],\n)\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.to_dataframe","title":"<code>to_dataframe(column=None, use_names=False)</code>","text":"<p>Returns values as a dataframe.</p> PARAMETER  DESCRIPTION <code>column</code> <p>Name for the column holding the data value. Defaults to the name of the algorithm used.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>use_names</code> <p>Whether to use data names instead of indices for the DataFrame's index.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>DataFrame</code> <p>A dataframe with two columns, one for the values, with name given as explained in <code>column</code>, and another with standard errors for approximate algorithms. The latter will be named <code>column+'_stderr'</code>.</p> <p>Raises:      ImportError: If pandas is not installed</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def to_dataframe(\nself, column: Optional[str] = None, use_names: bool = False\n) -&gt; pandas.DataFrame:\n\"\"\"Returns values as a dataframe.\n    Args:\n        column: Name for the column holding the data value. Defaults to\n            the name of the algorithm used.\n        use_names: Whether to use data names instead of indices for the\n            DataFrame's index.\n    Returns:\n        A dataframe with two columns, one for the values, with name\n            given as explained in `column`, and another with standard errors for\n            approximate algorithms. The latter will be named `column+'_stderr'`.\n    Raises:\n         ImportError: If pandas is not installed\n    \"\"\"\nif not pandas:\nraise ImportError(\"Pandas required for DataFrame export\")\ncolumn = column or self._algorithm\ndf = pandas.DataFrame(\nself._values[self._sort_positions],\nindex=self._names[self._sort_positions]\nif use_names\nelse self._indices[self._sort_positions],\ncolumns=[column],\n)\ndf[column + \"_stderr\"] = self.stderr[self._sort_positions]\nreturn df\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.from_random","title":"<code>from_random(size, total=None, seed=None, **kwargs)</code>  <code>classmethod</code>","text":"<p>Creates a ValuationResult object and fills it with an array of random values from a uniform distribution in [-1,1]. The values can be made to sum up to a given total number (doing so will change their range).</p> PARAMETER  DESCRIPTION <code>size</code> <p>Number of values to generate</p> <p> TYPE: <code>int</code> </p> <code>total</code> <p>If set, the values are normalized to sum to this number (\"efficiency\" property of Shapley values).</p> <p> TYPE: <code>Optional[float]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional options to pass to the constructor of ValuationResult. Use to override status, names, etc.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>'ValuationResult'</code> <p>A valuation result with its status set to</p> <code>'ValuationResult'</code> <p>Status.Converged by default.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If <code>size</code> is less than 1.</p> <p>Changed in version 0.6.0</p> <p>Added parameter <code>total</code>. Check for zero size</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>@classmethod\ndef from_random(\ncls,\nsize: int,\ntotal: Optional[float] = None,\nseed: Optional[Seed] = None,\n**kwargs,\n) -&gt; \"ValuationResult\":\n\"\"\"Creates a [ValuationResult][pydvl.value.result.ValuationResult] object and fills it with an array\n    of random values from a uniform distribution in [-1,1]. The values can\n    be made to sum up to a given total number (doing so will change their range).\n    Args:\n        size: Number of values to generate\n        total: If set, the values are normalized to sum to this number\n            (\"efficiency\" property of Shapley values).\n        kwargs: Additional options to pass to the constructor of\n            [ValuationResult][pydvl.value.result.ValuationResult]. Use to override status, names, etc.\n    Returns:\n        A valuation result with its status set to\n        [Status.Converged][pydvl.utils.status.Status] by default.\n    Raises:\n         ValueError: If `size` is less than 1.\n    !!! tip \"Changed in version 0.6.0\"\n        Added parameter `total`. Check for zero size\n    \"\"\"\nif size &lt; 1:\nraise ValueError(\"Size must be a positive integer\")\nrng = np.random.default_rng(seed)\nvalues = rng.uniform(low=-1, high=1, size=size)\nif total is not None:\nvalues *= total / np.sum(values)\noptions = dict(values=values, status=Status.Converged, algorithm=\"random\")\noptions.update(kwargs)\nreturn cls(**options)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.empty","title":"<code>empty(algorithm='', indices=None, data_names=None, n_samples=0)</code>  <code>classmethod</code>","text":"<p>Creates an empty ValuationResult object.</p> <p>Empty results are characterised by having an empty array of values. When another result is added to an empty one, the empty one is discarded.</p> PARAMETER  DESCRIPTION <code>algorithm</code> <p>Name of the algorithm used to compute the values</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> RETURNS DESCRIPTION <code>'ValuationResult'</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>@classmethod\n@deprecated(\ntarget=True,\ndeprecated_in=\"0.6.0\",\nremove_in=\"0.8.0\",\nargs_mapping=dict(indices=None, data_names=None, n_samples=None),\ntemplate_mgs=\"`%(source_name)s` is deprecated for generating zero-filled \"\n\"results, use `ValuationResult.zeros()` instead.\",\n)\ndef empty(\ncls,\nalgorithm: str = \"\",\nindices: Optional[Sequence[IndexT] | NDArray[IndexT]] = None,\ndata_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\nn_samples: int = 0,\n) -&gt; \"ValuationResult\":\n\"\"\"Creates an empty [ValuationResult][pydvl.value.result.ValuationResult] object.\n    Empty results are characterised by having an empty array of values. When\n    another result is added to an empty one, the empty one is discarded.\n    Args:\n        algorithm: Name of the algorithm used to compute the values\n    Returns:\n        Object with the results.\n    \"\"\"\nif indices is not None or data_names is not None or n_samples != 0:\nreturn cls.zeros(\nalgorithm=algorithm,\nindices=indices,\ndata_names=data_names,\nn_samples=n_samples,\n)\nreturn cls(algorithm=algorithm, status=Status.Pending, values=np.array([]))\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.zeros","title":"<code>zeros(algorithm='', indices=None, data_names=None, n_samples=0)</code>  <code>classmethod</code>","text":"<p>Creates an empty ValuationResult object.</p> <p>Empty results are characterised by having an empty array of values. When another result is added to an empty one, the empty one is ignored.</p> PARAMETER  DESCRIPTION <code>algorithm</code> <p>Name of the algorithm used to compute the values</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> <code>indices</code> <p>Data indices to use. A copy will be made. If not given, the indices will be set to the range <code>[0, n_samples)</code>.</p> <p> TYPE: <code>Optional[Sequence[IndexT] | NDArray[IndexT]]</code> DEFAULT: <code>None</code> </p> <code>data_names</code> <p>Data names to use. A copy will be made. If not given, the names will be set to the string representation of the indices.</p> <p> TYPE: <code>Optional[Sequence[NameT] | NDArray[NameT]]</code> DEFAULT: <code>None</code> </p> <code>n_samples</code> <p>Number of data points whose values are computed. If not given, the length of <code>indices</code> will be used.</p> <p> TYPE: <code>int</code> DEFAULT: <code>0</code> </p> RETURNS DESCRIPTION <code>'ValuationResult'</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>@classmethod\ndef zeros(\ncls,\nalgorithm: str = \"\",\nindices: Optional[Sequence[IndexT] | NDArray[IndexT]] = None,\ndata_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\nn_samples: int = 0,\n) -&gt; \"ValuationResult\":\n\"\"\"Creates an empty [ValuationResult][pydvl.value.result.ValuationResult] object.\n    Empty results are characterised by having an empty array of values. When\n    another result is added to an empty one, the empty one is ignored.\n    Args:\n        algorithm: Name of the algorithm used to compute the values\n        indices: Data indices to use. A copy will be made. If not given,\n            the indices will be set to the range `[0, n_samples)`.\n        data_names: Data names to use. A copy will be made. If not given,\n            the names will be set to the string representation of the indices.\n        n_samples: Number of data points whose values are computed. If\n            not given, the length of `indices` will be used.\n    Returns:\n        Object with the results.\n    \"\"\"\nif indices is None:\nindices = np.arange(n_samples, dtype=np.int_)\nelse:\nindices = np.array(indices)\nreturn cls(\nalgorithm=algorithm,\nstatus=Status.Pending,\nindices=indices,\ndata_names=data_names\nif data_names is not None\nelse np.empty_like(indices, dtype=object),\nvalues=np.zeros(len(indices)),\nvariances=np.zeros(len(indices)),\ncounts=np.zeros(len(indices), dtype=np.int_),\n)\n</code></pre>"},{"location":"api/pydvl/value/sampler/","title":"Sampler","text":"<p>Samplers iterate over subsets of indices.</p> <p>The classes in this module are used to iterate over indices and subsets of their complement in the whole set, as required for the computation of marginal utility for semi-values. The elements returned when iterating over any subclass of PowersetSampler are tuples of the form <code>(idx, subset)</code>, where <code>idx</code> is the index of the element being added to the subset, and <code>subset</code> is the subset of the complement of <code>idx</code>. The classes in this module are used to iterate over an index set \\(I\\) as required for the computation of marginal utility for semi-values. The elements returned when iterating over any subclass of :class:<code>PowersetSampler</code> are tuples of the form \\((i, S)\\), where \\(i\\) is an index of interest, and \\(S \\subset I \\setminus \\{i\\}\\) is a subset of the complement of \\(i\\).</p> <p>The iteration happens in two nested loops. An outer loop iterates over \\(I\\), and an inner loop iterates over the powerset of \\(I \\setminus \\{i\\}\\). The outer iteration can be either sequential or at random.</p> <p>Note</p> <p>This is the natural mode of iteration for the combinatorial definition of semi-values, in particular Shapley value. For the computation using permutations, adhering to this interface is not ideal, but we stick to it for consistency.</p> <p>The samplers are used in the semivalues module to compute any semi-value, in particular Shapley and Beta values, and Banzhaf indices.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler--slicing-of-samplers","title":"Slicing of samplers","text":"<p>The samplers can be sliced for parallel computation. For those which are embarrassingly parallel, this is done by slicing the set of \"outer\" indices and returning new samplers over those slices. This includes all truly powerset-based samplers, such as DeterministicUniformSampler and UniformSampler. In contrast, slicing a PermutationSampler creates a new sampler which iterates over the same indices.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler","title":"<code>PowersetSampler(indices, index_iteration=IndexIteration.Sequential, outer_indices=None)</code>","text":"<p>             Bases: <code>ABC</code>, <code>Iterable[SampleT]</code>, <code>Generic[T]</code></p> <p>Samplers are custom iterables over subsets of indices.</p> <p>Calling <code>iter()</code> on a sampler returns an iterator over tuples of the form \\((i, S)\\), where \\(i\\) is an index of interest, and \\(S \\subset I \\setminus \\{i\\}\\) is a subset of the complement of \\(i\\).</p> <p>This is done in two nested loops, where the outer loop iterates over the set of indices, and the inner loop iterates over subsets of the complement of the current index. The outer iteration can be either sequential or at random.</p> <p>Note</p> <p>Samplers are not iterators themselves, so that each call to <code>iter()</code> e.g. in a for loop creates a new iterator.</p> Example <pre><code>&gt;&gt;&gt;for idx, s in DeterministicUniformSampler(np.arange(2)):\n&gt;&gt;&gt;    print(s, end=\"\")\n[][2,][][1,]\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler--methods-required-in-subclasses","title":"Methods required in subclasses","text":"<p>Samplers must implement a weight() function to be used as a multiplier in Monte Carlo sums, so that the limit expectation coincides with the semi-value.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler--slicing-of-samplers","title":"Slicing of samplers","text":"<p>The samplers can be sliced for parallel computation. For those which are embarrassingly parallel, this is done by slicing the set of \"outer\" indices and returning new samplers over those slices.</p> <pre><code>index_iteration: the order in which indices are iterated over\nouter_indices: The set of items (indices) over which to iterate\nwhen sampling. Subsets are taken from the complement of each index\nin succession. For embarrassingly parallel computations, this set\nis sliced and the samplers are used to iterate over the slices.\n</code></pre> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(\nself,\nindices: NDArray[T],\nindex_iteration: IndexIteration = IndexIteration.Sequential,\nouter_indices: NDArray[T] | None = None,\n):\n\"\"\"\n    Args:\n        indices: The set of items (indices) to sample from.\n        index_iteration: the order in which indices are iterated over\n        outer_indices: The set of items (indices) over which to iterate\n            when sampling. Subsets are taken from the complement of each index\n            in succession. For embarrassingly parallel computations, this set\n            is sliced and the samplers are used to iterate over the slices.\n    \"\"\"\nself._indices = indices\nself._index_iteration = index_iteration\nself._outer_indices = outer_indices if outer_indices is not None else indices\nself._n = len(indices)\nself._n_samples = 0\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler.iterindices","title":"<code>iterindices()</code>","text":"<p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[T]:\n\"\"\"Iterates over indices in the order specified at construction.\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\nif self._index_iteration is PowersetSampler.IndexIteration.Sequential:\nfor idx in self._outer_indices:\nyield idx\nelif self._index_iteration is PowersetSampler.IndexIteration.Random:\nwhile True:\nyield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler.__len__","title":"<code>__len__()</code>","text":"<p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n\"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\nreturn len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler.weight","title":"<code>weight(n, subset_len)</code>  <code>abstractmethod</code> <code>classmethod</code>","text":"<p>Factor by which to multiply Monte Carlo samples, so that the mean converges to the desired expression.</p> <p>By the Law of Large Numbers, the sample mean of \\(\\delta_i(S_j)\\) converges to the expectation under the distribution from which \\(S_j\\) is sampled.</p> \\[ \\frac{1}{m}  \\sum_{j = 1}^m \\delta_i (S_j) c (S_j) \\longrightarrow    \\underset{S \\sim \\mathcal{D}_{- i}}{\\mathbb{E}} [\\delta_i (S) c (    S)]\\] <p>We add a factor \\(c(S_j)\\) in order to have this expectation coincide with the desired expression.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>@classmethod\n@abc.abstractmethod\ndef weight(cls, n: int, subset_len: int) -&gt; float:\nr\"\"\"Factor by which to multiply Monte Carlo samples, so that the\n    mean converges to the desired expression.\n    By the Law of Large Numbers, the sample mean of $\\delta_i(S_j)$\n    converges to the expectation under the distribution from which $S_j$ is\n    sampled.\n    $$ \\frac{1}{m}  \\sum_{j = 1}^m \\delta_i (S_j) c (S_j) \\longrightarrow\n       \\underset{S \\sim \\mathcal{D}_{- i}}{\\mathbb{E}} [\\delta_i (S) c (\n       S)]$$\n    We add a factor $c(S_j)$ in order to have this expectation coincide with\n    the desired expression.\n    \"\"\"\n...\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.StochasticSamplerMixin","title":"<code>StochasticSamplerMixin(*args, seed=None, **kwargs)</code>","text":"<p>Mixin class for samplers which use a random number generator.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\nsuper().__init__(*args, **kwargs)\nself._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicUniformSampler","title":"<code>DeterministicUniformSampler(indices, *args, **kwargs)</code>","text":"<p>             Bases: <code>PowersetSampler[T]</code></p> <p>For every index \\(i\\), each subset of the complement <code>indices - {i}</code> is returned.</p> <p>Note</p> <p>Indices are always iterated over sequentially, irrespective of the value of <code>index_iteration</code> upon construction.</p> Example <pre><code>&gt;&gt;&gt; for idx, s in DeterministicUniformSampler(np.arange(2)):\n&gt;&gt;&gt;    print(f\"{idx} - {s}\", end=\", \")\n1 - [], 1 - [2], 2 - [], 2 - [1],\n</code></pre> PARAMETER  DESCRIPTION <code>indices</code> <p>The set of items (indices) to sample from.</p> <p> TYPE: <code>NDArray[T]</code> </p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, indices: NDArray[T], *args, **kwargs):\n\"\"\"An iterator to perform uniform deterministic sampling of subsets.\n    For every index $i$, each subset of the complement `indices - {i}` is\n    returned.\n    !!! Note\n        Indices are always iterated over sequentially, irrespective of\n        the value of `index_iteration` upon construction.\n    ??? Example\n        ``` pycon\n        &gt;&gt;&gt; for idx, s in DeterministicUniformSampler(np.arange(2)):\n        &gt;&gt;&gt;    print(f\"{idx} - {s}\", end=\", \")\n        1 - [], 1 - [2], 2 - [], 2 - [1],\n        ```\n    Args:\n        indices: The set of items (indices) to sample from.\n    \"\"\"\n# Force sequential iteration\nkwargs.update({\"index_iteration\": PowersetSampler.IndexIteration.Sequential})\nsuper().__init__(indices, *args, **kwargs)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.UniformSampler","title":"<code>UniformSampler</code>","text":"<p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[T]</code></p> <p>An iterator to perform uniform random sampling of subsets.</p> <p>Iterating over every index \\(i\\), either in sequence or at random depending on the value of <code>index_iteration</code>, one subset of the complement <code>indices - {i}</code> is sampled with equal probability \\(2^{n-1}\\). The iterator never ends.</p> Example <p>The code <pre><code>for idx, s in UniformSampler(np.arange(3)):\nprint(f\"{idx} - {s}\", end=\", \")\n</code></pre> Produces the output: <pre><code>0 - [1 4], 1 - [2 3], 2 - [0 1 3], 3 - [], 4 - [2], 0 - [1 3 4], 1 - [0 2]\n(...)\n</code></pre></p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.UniformSampler.weight","title":"<code>weight(n, subset_len)</code>  <code>classmethod</code>","text":"<p>Correction coming from Monte Carlo integration so that the mean of the marginals converges to the value: the uniform distribution over the powerset of a set with n-1 elements has mass 2^{n-1} over each subset.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>@classmethod\ndef weight(cls, n: int, subset_len: int) -&gt; float:\n\"\"\"Correction coming from Monte Carlo integration so that the mean of\n    the marginals converges to the value: the uniform distribution over the\n    powerset of a set with n-1 elements has mass 2^{n-1} over each subset.\"\"\"\nreturn float(2 ** (n - 1)) if n &gt; 0 else 1.0\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticSampler","title":"<code>AntitheticSampler</code>","text":"<p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[T]</code></p> <p>An iterator to perform uniform random sampling of subsets, and their complements.</p> <p>Works as :class:<code>~pydvl.value.sampler.UniformSampler</code>, but for every tuple \\((i,S)\\), it subsequently returns \\((i,S^c)\\), where \\(S^c\\) is the complement of the set \\(S\\), including the index \\(i\\) itself.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PermutationSampler","title":"<code>PermutationSampler</code>","text":"<p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[T]</code></p> <p>Sample permutations of indices and iterate through each returning increasing subsets, as required for the permutation definition of semi-values.</p> <p>This sampler does not implement the two loops described in PowersetSampler. Instead, for a permutation <code>(3,1,4,2)</code>, it returns in sequence the tuples of index and sets:  <code>(3, {})</code>, <code>(1, {3})</code>, <code>(4, {3,1})</code> and <code>(2, {3,1,4})</code>.</p> <p>Note that the full index set is never returned.</p> <p>Warning</p> <p>This sampler requires caching to be enabled or computation will be doubled wrt. a \"direct\" implementation of permutation MC</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PermutationSampler.__getitem__","title":"<code>__getitem__(key)</code>","text":"<p>Permutation samplers cannot be split across indices, so we return a copy of the full sampler.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __getitem__(self, key: slice | list[int]) -&gt; PowersetSampler[T]:\n\"\"\"Permutation samplers cannot be split across indices, so we return\n    a copy of the full sampler.\"\"\"\nreturn super().__getitem__(slice(None))\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicPermutationSampler","title":"<code>DeterministicPermutationSampler</code>","text":"<p>             Bases: <code>PermutationSampler[T]</code></p> <p>Samples all n! permutations of the indices deterministically, and iterates through them, returning sets as required for the permutation-based definition of semi-values.</p> <p>Warning</p> <p>This sampler requires caching to be enabled or computation will be doubled wrt. a \"direct\" implementation of permutation MC</p> <p>Warning</p> <p>This sampler is not parallelizable, as it always iterates over the whole set of permutations in the same order. Different processes would always return the same values for all indices.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.RandomHierarchicalSampler","title":"<code>RandomHierarchicalSampler</code>","text":"<p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[T]</code></p> <p>For every index, sample a set size, then a set of that size.</p> <p>Todo</p> <p>This is unnecessary, but a step towards proper stratified sampling.</p>"},{"location":"api/pydvl/value/semivalues/","title":"Semivalues","text":"<p>This module provides the core functionality for the computation of generic semi-values. A semi-value is any valuation function with the form:</p> \\[v_\\text{semi}(i) = \\sum_{i=1}^n w(k) \\sum_{S \\subset D_{-i}^{(k)}} [U(S_{+i})-U(S)],\\] <p>where the coefficients \\(w(k)\\) satisfy the property:</p> \\[\\sum_{k=1}^n w(k) = 1.\\] <p>Note</p> <p>For implementation consistency, we slightly depart from the common definition of semi-values, which includes a factor \\(1/n\\) in the sum over subsets. Instead, we subsume this factor into the coefficient \\(w(k)\\).</p> <p>As such, the computation of a semi-value requires two components:</p> <ol> <li>A subset sampler that generates subsets of the set \\(D\\) of interest.</li> <li>A coefficient \\(w(k)\\) that assigns a weight to each subset size \\(k\\).</li> </ol> <p>Samplers can be found in sampler, and can be classified into two categories: powerset samplers and permutation samplers. Powerset samplers generate subsets of \\(D_{-i}\\), while the permutation sampler generates permutations of \\(D\\). The former conform to the above definition of semi-values, while the latter reformulates it as:</p> \\[ v(i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} \\tilde{w}( | \\sigma_{:i} | )[U(\\sigma_{:i} \\cup \\{i\\}) \u2212 U(\\sigma_{:i})], \\] <p>where \\(\\sigma_{:i}\\) denotes the set of indices in permutation sigma before the position where \\(i\\) appears (see Data valuation for details), and</p> \\[ \\tilde{w} (k) = n \\binom{n - 1}{k} w (k) \\] <p>is the weight correction due to the reformulation.</p> <p>Warning</p> <p>Both PermutationSampler and DeterministicPermutationSampler require caching to be enabled or computation will be doubled wrt. a 'direct' implementation of permutation MC.</p> <p>There are several pre-defined coefficients, including the Shapley value of (Ghorbani and Zou, 2019)<sup>1</sup>, the Banzhaf index of (Wang and Jia)<sup>3</sup>, and the Beta coefficient of (Kwon and Zou, 2022)<sup>2</sup>. For each of these methods, there is a convenience wrapper function. Respectively, these are: compute_shapley_semivalues, compute_banzhaf_semivalues, and compute_beta_shapley_semivalues. instead.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> <li> <p>Kwon, Y. and Zou, J., 2022. Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning. In: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022, Vol. 151. PMLR, Valencia, Spain.\u00a0\u21a9</p> </li> <li> <p>Wang, J.T. and Jia, R., 2022. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning. ArXiv preprint arXiv:2205.15466.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.SVCoefficient","title":"<code>SVCoefficient</code>","text":"<p>             Bases: <code>Protocol</code></p> <p>A coefficient for the computation of semi-values.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.SVCoefficient.__call__","title":"<code>__call__(n, k)</code>","text":"<p>Computes the coefficient for a given subset size.</p> PARAMETER  DESCRIPTION <code>n</code> <p>Total number of elements in the set.</p> <p> TYPE: <code>int</code> </p> <code>k</code> <p>Size of the subset for which the coefficient is being computed</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def __call__(self, n: int, k: int) -&gt; float:\n\"\"\"Computes the coefficient for a given subset size.\n    Args:\n        n: Total number of elements in the set.\n        k: Size of the subset for which the coefficient is being computed\n    \"\"\"\n...\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.SemiValueMode","title":"<code>SemiValueMode</code>","text":"<p>             Bases: <code>str</code>, <code>Enum</code></p> <p>Enumeration of semi-value modes.</p> <p>Deprecation notice</p> <p>This enum and the associated methods are deprecated and will be removed in 0.8.0.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_generic_semivalues","title":"<code>compute_generic_semivalues(sampler, u, coefficient, done, *, n_jobs=1, config=ParallelConfig(), progress=False)</code>","text":"<p>Computes semi-values for a given utility function and subset sampler.</p> PARAMETER  DESCRIPTION <code>sampler</code> <p>The subset sampler to use for utility computations.</p> <p> TYPE: <code>PowersetSampler</code> </p> <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>coefficient</code> <p>The semi-value coefficient</p> <p> TYPE: <code>SVCoefficient</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def compute_generic_semivalues(\nsampler: PowersetSampler,\nu: Utility,\ncoefficient: SVCoefficient,\ndone: StoppingCriterion,\n*,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\n) -&gt; ValuationResult:\n\"\"\"Computes semi-values for a given utility function and subset sampler.\n    Args:\n        sampler: The subset sampler to use for utility computations.\n        u: Utility object with model, data, and scoring function.\n        coefficient: The semi-value coefficient\n        done: Stopping criterion.\n        n_jobs: Number of parallel jobs to use.\n        config: Object configuring parallel computation, with cluster\n            address, number of cpus, etc.\n        progress: Whether to display a progress bar.\n    Returns:\n        Object with the results.\n    \"\"\"\nfrom concurrent.futures import FIRST_COMPLETED, Future, wait\nfrom pydvl.utils import effective_n_jobs, init_executor, init_parallel_backend\nif isinstance(sampler, PermutationSampler) and not u.enable_cache:\nlog.warning(\n\"PermutationSampler requires caching to be enabled or computation \"\n\"will be doubled wrt. a 'direct' implementation of permutation MC\"\n)\nresult = ValuationResult.zeros(\nalgorithm=f\"semivalue-{str(sampler)}-{coefficient.__name__}\",  # type: ignore\nindices=u.data.indices,\ndata_names=u.data.data_names,\n)\nparallel_backend = init_parallel_backend(config)\nu = parallel_backend.put(u)\ncorrection = parallel_backend.put(\nlambda n, k: coefficient(n, k) * sampler.weight(n, k)\n)\nmax_workers = effective_n_jobs(n_jobs, config)\nn_submitted_jobs = 2 * max_workers  # number of jobs in the queue\nsampler_it = iter(sampler)\npbar = tqdm(disable=not progress, total=100, unit=\"%\")\nwith init_executor(\nmax_workers=max_workers, config=config, cancel_futures=True\n) as executor:\npending: set[Future] = set()\nwhile True:\npbar.n = 100 * done.completion()\npbar.refresh()\ncompleted, pending = wait(pending, timeout=1, return_when=FIRST_COMPLETED)\nfor future in completed:\nidx, marginal = future.result()\nresult.update(idx, marginal)\nif done(result):\nreturn result\n# Ensure that we always have n_submitted_jobs running\ntry:\nfor _ in range(n_submitted_jobs - len(pending)):\npending.add(\nexecutor.submit(\n_marginal,\nu=u,\ncoefficient=correction,\nsample=next(sampler_it),\n)\n)\nexcept StopIteration:\nif len(pending) == 0:\nreturn result\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_shapley_semivalues","title":"<code>compute_shapley_semivalues(u, *, done=MaxUpdates(100), sampler_t=PermutationSampler, n_jobs=1, config=ParallelConfig(), progress=False, seed=None)</code>","text":"<p>Computes Shapley values for a given utility function.</p> <p>This is a convenience wrapper for compute_generic_semivalues with the Shapley coefficient. Use compute_shapley_values for a more flexible interface and additional methods, including TMCS.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> DEFAULT: <code>MaxUpdates(100)</code> </p> <code>sampler_t</code> <p>The sampler type to use. See :mod:<code>pydvl.value.sampler</code> for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def compute_shapley_semivalues(\nu: Utility,\n*,\ndone: StoppingCriterion = MaxUpdates(100),\nsampler_t: Type[StochasticSampler] = PermutationSampler,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\nseed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n\"\"\"Computes Shapley values for a given utility function.\n    This is a convenience wrapper for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    with the Shapley coefficient. Use\n    [compute_shapley_values][pydvl.value.shapley.common.compute_shapley_values]\n    for a more flexible interface and additional methods, including TMCS.\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: Stopping criterion.\n        sampler_t: The sampler type to use. See :mod:`pydvl.value.sampler`\n            for a list.\n        n_jobs: Number of parallel jobs to use.\n        config: Object configuring parallel computation, with cluster\n            address, number of cpus, etc.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n        progress: Whether to display a progress bar.\n    Returns:\n        Object with the results.\n    \"\"\"\nreturn compute_generic_semivalues(\nsampler_t(u.data.indices, seed=seed),\nu,\nshapley_coefficient,\ndone,\nn_jobs=n_jobs,\nconfig=config,\nprogress=progress,\n)\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_banzhaf_semivalues","title":"<code>compute_banzhaf_semivalues(u, *, done=MaxUpdates(100), sampler_t=PermutationSampler, n_jobs=1, config=ParallelConfig(), progress=False, seed=None)</code>","text":"<p>Computes Banzhaf values for a given utility function.</p> <p>This is a convenience wrapper for compute_generic_semivalues with the Banzhaf coefficient.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> DEFAULT: <code>MaxUpdates(100)</code> </p> <code>sampler_t</code> <p>The sampler type to use. See :mod:<code>pydvl.value.sampler</code> for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def compute_banzhaf_semivalues(\nu: Utility,\n*,\ndone: StoppingCriterion = MaxUpdates(100),\nsampler_t: Type[StochasticSampler] = PermutationSampler,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\nseed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n\"\"\"Computes Banzhaf values for a given utility function.\n    This is a convenience wrapper for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    with the Banzhaf coefficient.\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: Stopping criterion.\n        sampler_t: The sampler type to use. See :mod:`pydvl.value.sampler` for a\n            list.\n        n_jobs: Number of parallel jobs to use.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        progress: Whether to display a progress bar.\n    Returns:\n        Object with the results.\n    \"\"\"\nreturn compute_generic_semivalues(\nsampler_t(u.data.indices, seed=seed),\nu,\nbanzhaf_coefficient,\ndone,\nn_jobs=n_jobs,\nconfig=config,\nprogress=progress,\n)\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_beta_shapley_semivalues","title":"<code>compute_beta_shapley_semivalues(u, *, alpha=1, beta=1, done=MaxUpdates(100), sampler_t=PermutationSampler, n_jobs=1, config=ParallelConfig(), progress=False, seed=None)</code>","text":"<p>Computes Beta Shapley values for a given utility function.</p> <p>This is a convenience wrapper for compute_generic_semivalues with the Beta Shapley coefficient.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>alpha</code> <p>Alpha parameter of the Beta distribution.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1</code> </p> <code>beta</code> <p>Beta parameter of the Beta distribution.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> DEFAULT: <code>MaxUpdates(100)</code> </p> <code>sampler_t</code> <p>The sampler type to use. See :mod:<code>pydvl.value.sampler</code> for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def compute_beta_shapley_semivalues(\nu: Utility,\n*,\nalpha: float = 1,\nbeta: float = 1,\ndone: StoppingCriterion = MaxUpdates(100),\nsampler_t: Type[StochasticSampler] = PermutationSampler,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\nseed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n\"\"\"Computes Beta Shapley values for a given utility function.\n    This is a convenience wrapper for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    with the Beta Shapley coefficient.\n    Args:\n        u: Utility object with model, data, and scoring function.\n        alpha: Alpha parameter of the Beta distribution.\n        beta: Beta parameter of the Beta distribution.\n        done: Stopping criterion.\n        sampler_t: The sampler type to use. See :mod:`pydvl.value.sampler` for a list.\n        n_jobs: Number of parallel jobs to use.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n        config: Object configuring parallel computation, with cluster address, number of\n            cpus, etc.\n        progress: Whether to display a progress bar.\n    Returns:\n        Object with the results.\n    \"\"\"\nreturn compute_generic_semivalues(\nsampler_t(u.data.indices, seed=seed),\nu,\nbeta_coefficient(alpha, beta),\ndone,\nn_jobs=n_jobs,\nconfig=config,\nprogress=progress,\n)\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_semivalues","title":"<code>compute_semivalues(u, *, done=MaxUpdates(100), mode=SemiValueMode.Shapley, sampler_t=PermutationSampler, n_jobs=1, seed=None, **kwargs)</code>","text":"<p>Convenience entry point for most common semi-value computations.</p> <p>Deprecation warning</p> <p>This method is deprecated and will be replaced in 0.8.0 by the more general implementation of compute_generic_semivalues. Use compute_shapley_semivalues, compute_banzhaf_semivalues, or compute_beta_shapley_semivalues instead.</p> <p>The modes supported with this interface are the following. For greater flexibility use compute_generic_semivalues directly.</p> <ul> <li>SemiValueMode.Shapley:   Shapley values.</li> <li>SemiValueMode.BetaShapley:   Implements the Beta Shapley semi-value as introduced in   (Kwon and Zou, 2022)<sup>1</sup>.   Pass additional keyword arguments <code>alpha</code> and <code>beta</code> to set the   parameters of the Beta distribution (both default to 1).</li> <li>SemiValueMode.Banzhaf: Implements the Banzhaf   semi-value as introduced in (Wang and Jia, 2022)<sup>1</sup>.</li> </ul> <p>See [[data-valuation]] for an overview of valuation. - SemiValueMode.Banzhaf: Implements   the Banzhaf semi-value as introduced in [@wang_data_2022].</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> DEFAULT: <code>MaxUpdates(100)</code> </p> <code>mode</code> <p>The semi-value mode to use. See SemiValueMode for a list.</p> <p> TYPE: <code>SemiValueMode</code> DEFAULT: <code>Shapley</code> </p> <code>sampler_t</code> <p>The sampler type to use. See sampler for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments passed to compute_generic_semivalues.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(target=True, deprecated_in=\"0.7.0\", remove_in=\"0.8.0\")\ndef compute_semivalues(\nu: Utility,\n*,\ndone: StoppingCriterion = MaxUpdates(100),\nmode: SemiValueMode = SemiValueMode.Shapley,\nsampler_t: Type[StochasticSampler] = PermutationSampler,\nn_jobs: int = 1,\nseed: Optional[Seed] = None,\n**kwargs,\n) -&gt; ValuationResult:\n\"\"\"Convenience entry point for most common semi-value computations.\n    !!! warning \"Deprecation warning\"\n        This method is deprecated and will be replaced in 0.8.0 by the more\n        general implementation of\n        [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].\n        Use\n        [compute_shapley_semivalues][pydvl.value.semivalues.compute_shapley_semivalues],\n        [compute_banzhaf_semivalues][pydvl.value.semivalues.compute_banzhaf_semivalues],\n        or\n        [compute_beta_shapley_semivalues][pydvl.value.semivalues.compute_beta_shapley_semivalues]\n        instead.\n    The modes supported with this interface are the following. For greater\n    flexibility use\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    directly.\n    - [SemiValueMode.Shapley][pydvl.value.semivalues.SemiValueMode]:\n      Shapley values.\n    - [SemiValueMode.BetaShapley][pydvl.value.semivalues.SemiValueMode.BetaShapley]:\n      Implements the Beta Shapley semi-value as introduced in\n      (Kwon and Zou, 2022)&lt;sup&gt;&lt;a href=\"#kwon_beta_2022\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n      Pass additional keyword arguments `alpha` and `beta` to set the\n      parameters of the Beta distribution (both default to 1).\n    - [SemiValueMode.Banzhaf][SemiValueMode.Banzhaf]: Implements the Banzhaf\n      semi-value as introduced in (Wang and Jia, 2022)&lt;sup&gt;&lt;a href=\"#wang_data_2022\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    See [[data-valuation]] for an overview of valuation.\n    - [SemiValueMode.Banzhaf][pydvl.value.semivalues.SemiValueMode]: Implements\n      the Banzhaf semi-value as introduced in [@wang_data_2022].\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: Stopping criterion.\n        mode: The semi-value mode to use. See\n            [SemiValueMode][pydvl.value.semivalues.SemiValueMode] for a list.\n        sampler_t: The sampler type to use. See [sampler][pydvl.value.sampler]\n            for a list.\n        n_jobs: Number of parallel jobs to use.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n        kwargs: Additional keyword arguments passed to\n            [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].\n    Returns:\n        Object with the results.\n    \"\"\"\nif mode == SemiValueMode.Shapley:\ncoefficient = shapley_coefficient\nelif mode == SemiValueMode.BetaShapley:\nalpha = kwargs.pop(\"alpha\", 1)\nbeta = kwargs.pop(\"beta\", 1)\ncoefficient = beta_coefficient(alpha, beta)\nelif mode == SemiValueMode.Banzhaf:\ncoefficient = banzhaf_coefficient\nelse:\nraise ValueError(f\"Unknown mode {mode}\")\ncoefficient = cast(SVCoefficient, coefficient)\nreturn compute_generic_semivalues(\nsampler_t(u.data.indices, seed=seed),\nu,\ncoefficient,\ndone,\nn_jobs=n_jobs,\n**kwargs,\n)\n</code></pre>"},{"location":"api/pydvl/value/stopping/","title":"Stopping","text":"<p>Stopping criteria for value computations.</p> <p>This module provides a basic set of stopping criteria, like MaxUpdates, MaxTime, or HistoryDeviation among others. These can behave in different ways depending on the context. For example, MaxUpdates limits the number of updates to values, which depending on the algorithm may mean a different number of utility evaluations or imply other computations like solving a linear or quadratic program.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--creating-stopping-criteria","title":"Creating stopping criteria","text":"<p>The easiest way is to declare a function implementing the interface StoppingCriterionCallable and wrap it with make_criterion(). This creates a StoppingCriterion object that can be composed with other stopping criteria.</p> <p>Alternatively, and in particular if reporting of completion is required, one can inherit from this class and implement the abstract methods _check and completion.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--composing-stopping-criteria","title":"Composing stopping criteria","text":"<p>Objects of type StoppingCriterion can be composed with the binary operators <code>&amp;</code> (and), and <code>|</code> (or), following the truth tables of Status. The unary operator <code>~</code> (not) is also supported. See StoppingCriterion for details on how these operations affect the behavior of the stopping criteria.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterionCallable","title":"<code>StoppingCriterionCallable</code>","text":"<p>             Bases: <code>Protocol</code></p> <p>Signature for a stopping criterion</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion","title":"<code>StoppingCriterion(modify_result=True)</code>","text":"<p>             Bases: <code>ABC</code></p> <p>A composable callable object to determine whether a computation must stop.</p> <p>A <code>StoppingCriterion</code> is a callable taking a ValuationResult and returning a Status. It also keeps track of individual convergence of values with converged, and reports the overall completion of the computation with completion.</p> <p>Instances of <code>StoppingCriterion</code> can be composed with the binary operators <code>&amp;</code> (and), and <code>|</code> (or), following the truth tables of Status. The unary operator <code>~</code> (not) is also supported. These boolean operations act according to the following rules:</p> <ul> <li>The results of _check are   combined with the operator. See Status for   the truth tables.</li> <li>The results of   converged are combined   with the operator (returning another boolean array).</li> <li>The completion   method returns the min, max, or the complement to 1 of the completions of   the operands, for AND, OR and NOT respectively. This is required for cases   where one of the criteria does not keep track of the convergence of single   values, e.g. MaxUpdates, because   completion by   default returns the mean of the boolean convergence array.</li> </ul>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion--subclassing","title":"Subclassing","text":"<p>Subclassing this class requires implementing a _check method that returns a Status object based on a given ValuationResult. This method should update the attribute _converged, which is a boolean array indicating whether the value for each index has converged. When this does not make sense for a particular stopping criterion, completion should be overridden to provide an overall completion value, since its default implementation attempts to compute the mean of _converged.</p> PARAMETER  DESCRIPTION <code>modify_result</code> <p>If <code>True</code> the status of the input ValuationResult is modified in place after the call.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, modify_result: bool = True):\nself.modify_result = modify_result\nself._converged = np.full(0, False)\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion.converged","title":"<code>converged: NDArray[np.bool_]</code>  <code>property</code>","text":"<p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their _check.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion.completion","title":"<code>completion()</code>","text":"<p>Returns a value between 0 and 1 indicating the completion of the computation.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def completion(self) -&gt; float:\n\"\"\"Returns a value between 0 and 1 indicating the completion of the\n    computation.\n    \"\"\"\nif self.converged.size == 0:\nreturn 0.0\nreturn float(np.mean(self.converged).item())\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion.__call__","title":"<code>__call__(result)</code>","text":"<p>Calls _check, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n\"\"\"Calls [_check][pydvl.value.stopping.StoppingCriterion._check], maybe updating the result.\"\"\"\nif len(result) == 0:\nlogger.warning(\n\"At least one iteration finished but no results where generated. \"\n\"Please check that your scorer and utility return valid numbers.\"\n)\nstatus = self._check(result)\nif self.modify_result:  # FIXME: this is not nice\nresult._status = status\nreturn status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.AbsoluteStandardError","title":"<code>AbsoluteStandardError(threshold, fraction=1.0, burn_in=4, modify_result=True)</code>","text":"<p>             Bases: <code>StoppingCriterion</code></p> <p>Determine convergence based on the standard error of the values.</p> <p>If \\(s_i\\) is the standard error for datum \\(i\\) and \\(v_i\\) its value, then this criterion returns Converged if \\(s_i &lt; \\epsilon\\) for all \\(i\\) and a threshold value \\(\\epsilon \\gt 0\\).</p> PARAMETER  DESCRIPTION <code>threshold</code> <p>A value is considered to have converged if the standard error is below this value. A way of choosing it is to pick some percentage of the range of the values. For Shapley values this is the difference between the maximum and minimum of the utility function (to see this substitute the maximum and minimum values of the utility into the marginal contribution formula).</p> <p> TYPE: <code>float</code> </p> <code>fraction</code> <p>The fraction of values that must have converged for the criterion to return Converged.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>burn_in</code> <p>The number of iterations to ignore before checking for convergence. This is required because computations typically start with zero variance, as a result of using empty(). The default is set to an arbitrary minimum which is usually enough but may need to be increased.</p> <p> TYPE: <code>int</code> DEFAULT: <code>4</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(\nself,\nthreshold: float,\nfraction: float = 1.0,\nburn_in: int = 4,\nmodify_result: bool = True,\n):\nsuper().__init__(modify_result=modify_result)\nself.threshold = threshold\nself.fraction = fraction\nself.burn_in = burn_in\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxChecks","title":"<code>MaxChecks(n_checks, modify_result=True)</code>","text":"<p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate as soon as the number of checks exceeds the threshold.</p> <p>A \"check\" is one call to the criterion.</p> PARAMETER  DESCRIPTION <code>n_checks</code> <p>Threshold: if <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[int]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, n_checks: Optional[int], modify_result: bool = True):\nsuper().__init__(modify_result=modify_result)\nif n_checks is not None and n_checks &lt; 1:\nraise ValueError(\"n_iterations must be at least 1 or None\")\nself.n_checks = n_checks\nself._count = 0\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxUpdates","title":"<code>MaxUpdates(n_updates, modify_result=True)</code>","text":"<p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate if any number of value updates exceeds or equals the given threshold.</p> <p>Note</p> <p>If you want to ensure that all values have been updated, you probably want MinUpdates instead.</p> <p>This checks the <code>counts</code> field of a ValuationResult, i.e. the number of times that each index has been updated. For powerset samplers, the maximum of this number coincides with the maximum number of subsets sampled. For permutation samplers, it coincides with the number of permutations sampled.</p> PARAMETER  DESCRIPTION <code>n_updates</code> <p>Threshold: if <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[int]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, n_updates: Optional[int], modify_result: bool = True):\nsuper().__init__(modify_result=modify_result)\nif n_updates is not None and n_updates &lt; 1:\nraise ValueError(\"n_updates must be at least 1 or None\")\nself.n_updates = n_updates\nself.last_max = 0\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MinUpdates","title":"<code>MinUpdates(n_updates, modify_result=True)</code>","text":"<p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate as soon as all value updates exceed or equal the given threshold.</p> <p>This checks the <code>counts</code> field of a ValuationResult, i.e. the number of times that each index has been updated. For powerset samplers, the minimum of this number is a lower bound for the number of subsets sampled. For permutation samplers, it lower-bounds the amount of permutations sampled.</p> PARAMETER  DESCRIPTION <code>n_updates</code> <p>Threshold: if <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[int]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, n_updates: Optional[int], modify_result: bool = True):\nsuper().__init__(modify_result=modify_result)\nself.n_updates = n_updates\nself.last_min = 0\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxTime","title":"<code>MaxTime(seconds, modify_result=True)</code>","text":"<p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate if the computation time exceeds the given number of seconds.</p> <p>Checks the elapsed time since construction</p> PARAMETER  DESCRIPTION <code>seconds</code> <p>Threshold: The computation is terminated if the elapsed time between object construction and a _check exceeds this value. If <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[float]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, seconds: Optional[float], modify_result: bool = True):\nsuper().__init__(modify_result=modify_result)\nself.max_seconds = seconds or np.inf\nif self.max_seconds &lt;= 0:\nraise ValueError(\"Number of seconds for MaxTime must be positive or None\")\nself.start = time()\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.HistoryDeviation","title":"<code>HistoryDeviation(n_steps, rtol, pin_converged=True, modify_result=True)</code>","text":"<p>             Bases: <code>StoppingCriterion</code></p> <p>A simple check for relative distance to a previous step in the computation.</p> <p>The method used by (Ghorbani and Zou, 2019)<sup>1</sup> computes the relative distances between the current values \\(v_i^t\\) and the values at the previous checkpoint \\(v_i^{t-\\tau}\\). If the sum is below a given threshold, the computation is terminated.</p> \\[\\sum_{i=1}^n \\frac{\\left| v_i^t - v_i^{t-\\tau} \\right|}{v_i^t} &lt; \\epsilon.\\] <p>When the denominator is zero, the summand is set to the value of \\(v_i^{ t-\\tau}\\).</p> <p>This implementation is slightly generalised to allow for different number of updates to individual indices, as happens with powerset samplers instead of permutations. Every subset of indices that is found to converge can be pinned to that state. Once all indices have converged the method has converged.</p> <p>Warning</p> <p>This criterion is meant for the reproduction of the results in the paper, but we do not recommend using it in practice.</p> PARAMETER  DESCRIPTION <code>n_steps</code> <p>Checkpoint values every so many updates and use these saved values to compare.</p> <p> TYPE: <code>int</code> </p> <code>rtol</code> <p>Relative tolerance for convergence (\\(\\epsilon\\) in the formula).</p> <p> TYPE: <code>float</code> </p> <code>pin_converged</code> <p>If <code>True</code>, once an index has converged, it is pinned</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(\nself,\nn_steps: int,\nrtol: float,\npin_converged: bool = True,\nmodify_result: bool = True,\n):\nsuper().__init__(modify_result=modify_result)\nif n_steps &lt; 1:\nraise ValueError(\"n_steps must be at least 1\")\nif rtol &lt;= 0 or rtol &gt;= 1:\nraise ValueError(\"rtol must be in (0, 1)\")\nself.n_steps = n_steps\nself.rtol = rtol\nself.update_op = np.logical_or if pin_converged else np.logical_and\nself._memory = None  # type: ignore\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.make_criterion","title":"<code>make_criterion(fun, converged=None, completion=None, name=None)</code>","text":"<p>Create a new StoppingCriterion from a function. Use this to enable simpler functions to be composed with bitwise operators</p> PARAMETER  DESCRIPTION <code>fun</code> <p>The callable to wrap.</p> <p> TYPE: <code>StoppingCriterionCallable</code> </p> <code>converged</code> <p>A callable that returns a boolean array indicating what values have converged.</p> <p> TYPE: <code>Callable[[], NDArray[bool_]] | None</code> DEFAULT: <code>None</code> </p> <code>completion</code> <p>A callable that returns a value between 0 and 1 indicating the rate of completion of the computation. If not provided, the fraction of converged values is used.</p> <p> TYPE: <code>Callable[[], float] | None</code> DEFAULT: <code>None</code> </p> <code>name</code> <p>The name of the new criterion. If <code>None</code>, the <code>__name__</code> of the function is used.</p> <p> TYPE: <code>str | None</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Type[StoppingCriterion]</code> <p>A new subclass of StoppingCriterion.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def make_criterion(\nfun: StoppingCriterionCallable,\nconverged: Callable[[], NDArray[np.bool_]] | None = None,\ncompletion: Callable[[], float] | None = None,\nname: str | None = None,\n) -&gt; Type[StoppingCriterion]:\n\"\"\"Create a new [StoppingCriterion][pydvl.value.stopping.StoppingCriterion] from a function.\n    Use this to enable simpler functions to be composed with bitwise operators\n    Args:\n        fun: The callable to wrap.\n        converged: A callable that returns a boolean array indicating what\n            values have converged.\n        completion: A callable that returns a value between 0 and 1 indicating\n            the rate of completion of the computation. If not provided, the fraction\n            of converged values is used.\n        name: The name of the new criterion. If `None`, the `__name__` of\n            the function is used.\n    Returns:\n        A new subclass of [StoppingCriterion][pydvl.value.stopping.StoppingCriterion].\n    \"\"\"\nclass WrappedCriterion(StoppingCriterion):\ndef __init__(self, modify_result: bool = True):\nsuper().__init__(modify_result=modify_result)\nself._name = name or getattr(fun, \"__name__\", \"WrappedCriterion\")\ndef _check(self, result: ValuationResult) -&gt; Status:\nreturn fun(result)\n@property\ndef converged(self) -&gt; NDArray[np.bool_]:\nif converged is None:\nreturn super().converged\nreturn converged()\n@property\ndef name(self):\nreturn self._name\ndef completion(self) -&gt; float:\nif completion is None:\nreturn super().completion()\nreturn completion()\nreturn WrappedCriterion\n</code></pre>"},{"location":"api/pydvl/value/least_core/","title":"Least core","text":"<p>New in version 0.4.0</p> <p>This package holds all routines for the computation of Least Core data values.</p> <p>Please refer to Data valuation for an overview.</p> <p>In addition to the standard interface via compute_least_core_values(), because computing the Least Core values requires the solution of a linear and a quadratic problem after computing all the utility values, there is the possibility of performing each step separately. This is useful when running multiple experiments: use lc_prepare_problem() or mclc_prepare_problem() to prepare a list of problems to solve, then solve them in parallel with lc_solve_problems().</p> <p>Note that mclc_prepare_problem() is parallelized itself, so preparing the problems should be done in sequence in this case. The solution of the linear systems can then be done in parallel.</p>"},{"location":"api/pydvl/value/least_core/#pydvl.value.least_core.LeastCoreMode","title":"<code>LeastCoreMode</code>","text":"<p>             Bases: <code>Enum</code></p> <p>Available Least Core algorithms.</p>"},{"location":"api/pydvl/value/least_core/#pydvl.value.least_core.compute_least_core_values","title":"<code>compute_least_core_values(u, *, n_jobs=1, n_iterations=None, mode=LeastCoreMode.MonteCarlo, non_negative_subsidy=False, solver_options=None, **kwargs)</code>","text":"<p>Umbrella method to compute Least Core values with any of the available algorithms.</p> <p>See Data valuation for an overview.</p> <p>The following algorithms are available. Note that the exact method can only work with very small datasets and is thus intended only for testing.</p> <ul> <li><code>exact</code>: uses the complete powerset of the training set for the constraints   combinatorial_exact_shapley().</li> <li><code>montecarlo</code>:  uses the approximate Monte Carlo Least Core algorithm.   Implemented in montecarlo_least_core().</li> </ul> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_jobs</code> <p>Number of jobs to run in parallel. Only used for Monte Carlo Least Core.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_iterations</code> <p>Number of subsets to sample and evaluate the utility on. Only used for Monte Carlo Least Core.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>Algorithm to use. See LeastCoreMode for available options.</p> <p> TYPE: <code>LeastCoreMode</code> DEFAULT: <code>MonteCarlo</code> </p> <code>non_negative_subsidy</code> <p>If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>solver_options</code> <p>Optional dictionary of options passed to the solvers.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the computed values.</p> <p>New in version 0.5.0</p> Source code in <code>src/pydvl/value/least_core/__init__.py</code> <pre><code>def compute_least_core_values(\nu: Utility,\n*,\nn_jobs: int = 1,\nn_iterations: Optional[int] = None,\nmode: LeastCoreMode = LeastCoreMode.MonteCarlo,\nnon_negative_subsidy: bool = False,\nsolver_options: Optional[dict] = None,\n**kwargs,\n) -&gt; ValuationResult:\n\"\"\"Umbrella method to compute Least Core values with any of the available\n    algorithms.\n    See [Data valuation][computing-data-values] for an overview.\n    The following algorithms are available. Note that the exact method can only\n    work with very small datasets and is thus intended only for testing.\n    - `exact`: uses the complete powerset of the training set for the constraints\n      [combinatorial_exact_shapley()][pydvl.value.shapley.naive.combinatorial_exact_shapley].\n    - `montecarlo`:  uses the approximate Monte Carlo Least Core algorithm.\n      Implemented in [montecarlo_least_core()][pydvl.value.least_core.montecarlo.montecarlo_least_core].\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_jobs: Number of jobs to run in parallel. Only used for Monte Carlo\n            Least Core.\n        n_iterations: Number of subsets to sample and evaluate the utility on.\n            Only used for Monte Carlo Least Core.\n        mode: Algorithm to use. See\n            [LeastCoreMode][pydvl.value.least_core.LeastCoreMode] for available\n            options.\n        non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Optional dictionary of options passed to the solvers.\n    Returns:\n        Object with the computed values.\n    !!! tip \"New in version 0.5.0\"\n    \"\"\"\nprogress: bool = kwargs.pop(\"progress\", False)\n# TODO: remove this before releasing version 0.7.0\nif kwargs:\nwarnings.warn(\nDeprecationWarning(\n\"Passing solver options as kwargs was deprecated in 0.6.0, will \"\n\"be removed in 0.7.0. `Use solver_options` instead.\"\n)\n)\nif solver_options is None:\nsolver_options = kwargs\nelse:\nsolver_options.update(kwargs)\nif mode == LeastCoreMode.MonteCarlo:\n# TODO fix progress showing and maybe_progress in remote case\nprogress = False\nif n_iterations is None:\nraise ValueError(\"n_iterations cannot be None for Monte Carlo Least Core\")\nreturn montecarlo_least_core(\nu=u,\nn_iterations=n_iterations,\nn_jobs=n_jobs,\nprogress=progress,\nnon_negative_subsidy=non_negative_subsidy,\nsolver_options=solver_options,\n**kwargs,\n)\nelif mode == LeastCoreMode.Exact:\nreturn exact_least_core(\nu=u,\nprogress=progress,\nnon_negative_subsidy=non_negative_subsidy,\nsolver_options=solver_options,\n)\nraise ValueError(f\"Invalid value encountered in {mode=}\")\n</code></pre>"},{"location":"api/pydvl/value/least_core/common/","title":"Common","text":""},{"location":"api/pydvl/value/least_core/common/#pydvl.value.least_core.common.lc_solve_problem","title":"<code>lc_solve_problem(problem, *, u, algorithm, non_negative_subsidy=False, solver_options=None, **options)</code>","text":"<p>Solves a linear problem as prepared by mclc_prepare_problem(). Useful for parallel execution of multiple experiments by running this as a remote task.</p> <p>See exact_least_core() or montecarlo_least_core() for argument descriptions.</p> Source code in <code>src/pydvl/value/least_core/common.py</code> <pre><code>def lc_solve_problem(\nproblem: LeastCoreProblem,\n*,\nu: Utility,\nalgorithm: str,\nnon_negative_subsidy: bool = False,\nsolver_options: Optional[dict] = None,\n**options,\n) -&gt; ValuationResult:\n\"\"\"Solves a linear problem as prepared by\n    [mclc_prepare_problem()][pydvl.value.least_core.montecarlo.mclc_prepare_problem].\n    Useful for parallel execution of multiple experiments by running this as a\n    remote task.\n    See [exact_least_core()][pydvl.value.least_core.naive.exact_least_core] or\n    [montecarlo_least_core()][pydvl.value.least_core.montecarlo.montecarlo_least_core] for\n    argument descriptions.\n    \"\"\"\nn = len(u.data)\nif np.any(np.isnan(problem.utility_values)):\nwarnings.warn(\nf\"Calculation returned \"\nf\"{np.sum(np.isnan(problem.utility_values))} NaN \"\nf\"values out of {problem.utility_values.size}\",\nRuntimeWarning,\n)\n# TODO: remove this before releasing version 0.7.0\nif options:\nwarnings.warn(\nDeprecationWarning(\n\"Passing solver options as kwargs was deprecated in \"\n\"0.6.0, will be removed in 0.7.0. `Use solver_options` \"\n\"instead.\"\n)\n)\nif solver_options is None:\nsolver_options = options\nelse:\nsolver_options.update(options)\nif solver_options is None:\nsolver_options = {}\nif \"solver\" not in solver_options:\nsolver_options[\"solver\"] = cp.SCS\nif \"max_iters\" not in solver_options and solver_options[\"solver\"] == cp.SCS:\nsolver_options[\"max_iters\"] = 10000\nlogger.debug(\"Removing possible duplicate values in lower bound array\")\nb_lb = problem.utility_values\nA_lb, unique_indices = np.unique(problem.A_lb, return_index=True, axis=0)\nb_lb = b_lb[unique_indices]\nlogger.debug(\"Building equality constraint\")\nA_eq = np.ones((1, n))\n# We might have already computed the total utility one or more times.\n# This is the index of the row(s) in A_lb with all ones.\ntotal_utility_indices = np.where(A_lb.sum(axis=1) == n)[0]\nif len(total_utility_indices) == 0:\nb_eq = np.array([u(u.data.indices)])\nelse:\nb_eq = b_lb[total_utility_indices]\n# Remove the row(s) corresponding to the total utility\n# from the lower bound constraints\n# because given the equality constraint\n# it is the same as using the constraint e &gt;= 0\n# (i.e. setting non_negative_subsidy = True).\nmask: NDArray[np.bool_] = np.ones_like(b_lb, dtype=bool)\nmask[total_utility_indices] = False\nb_lb = b_lb[mask]\nA_lb = A_lb[mask]\n# Remove the row(s) corresponding to the empty subset\n# because, given u(\u2205) = (which is almost always the case,\n# it is the same as using the constraint e &gt;= 0\n# (i.e. setting non_negative_subsidy = True).\nemptyset_utility_indices = np.where(A_lb.sum(axis=1) == 0)[0]\nif len(emptyset_utility_indices) &gt; 0:\nmask = np.ones_like(b_lb, dtype=bool)\nmask[emptyset_utility_indices] = False\nb_lb = b_lb[mask]\nA_lb = A_lb[mask]\n_, subsidy = _solve_least_core_linear_program(\nA_eq=A_eq,\nb_eq=b_eq,\nA_lb=A_lb,\nb_lb=b_lb,\nnon_negative_subsidy=non_negative_subsidy,\nsolver_options=solver_options,\n)\nvalues: Optional[NDArray[np.float_]]\nif subsidy is None:\nlogger.debug(\"No values were found\")\nstatus = Status.Failed\nvalues = np.empty(n)\nvalues[:] = np.nan\nsubsidy = np.nan\nelse:\nvalues = _solve_egalitarian_least_core_quadratic_program(\nsubsidy,\nA_eq=A_eq,\nb_eq=b_eq,\nA_lb=A_lb,\nb_lb=b_lb,\nsolver_options=solver_options,\n)\nif values is None:\nlogger.debug(\"No values were found\")\nstatus = Status.Failed\nvalues = np.empty(n)\nvalues[:] = np.nan\nsubsidy = np.nan\nelse:\nstatus = Status.Converged\nreturn ValuationResult(\nalgorithm=algorithm,\nstatus=status,\nvalues=values,\nsubsidy=subsidy,\nstderr=None,\ndata_names=u.data.data_names,\n)\n</code></pre>"},{"location":"api/pydvl/value/least_core/common/#pydvl.value.least_core.common.lc_solve_problems","title":"<code>lc_solve_problems(problems, u, algorithm, config=ParallelConfig(), n_jobs=1, non_negative_subsidy=True, solver_options=None, **options)</code>","text":"<p>Solves a list of linear problems in parallel.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility.</p> <p> TYPE: <code>Utility</code> </p> <code>problems</code> <p>Least Core problems to solve, as returned by mclc_prepare_problem().</p> <p> TYPE: <code>Sequence[LeastCoreProblem]</code> </p> <code>algorithm</code> <p>Name of the valuation algorithm.</p> <p> TYPE: <code>str</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to run.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>non_negative_subsidy</code> <p>If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>solver_options</code> <p>Additional options to pass to the solver.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>List[ValuationResult]</code> <p>List of solutions.</p> Source code in <code>src/pydvl/value/least_core/common.py</code> <pre><code>def lc_solve_problems(\nproblems: Sequence[LeastCoreProblem],\nu: Utility,\nalgorithm: str,\nconfig: ParallelConfig = ParallelConfig(),\nn_jobs: int = 1,\nnon_negative_subsidy: bool = True,\nsolver_options: Optional[dict] = None,\n**options,\n) -&gt; List[ValuationResult]:\n\"\"\"Solves a list of linear problems in parallel.\n    Args:\n        u: Utility.\n        problems: Least Core problems to solve, as returned by\n            [mclc_prepare_problem()][pydvl.value.least_core.montecarlo.mclc_prepare_problem].\n        algorithm: Name of the valuation algorithm.\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        n_jobs: Number of parallel jobs to run.\n        non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Additional options to pass to the solver.\n    Returns:\n        List of solutions.\n    \"\"\"\ndef _map_func(\nproblems: List[LeastCoreProblem], *args, **kwargs\n) -&gt; List[ValuationResult]:\nreturn [lc_solve_problem(p, *args, **kwargs) for p in problems]\nmap_reduce_job: MapReduceJob[\n\"LeastCoreProblem\", \"List[ValuationResult]\"\n] = MapReduceJob(\ninputs=problems,\nmap_func=_map_func,\nmap_kwargs=dict(\nu=u,\nalgorithm=algorithm,\nnon_negative_subsidy=non_negative_subsidy,\nsolver_options=solver_options,\n**options,\n),\nreduce_func=lambda x: list(itertools.chain(*x)),\nconfig=config,\nn_jobs=n_jobs,\n)\nsolutions = map_reduce_job()\nreturn solutions\n</code></pre>"},{"location":"api/pydvl/value/least_core/montecarlo/","title":"Montecarlo","text":""},{"location":"api/pydvl/value/least_core/montecarlo/#pydvl.value.least_core.montecarlo.montecarlo_least_core","title":"<code>montecarlo_least_core(u, n_iterations, *, n_jobs=1, config=ParallelConfig(), non_negative_subsidy=False, solver_options=None, options=None, progress=False)</code>","text":"<p>Computes approximate Least Core values using a Monte Carlo approach.</p> \\[ \\begin{array}{lll} \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\ \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\ &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp; , \\forall S \\in \\{S_1, S_2, \\dots, S_m \\overset{\\mathrm{iid}}{\\sim} U(2^N) \\} \\end{array} \\] <p>Where:</p> <ul> <li>\\(U(2^N)\\) is the uniform distribution over the powerset of \\(N\\).</li> <li>\\(m\\) is the number of subsets that will be sampled and whose utility will   be computed and used to compute the data values.</li> </ul> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_iterations</code> <p>total number of iterations to use</p> <p> TYPE: <code>int</code> </p> <code>n_jobs</code> <p>number of jobs across which to distribute the computation</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>non_negative_subsidy</code> <p>If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>solver_options</code> <p>Dictionary of options that will be used to select a solver and to configure it. Refer to cvxpy's documentation for all possible options.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> <code>options</code> <p>(Deprecated) Dictionary of solver options. Use solver_options instead.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, shows a tqdm progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values and the least core value.</p> Source code in <code>src/pydvl/value/least_core/montecarlo.py</code> <pre><code>def montecarlo_least_core(\nu: Utility,\nn_iterations: int,\n*,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nnon_negative_subsidy: bool = False,\nsolver_options: Optional[dict] = None,\noptions: Optional[dict] = None,\nprogress: bool = False,\n) -&gt; ValuationResult:\nr\"\"\"Computes approximate Least Core values using a Monte Carlo approach.\n    $$\n    \\begin{array}{lll}\n    \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\\n    \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\\n    &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp; ,\n    \\forall S \\in \\{S_1, S_2, \\dots, S_m \\overset{\\mathrm{iid}}{\\sim} U(2^N) \\}\n    \\end{array}\n    $$\n    Where:\n    * $U(2^N)$ is the uniform distribution over the powerset of $N$.\n    * $m$ is the number of subsets that will be sampled and whose utility will\n      be computed and used to compute the data values.\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_iterations: total number of iterations to use\n        n_jobs: number of jobs across which to distribute the computation\n        config: Object configuring parallel computation, with cluster\n            address, number of cpus, etc.\n        non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Dictionary of options that will be used to select a solver\n            and to configure it. Refer to [cvxpy's\n            documentation](https://www.cvxpy.org/tutorial/advanced/index.html#setting-solver-options)\n            for all possible options.\n        options: (Deprecated) Dictionary of solver options. Use solver_options\n            instead.\n        progress: If True, shows a tqdm progress bar\n    Returns:\n        Object with the data values and the least core value.\n    \"\"\"\n# TODO: remove this before releasing version 0.7.0\nif options:\nwarnings.warn(\nDeprecationWarning(\n\"Passing solver options as kwargs was deprecated in \"\n\"0.6.0, will be removed in 0.7.0. `Use solver_options` \"\n\"instead.\"\n)\n)\nif solver_options is None:\nsolver_options = options\nelse:\nsolver_options.update(options)\nproblem = mclc_prepare_problem(\nu, n_iterations, n_jobs=n_jobs, config=config, progress=progress\n)\nreturn lc_solve_problem(\nproblem,\nu=u,\nalgorithm=\"montecarlo_least_core\",\nnon_negative_subsidy=non_negative_subsidy,\nsolver_options=solver_options,\n)\n</code></pre>"},{"location":"api/pydvl/value/least_core/montecarlo/#pydvl.value.least_core.montecarlo.mclc_prepare_problem","title":"<code>mclc_prepare_problem(u, n_iterations, *, n_jobs=1, config=ParallelConfig(), progress=False)</code>","text":"<p>Prepares a linear problem by sampling subsets of the data. Use this to separate the problem preparation from the solving with lc_solve_problem(). Useful for parallel execution of multiple experiments.</p> <p>See montecarlo_least_core for argument descriptions.</p> Source code in <code>src/pydvl/value/least_core/montecarlo.py</code> <pre><code>def mclc_prepare_problem(\nu: Utility,\nn_iterations: int,\n*,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\n) -&gt; LeastCoreProblem:\n\"\"\"Prepares a linear problem by sampling subsets of the data. Use this to\n    separate the problem preparation from the solving with\n    [lc_solve_problem()][pydvl.value.least_core.common.lc_solve_problem]. Useful\n    for parallel execution of multiple experiments.\n    See\n    [montecarlo_least_core][pydvl.value.least_core.montecarlo.montecarlo_least_core]\n    for argument descriptions.\n    \"\"\"\nn = len(u.data)\nif n_iterations &lt; n:\nwarnings.warn(\nf\"Number of iterations '{n_iterations}' is smaller the size of the dataset '{n}'. \"\nf\"This is not optimal because in the worst case we need at least '{n}' constraints \"\n\"to satisfy the individual rationality condition.\"\n)\nif n_iterations &gt; 2**n:\nwarnings.warn(\nf\"Passed n_iterations is greater than the number subsets! \"\nf\"Setting it to 2^{n}\",\nRuntimeWarning,\n)\nn_iterations = 2**n\niterations_per_job = max(1, n_iterations // effective_n_jobs(n_jobs, config))\nmap_reduce_job: MapReduceJob[\"Utility\", \"LeastCoreProblem\"] = MapReduceJob(\ninputs=u,\nmap_func=_montecarlo_least_core,\nreduce_func=_reduce_func,\nmap_kwargs=dict(n_iterations=iterations_per_job, progress=progress),\nn_jobs=n_jobs,\nconfig=config,\n)\nreturn map_reduce_job()\n</code></pre>"},{"location":"api/pydvl/value/least_core/naive/","title":"Naive","text":""},{"location":"api/pydvl/value/least_core/naive/#pydvl.value.least_core.naive.exact_least_core","title":"<code>exact_least_core(u, *, non_negative_subsidy=False, solver_options=None, options=None, progress=True)</code>","text":"<p>Computes the exact Least Core values.</p> <p>Note</p> <p>If the training set contains more than 20 instances a warning is printed because the computation is very expensive. This method is mostly used for internal testing and simple use cases. Please refer to the Monte Carlo method for practical applications.</p> <p>The least core is the solution to the following Linear Programming problem:</p> \\[ \\begin{array}{lll} \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\ \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\ &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp;, \\forall S \\subseteq N \\\\ \\end{array} \\] <p>Where \\(N = \\{1, 2, \\dots, n\\}\\) are the training set's indices.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function non_negative_subsidy: If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>Utility</code> </p> <code>solver_options</code> <p>Dictionary of options that will be used to select a solver and to configure it. Refer to the cvxpy's documentation for all possible options.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> <code>options</code> <p>(Deprecated) Dictionary of solver options. Use <code>solver_options</code> instead.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, shows a tqdm progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values and the least core value.</p> Source code in <code>src/pydvl/value/least_core/naive.py</code> <pre><code>def exact_least_core(\nu: Utility,\n*,\nnon_negative_subsidy: bool = False,\nsolver_options: Optional[dict] = None,\noptions: Optional[dict] = None,\nprogress: bool = True,\n) -&gt; ValuationResult:\nr\"\"\"Computes the exact Least Core values.\n    !!! Note\n        If the training set contains more than 20 instances a warning is printed\n        because the computation is very expensive. This method is mostly used for\n        internal testing and simple use cases. Please refer to the\n        [Monte Carlo method][pydvl.value.least_core.montecarlo.montecarlo_least_core]\n        for practical applications.\n    The least core is the solution to the following Linear Programming problem:\n    $$\n    \\begin{array}{lll}\n    \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\\n    \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\\n    &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp;, \\forall S \\subseteq N \\\\\n    \\end{array}\n    $$\n    Where $N = \\{1, 2, \\dots, n\\}$ are the training set's indices.\n    Args:\n        u: Utility object with model, data, and scoring function\n            non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Dictionary of options that will be used to select a solver\n            and to configure it. Refer to the [cvxpy's\n            documentation](https://www.cvxpy.org/tutorial/advanced/index.html#setting-solver-options)\n            for all possible options.\n        options: (Deprecated) Dictionary of solver options. Use `solver_options`\n            instead.\n        progress: If True, shows a tqdm progress bar\n    Returns:\n        Object with the data values and the least core value.\n    \"\"\"\nn = len(u.data)\nif n &gt; 20:  # Arbitrary choice, will depend on time required, caching, etc.\nwarnings.warn(f\"Large dataset! Computation requires 2^{n} calls to model.fit()\")\n# TODO: remove this before releasing version 0.7.0\nif options:\nwarnings.warn(\nDeprecationWarning(\n\"Passing solver options as kwargs was deprecated in \"\n\"0.6.0, will \"\n\"be removed in 0.7.0. `Use solver_options` instead.\"\n)\n)\nif solver_options is None:\nsolver_options = options\nelse:\nsolver_options.update(options)\nproblem = lc_prepare_problem(u, progress=progress)\nreturn lc_solve_problem(\nproblem=problem,\nu=u,\nalgorithm=\"exact_least_core\",\nnon_negative_subsidy=non_negative_subsidy,\nsolver_options=solver_options,\n)\n</code></pre>"},{"location":"api/pydvl/value/least_core/naive/#pydvl.value.least_core.naive.lc_prepare_problem","title":"<code>lc_prepare_problem(u, progress=False)</code>","text":"<p>Prepares a linear problem with all subsets of the data Use this to separate the problem preparation from the solving with lc_solve_problem(). Useful for parallel execution of multiple experiments.</p> <p>See exact_least_core() for argument descriptions.</p> Source code in <code>src/pydvl/value/least_core/naive.py</code> <pre><code>def lc_prepare_problem(u: Utility, progress: bool = False) -&gt; LeastCoreProblem:\n\"\"\"Prepares a linear problem with all subsets of the data\n    Use this to separate the problem preparation from the solving with\n    [lc_solve_problem()][pydvl.value.least_core.common.lc_solve_problem]. Useful for\n    parallel execution of multiple experiments.\n    See [exact_least_core()][pydvl.value.least_core.naive.exact_least_core] for argument\n    descriptions.\n    \"\"\"\nn = len(u.data)\nlogger.debug(\"Building vectors and matrices for linear programming problem\")\npowerset_size = 2**n\nA_lb = np.zeros((powerset_size, n))\nlogger.debug(\"Iterating over all subsets\")\nutility_values = np.zeros(powerset_size)\nfor i, subset in enumerate(\nmaybe_progress(\npowerset(u.data.indices), progress, total=powerset_size - 1, position=0\n)\n):\nindices: NDArray[np.bool_] = np.zeros(n, dtype=bool)\nindices[list(subset)] = True\nA_lb[i, indices] = 1\nutility_values[i] = u(subset)\nreturn LeastCoreProblem(utility_values, A_lb)\n</code></pre>"},{"location":"api/pydvl/value/loo/","title":"Loo","text":""},{"location":"api/pydvl/value/loo/loo/","title":"Loo","text":""},{"location":"api/pydvl/value/loo/loo/#pydvl.value.loo.loo.compute_loo","title":"<code>compute_loo(u, *, n_jobs=1, config=ParallelConfig(), progress=True)</code>","text":"<p>Computes leave one out value:</p> \\[v(i) = u(D) - u(D \\setminus \\{i\\}) \\] PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>progress</code> <p>If True, display a progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>If True, display a progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>New in version 0.7.0</p> <p>Renamed from <code>naive_loo</code> and added parallel computation.</p> Source code in <code>src/pydvl/value/loo/loo.py</code> <pre><code>def compute_loo(\nu: Utility,\n*,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = True,\n) -&gt; ValuationResult:\nr\"\"\"Computes leave one out value:\n    $$v(i) = u(D) - u(D \\setminus \\{i\\}) $$\n    Args:\n        u: Utility object with model, data, and scoring function\n        progress: If True, display a progress bar\n        n_jobs: Number of parallel jobs to use\n        config: Object configuring parallel computation, with cluster\n            address, number of cpus, etc.\n        progress: If True, display a progress bar\n    Returns:\n        Object with the data values.\n    !!! tip \"New in version 0.7.0\"\n        Renamed from `naive_loo` and added parallel computation.\n    \"\"\"\nif len(u.data) &lt; 3:\nraise ValueError(\"Dataset must have at least 2 elements\")\nresult = ValuationResult.zeros(\nalgorithm=\"loo\",\nindices=u.data.indices,\ndata_names=u.data.data_names,\n)\nall_indices = set(u.data.indices)\ntotal_utility = u(u.data.indices)\ndef fun(idx: int) -&gt; tuple[int, float]:\nreturn idx, total_utility - u(all_indices.difference({idx}))\nmax_workers = effective_n_jobs(n_jobs, config)\nn_submitted_jobs = 2 * max_workers  # number of jobs in the queue\n# NOTE: this could be done with a simple executor.map(), but we want to\n# display a progress bar\nwith init_executor(\nmax_workers=max_workers, config=config, cancel_futures=True\n) as executor:\npending: set[Future] = set()\nindex_it = iter(u.data.indices)\npbar = tqdm(disable=not progress, total=100, unit=\"%\")\nwhile True:\npbar.n = 100 * sum(result.counts) / len(u.data)\npbar.refresh()\ncompleted, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED)\nfor future in completed:\nidx, marginal = future.result()\nresult.update(idx, marginal)\n# Ensure that we always have n_submitted_jobs running\ntry:\nfor _ in range(n_submitted_jobs - len(pending)):\npending.add(executor.submit(fun, next(index_it)))\nexcept StopIteration:\nif len(pending) == 0:\nreturn result\n</code></pre>"},{"location":"api/pydvl/value/loo/naive/","title":"Naive","text":""},{"location":"api/pydvl/value/loo/naive/#pydvl.value.loo.naive.naive_loo","title":"<code>naive_loo(u, *, progress=True, **kwargs)</code>","text":"<p>Deprecated. Use compute_loo instead.</p> Source code in <code>src/pydvl/value/loo/naive.py</code> <pre><code>@deprecated(\ntarget=compute_loo,\ndeprecated_in=\"0.7.0\",\nremove_in=\"0.8.0\",\nargs_extra=dict(n_jobs=1),\n)\ndef naive_loo(u: Utility, *, progress: bool = True, **kwargs) -&gt; ValuationResult:\n\"\"\"Deprecated. Use [compute_loo][pydvl.value.loo.compute_loo] instead.\"\"\"\npass  # type: ignore\n</code></pre>"},{"location":"api/pydvl/value/oob/","title":"Oob","text":""},{"location":"api/pydvl/value/oob/oob/","title":"Oob","text":""},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob--references","title":"References","text":"<ol> <li> <p>Kwon et al. Data-OOB: Out-of-bag Estimate as a Simple and Efficient Data Value. In: Published at ICML 2023\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob.compute_data_oob","title":"<code>compute_data_oob(u, n_est=10, max_samples=0.8, n_jobs=None, loss=None, *, progress=False)</code>","text":"<p>Computes Data out of bag values</p> <p>This implements the method described in (Kwon and Zou, 2023) <sup>1</sup>. It fits several base estimators provided through u.model through a bagging process. The point value corresponds to the average loss of estimators which were not fit on it.</p> <p>\\(w_{bj}\\in Z\\) is the number of times the j-th datum \\((x_j, y_j)\\) is selected in the b-th bootstrap dataset.</p> \\[\\psi((x_i,y_i),\\Theta_B):=\\frac{\\sum_{b=1}^{B}\\mathbb{1}(w_{bi}=0)T(y_i, \\hat{f}_b(x_i))}{\\sum_{b=1}^{B} \\mathbb{1} (w_{bi}=0)}\\] <p>With:</p> \\[ T: Y \\times Y \\rightarrow \\mathbb{R} \\] <p>T is a score function that represents the goodness of a weak learner \\(\\hat{f}_b\\) at the i-th datum \\((x_i, y_i)\\).</p> <p>There is a need to tune n_est and max_samples jointly to ensure all samples are at least 1 time oob, otherwise the result could include a nan value for that datum.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>n_est</code> <p>Number of estimator used in the bagging procedure.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>max_samples</code> <p>The fraction of samples to draw to train each base estimator.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>n_jobs</code> <p>The number of jobs to run in parallel used in the bagging procedure for both fit and predict.</p> <p> TYPE: <code>int</code> DEFAULT: <code>None</code> </p> <code>loss</code> <p>A function taking as parameters model prediction and corresponding data labels(preds, y) and returning an array of point-wise errors.</p> <p> TYPE: <code>Callable</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> Source code in <code>src/pydvl/value/oob/oob.py</code> <pre><code>def compute_data_oob(\nu: Utility,\nn_est: int = 10,\nmax_samples: float = 0.8,\nn_jobs: int = None,\nloss: Callable = None,\n*,\nprogress: bool = False,\n) -&gt; ValuationResult:\nr\"\"\"Computes Data out of bag values\n    This implements the method described in (Kwon and Zou, 2023) &lt;sup&gt;&lt;a href=\"kwon_data_2023\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    It fits several base estimators provided through u.model through a bagging process. The point value corresponds to the average loss of estimators which were not fit on it.\n    $w_{bj}\\in Z$ is the number of times the j-th datum $(x_j, y_j)$ is selected in the b-th bootstrap dataset.\n    $$\\psi((x_i,y_i),\\Theta_B):=\\frac{\\sum_{b=1}^{B}\\mathbb{1}(w_{bi}=0)T(y_i, \\hat{f}_b(x_i))}{\\sum_{b=1}^{B}\n    \\mathbb{1}\n    (w_{bi}=0)}$$\n    With:\n    $$\n    T: Y \\times Y\n    \\rightarrow \\mathbb{R}\n    $$\n    T is a score function that represents the goodness of a weak learner $\\hat{f}_b$ at the i-th datum $(x_i, y_i)$.\n    There is a need to tune n_est and max_samples jointly to ensure all samples are at least 1 time oob, otherwise the result could include a nan value for that datum.\n    Args:\n        u: Utility object with model, data, and scoring function.\n        n_est: Number of estimator used in the bagging procedure.\n        max_samples: The fraction of samples to draw to train each base estimator.\n        n_jobs: The number of jobs to run in parallel used in the bagging\n            procedure for both fit and predict.\n        loss: A function taking as parameters model prediction and corresponding\n            data labels(preds, y) and returning an array of point-wise errors.\n        progress: If True, display a progress bar.\n    Returns:\n        Object with the data values.\n    \"\"\"\nresult: ValuationResult[np.int_, np.float_] = ValuationResult.empty(\nalgorithm=\"data_oob\", indices=u.data.indices, data_names=u.data.data_names\n)\nif is_classifier(u.model):\nbag = BaggingClassifier(\nu.model, n_estimators=n_est, max_samples=max_samples, n_jobs=n_jobs\n)\nif loss is None:\nloss = point_wise_accuracy\nelif is_regressor(u.model):\nbag = BaggingRegressor(\nu.model, n_estimators=n_est, max_samples=max_samples, n_jobs=n_jobs\n)\nif loss is None:\nloss = neg_l2_distance\nelse:\nraise Exception(\n\"Model has to be a classifier or a regressor in sklearn format.\"\n)\nbag.fit(u.data.x_train, u.data.y_train)\nfor est, samples in maybe_progress(\nzip(bag.estimators_, bag.estimators_samples_), progress, total=n_est\n):  # The bottleneck is the bag fitting not this part so TQDM is not very useful here\noob_idx = np.setxor1d(u.data.indices, np.unique(samples))\narray_loss = loss(\npreds=est.predict(u.data.x_train[oob_idx]), y=u.data.y_train[oob_idx]\n)\nresult += ValuationResult(\nalgorithm=\"data_oob\",\nindices=oob_idx,\nvalues=array_loss,\ncounts=np.ones_like(array_loss, dtype=u.data.indices.dtype),\n)\nreturn result\n</code></pre>"},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob.point_wise_accuracy","title":"<code>point_wise_accuracy(preds, y)</code>","text":"<p>Computes point wise accuracy</p> PARAMETER  DESCRIPTION <code>preds</code> <p>Model prediction on</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>data labels corresponding to the model predictions</p> <p> TYPE: <code>NDArray</code> </p> RETURNS DESCRIPTION <code>NDArray</code> <p>Array of point wise accuracy</p> Source code in <code>src/pydvl/value/oob/oob.py</code> <pre><code>def point_wise_accuracy(preds: NDArray, y: NDArray) -&gt; NDArray:\nr\"\"\"Computes point wise accuracy\n    Args:\n        preds: Model prediction on\n        y:  data labels corresponding to the model predictions\n    Returns:\n        Array of point wise accuracy\n    \"\"\"\nreturn np.array(preds == y, dtype=np.int_)\n</code></pre>"},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob.neg_l2_distance","title":"<code>neg_l2_distance(preds, y)</code>","text":"<p>Computes negative l2 distance between label and model prediction</p> PARAMETER  DESCRIPTION <code>preds</code> <p>Model prediction on</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>y</code> <p>data labels corresponding to the model predictions</p> <p> TYPE: <code>NDArray[T]</code> </p> RETURNS DESCRIPTION <code>NDArray[T]</code> <p>Array with point wise negative l2 distance between label and model prediction</p> Source code in <code>src/pydvl/value/oob/oob.py</code> <pre><code>def neg_l2_distance(preds: NDArray[T], y: NDArray[T]) -&gt; NDArray[T]:\nr\"\"\"Computes negative l2 distance between label and model prediction\n    Args:\n        preds: Model prediction on\n        y:  data labels corresponding to the model predictions\n    Returns:\n        Array with point wise negative l2 distance between label and model prediction\n    \"\"\"\nreturn -np.square(\nnp.array(\npreds - y,\ndtype=np.float64,\n)\n)\n</code></pre>"},{"location":"api/pydvl/value/shapley/","title":"Shapley","text":"<p>This package holds all routines for the computation of Shapley Data value. Users will want to use compute_shapley_values or compute_semivalues as interfaces to most methods defined in the modules.</p> <p>Please refer to the guide on data valuation for an overview of all methods.</p>"},{"location":"api/pydvl/value/shapley/common/","title":"Common","text":""},{"location":"api/pydvl/value/shapley/common/#pydvl.value.shapley.common.compute_shapley_values","title":"<code>compute_shapley_values(u, *, done=MaxUpdates(100), mode=ShapleyMode.TruncatedMontecarlo, n_jobs=1, seed=None, **kwargs)</code>","text":"<p>Umbrella method to compute Shapley values with any of the available algorithms.</p> <p>See [[data-valuation]] for an overview.</p> <p>The following algorithms are available. Note that the exact methods can only work with very small datasets and are thus intended only for testing. Some algorithms also accept additional arguments, please refer to the documentation of each particular method.</p> <ul> <li><code>combinatorial_exact</code>: uses the combinatorial implementation of data   Shapley. Implemented in   combinatorial_exact_shapley().</li> <li><code>combinatorial_montecarlo</code>:  uses the approximate Monte Carlo   implementation of combinatorial data Shapley. Implemented in   combinatorial_montecarlo_shapley().</li> <li><code>permutation_exact</code>: uses the permutation-based implementation of data   Shapley. Computation is not parallelized. Implemented in   permutation_exact_shapley().</li> <li><code>permutation_montecarlo</code>: uses the approximate Monte Carlo   implementation of permutation data Shapley. Accepts a   TruncationPolicy to stop   computing marginals. Implemented in   permutation_montecarlo_shapley().</li> <li><code>owen_sampling</code>: Uses the Owen continuous extension of the utility   function to the unit cube. Implemented in   owen_sampling_shapley(). This   method does not take a StoppingCriterion   but instead requires a parameter <code>q_max</code> for the number of subdivisions   of the unit interval to use for integration, and another parameter   <code>n_samples</code> for the number of subsets to sample for each \\(q\\).</li> <li><code>owen_halved</code>: Same as 'owen_sampling' but uses correlated samples in the   expectation. Implemented in   owen_sampling_shapley().   This method  requires an additional parameter <code>q_max</code> for the number of   subdivisions of the interval [0,0.5] to use for integration, and another   parameter <code>n_samples</code> for the number of subsets to sample for each \\(q\\).</li> <li><code>group_testing</code>: estimates differences of Shapley values and solves a   constraint satisfaction problem. High sample complexity, not recommended.   Implemented in group_testing_shapley(). This   method does not take a StoppingCriterion   but instead requires a parameter <code>n_samples</code> for the number of   iterations to run.</li> </ul> <p>Additionally, one can use model-specific methods:</p> <ul> <li><code>knn</code>: Exact method for K-Nearest neighbour models. Implemented in   knn_shapley().</li> </ul> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Object used to determine when to stop the computation for Monte Carlo methods. The default is to stop after 100 iterations. See the available criteria in stopping. It is possible to combine several of them using boolean operators. Some methods ignore this argument, others require specific subtypes.</p> <p> TYPE: <code>StoppingCriterion</code> DEFAULT: <code>MaxUpdates(100)</code> </p> <code>n_jobs</code> <p>Number of parallel jobs (available only to some methods)</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>Choose which shapley algorithm to use. See ShapleyMode for a list of allowed value.</p> <p> TYPE: <code>ShapleyMode</code> DEFAULT: <code>TruncatedMontecarlo</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/shapley/common.py</code> <pre><code>def compute_shapley_values(\nu: Utility,\n*,\ndone: StoppingCriterion = MaxUpdates(100),\nmode: ShapleyMode = ShapleyMode.TruncatedMontecarlo,\nn_jobs: int = 1,\nseed: Optional[Seed] = None,\n**kwargs,\n) -&gt; ValuationResult:\n\"\"\"Umbrella method to compute Shapley values with any of the available\n    algorithms.\n    See [[data-valuation]] for an overview.\n    The following algorithms are available. Note that the exact methods can only\n    work with very small datasets and are thus intended only for testing. Some\n    algorithms also accept additional arguments, please refer to the\n    documentation of each particular method.\n    - `combinatorial_exact`: uses the combinatorial implementation of data\n      Shapley. Implemented in\n      [combinatorial_exact_shapley()][pydvl.value.shapley.naive.combinatorial_exact_shapley].\n    - `combinatorial_montecarlo`:  uses the approximate Monte Carlo\n      implementation of combinatorial data Shapley. Implemented in\n      [combinatorial_montecarlo_shapley()][pydvl.value.shapley.montecarlo.combinatorial_montecarlo_shapley].\n    - `permutation_exact`: uses the permutation-based implementation of data\n      Shapley. Computation is **not parallelized**. Implemented in\n      [permutation_exact_shapley()][pydvl.value.shapley.naive.permutation_exact_shapley].\n    - `permutation_montecarlo`: uses the approximate Monte Carlo\n      implementation of permutation data Shapley. Accepts a\n      [TruncationPolicy][pydvl.value.shapley.truncated.TruncationPolicy] to stop\n      computing marginals. Implemented in\n      [permutation_montecarlo_shapley()][pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley].\n    - `owen_sampling`: Uses the Owen continuous extension of the utility\n      function to the unit cube. Implemented in\n      [owen_sampling_shapley()][pydvl.value.shapley.owen.owen_sampling_shapley]. This\n      method does not take a [StoppingCriterion][pydvl.value.stopping.StoppingCriterion]\n      but instead requires a parameter `q_max` for the number of subdivisions\n      of the unit interval to use for integration, and another parameter\n      `n_samples` for the number of subsets to sample for each $q$.\n    - `owen_halved`: Same as 'owen_sampling' but uses correlated samples in the\n      expectation. Implemented in\n      [owen_sampling_shapley()][pydvl.value.shapley.owen.owen_sampling_shapley].\n      This method  requires an additional parameter `q_max` for the number of\n      subdivisions of the interval [0,0.5] to use for integration, and another\n      parameter `n_samples` for the number of subsets to sample for each $q$.\n    - `group_testing`: estimates differences of Shapley values and solves a\n      constraint satisfaction problem. High sample complexity, not recommended.\n      Implemented in [group_testing_shapley()][pydvl.value.shapley.gt.group_testing_shapley]. This\n      method does not take a [StoppingCriterion][pydvl.value.stopping.StoppingCriterion]\n      but instead requires a parameter `n_samples` for the number of\n      iterations to run.\n    Additionally, one can use model-specific methods:\n    - `knn`: Exact method for K-Nearest neighbour models. Implemented in\n      [knn_shapley()][pydvl.value.shapley.knn.knn_shapley].\n    Args:\n        u: [Utility][pydvl.utils.utility.Utility] object with model, data, and\n            scoring function.\n        done: Object used to determine when to stop the computation for Monte\n            Carlo methods. The default is to stop after 100 iterations. See the\n            available criteria in [stopping][pydvl.value.stopping]. It is\n            possible to combine several of them using boolean operators. Some\n            methods ignore this argument, others require specific subtypes.\n        n_jobs: Number of parallel jobs (available only to some methods)\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n        mode: Choose which shapley algorithm to use. See\n            [ShapleyMode][pydvl.value.shapley.ShapleyMode] for a list of allowed\n            value.\n    Returns:\n        Object with the results.\n    \"\"\"\nprogress: bool = kwargs.pop(\"progress\", False)\nif mode not in list(ShapleyMode):\nraise ValueError(f\"Invalid value encountered in {mode=}\")\nif mode in (\nShapleyMode.PermutationMontecarlo,\nShapleyMode.ApproShapley,\nShapleyMode.TruncatedMontecarlo,\n):\ntruncation = kwargs.pop(\"truncation\", NoTruncation())\nreturn permutation_montecarlo_shapley(  # type: ignore\nu=u, done=done, truncation=truncation, n_jobs=n_jobs, seed=seed, **kwargs\n)\nelif mode == ShapleyMode.CombinatorialMontecarlo:\nreturn combinatorial_montecarlo_shapley(\nu, done=done, n_jobs=n_jobs, seed=seed, progress=progress\n)\nelif mode == ShapleyMode.CombinatorialExact:\nreturn combinatorial_exact_shapley(u, n_jobs=n_jobs, progress=progress)\nelif mode == ShapleyMode.PermutationExact:\nreturn permutation_exact_shapley(u, progress=progress)\nelif mode == ShapleyMode.Owen or mode == ShapleyMode.OwenAntithetic:\nif kwargs.get(\"n_samples\") is None:\nraise ValueError(\"n_samples cannot be None for Owen methods\")\nif kwargs.get(\"max_q\") is None:\nraise ValueError(\"Owen Sampling requires max_q for the outer integral\")\nmethod = (\nOwenAlgorithm.Standard\nif mode == ShapleyMode.Owen\nelse OwenAlgorithm.Antithetic\n)\nreturn owen_sampling_shapley(\nu,\nn_samples=int(kwargs.get(\"n_samples\", -1)),\nmax_q=int(kwargs.get(\"max_q\", -1)),\nmethod=method,\nn_jobs=n_jobs,\nseed=seed,\n)\nelif mode == ShapleyMode.KNN:\nreturn knn_shapley(u, progress=progress)\nelif mode == ShapleyMode.GroupTesting:\nn_samples = kwargs.pop(\"n_samples\")\nif n_samples is None:\nraise ValueError(\"n_samples cannot be None for Group Testing\")\nepsilon = kwargs.pop(\"epsilon\")\nif epsilon is None:\nraise ValueError(\"Group Testing requires error bound epsilon\")\ndelta = kwargs.pop(\"delta\", 0.05)\nreturn group_testing_shapley(\nu,\nepsilon=float(epsilon),\ndelta=delta,\nn_samples=int(n_samples),\nn_jobs=n_jobs,\nprogress=progress,\nseed=seed,\n**kwargs,\n)\nelse:\nraise ValueError(f\"Invalid value encountered in {mode=}\")\n</code></pre>"},{"location":"api/pydvl/value/shapley/gt/","title":"Gt","text":"<p>This module implements Group Testing for the approximation of Shapley values, as introduced in (Jia, R. et al., 2019)<sup>1</sup>. The sampling of index subsets is done in such a way that an approximation to the true Shapley values can be computed with guarantees.</p> <p>Warning</p> <p>This method is very inefficient. Potential improvements to the implementation notwithstanding, convergence seems to be very slow (in terms of evaluations of the utility required). We recommend other Monte Carlo methods instead.</p> <p>You can read more in the documentation.</p> <p>New in version 0.4.0</p>"},{"location":"api/pydvl/value/shapley/gt/#pydvl.value.shapley.gt--references","title":"References","text":"<ol> <li> <p>Jia, R. et al., 2019. Towards Efficient Data Valuation Based on the Shapley Value. In: Proceedings of the 22nd International Conference on Artificial Intelligence and Statistics, pp. 1167\u20131176. PMLR.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/gt/#pydvl.value.shapley.gt.num_samples_eps_delta","title":"<code>num_samples_eps_delta(eps, delta, n, utility_range)</code>","text":"<p>Implements the formula in Theorem 3 of (Jia, R. et al., 2019)<sup>1</sup> which gives a lower bound on the number of samples required to obtain an (\u03b5/\u221an,\u03b4/(N(N-1))-approximation to all pair-wise differences of Shapley values, wrt. \\(\\ell_2\\) norm.</p> PARAMETER  DESCRIPTION <code>eps</code> <p>\u03b5</p> <p> TYPE: <code>float</code> </p> <code>delta</code> <p>\u03b4</p> <p> TYPE: <code>float</code> </p> <code>n</code> <p>Number of data points</p> <p> TYPE: <code>int</code> </p> <code>utility_range</code> <p>Range of the Utility function</p> <p> TYPE: <code>float</code> </p> <p>Returns:     Number of samples from \\(2^{[n]}\\) guaranteeing \u03b5/\u221an-correct Shapley         pair-wise differences of values with probability 1-\u03b4/(N(N-1)).</p> <p>New in version 0.4.0</p> Source code in <code>src/pydvl/value/shapley/gt.py</code> <pre><code>def num_samples_eps_delta(\neps: float, delta: float, n: int, utility_range: float\n) -&gt; int:\nr\"\"\"Implements the formula in Theorem 3 of (Jia, R. et al., 2019)&lt;sup&gt;&lt;a href=\"#jia_efficient_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;\n    which gives a lower bound on the number of samples required to obtain an\n    (\u03b5/\u221an,\u03b4/(N(N-1))-approximation to all pair-wise differences of Shapley\n    values, wrt. $\\ell_2$ norm.\n    Args:\n        eps: \u03b5\n        delta: \u03b4\n        n: Number of data points\n        utility_range: Range of the [Utility][pydvl.utils.utility.Utility] function\n    Returns:\n        Number of samples from $2^{[n]}$ guaranteeing \u03b5/\u221an-correct Shapley\n            pair-wise differences of values with probability 1-\u03b4/(N(N-1)).\n    !!! tip \"New in version 0.4.0\"\n    \"\"\"\nconstants = _constants(n=n, epsilon=eps, delta=delta, utility_range=utility_range)\nreturn int(constants.T)\n</code></pre>"},{"location":"api/pydvl/value/shapley/gt/#pydvl.value.shapley.gt.group_testing_shapley","title":"<code>group_testing_shapley(u, n_samples, epsilon, delta, *, n_jobs=1, config=ParallelConfig(), progress=False, seed=None, **options)</code>","text":"<p>Implements group testing for approximation of Shapley values as described in (Jia, R. et al., 2019)<sup>1</sup>.</p> <p>Warning</p> <p>This method is very inefficient. It requires several orders of magnitude more evaluations of the utility than others in montecarlo. It also uses several intermediate objects like the results from the runners and the constraint matrices which can become rather large.</p> <p>By picking a specific distribution over subsets, the differences in Shapley values can be approximated with a Monte Carlo sum. These are then used to solve for the individual values in a feasibility problem.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_samples</code> <p>Number of tests to perform. Use num_samples_eps_delta to estimate this.</p> <p> TYPE: <code>int</code> </p> <code>epsilon</code> <p>From the (\u03b5,\u03b4) sample bound. Use the same as for the estimation of <code>n_iterations</code>.</p> <p> TYPE: <code>float</code> </p> <code>delta</code> <p>From the (\u03b5,\u03b4) sample bound. Use the same as for the estimation of <code>n_iterations</code>.</p> <p> TYPE: <code>float</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use. Each worker performs a chunk of all tests (i.e. utility evaluations).</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>options</code> <p>Additional options to pass to cvxpy.Problem.solve(). E.g. to change the solver (which defaults to <code>cvxpy.SCS</code>) pass <code>solver=cvxpy.CVXOPT</code>.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>New in version 0.4.0</p> <p>Changed in version 0.5.0</p> <p>Changed the solver to cvxpy instead of scipy's linprog. Added the ability to pass arbitrary options to it.</p> Source code in <code>src/pydvl/value/shapley/gt.py</code> <pre><code>def group_testing_shapley(\nu: Utility,\nn_samples: int,\nepsilon: float,\ndelta: float,\n*,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\nseed: Optional[Seed] = None,\n**options,\n) -&gt; ValuationResult:\n\"\"\"Implements group testing for approximation of Shapley values as described\n    in (Jia, R. et al., 2019)&lt;sup&gt;&lt;a href=\"#jia_efficient_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    !!! Warning\n        This method is very inefficient. It requires several orders of magnitude\n        more evaluations of the utility than others in\n        [montecarlo][pydvl.value.shapley.montecarlo]. It also uses several intermediate\n        objects like the results from the runners and the constraint matrices\n        which can become rather large.\n    By picking a specific distribution over subsets, the differences in Shapley\n    values can be approximated with a Monte Carlo sum. These are then used to\n    solve for the individual values in a feasibility problem.\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_samples: Number of tests to perform. Use\n            [num_samples_eps_delta][pydvl.value.shapley.gt.num_samples_eps_delta]\n            to estimate this.\n        epsilon: From the (\u03b5,\u03b4) sample bound. Use the same as for the\n            estimation of `n_iterations`.\n        delta: From the (\u03b5,\u03b4) sample bound. Use the same as for the\n            estimation of `n_iterations`.\n        n_jobs: Number of parallel jobs to use. Each worker performs a chunk\n            of all tests (i.e. utility evaluations).\n        config: Object configuring parallel computation, with cluster\n            address, number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n        options: Additional options to pass to\n            [cvxpy.Problem.solve()](https://www.cvxpy.org/tutorial/advanced/index.html#solve-method-options).\n            E.g. to change the solver (which defaults to `cvxpy.SCS`) pass\n            `solver=cvxpy.CVXOPT`.\n    Returns:\n        Object with the data values.\n    !!! tip \"New in version 0.4.0\"\n    !!! tip \"Changed in version 0.5.0\"\n        Changed the solver to cvxpy instead of scipy's linprog. Added the ability\n        to pass arbitrary options to it.\n    \"\"\"\nn = len(u.data.indices)\nconst = _constants(\nn=n,\nepsilon=epsilon,\ndelta=delta,\nutility_range=u.score_range.max() - u.score_range.min(),\n)\nT = n_samples\nif T &lt; const.T:\nlog.warning(\nf\"n_samples of {T} are below the required {const.T} for the \"\nf\"\u03b5={epsilon:.02f} guarantee at \u03b4={1 - delta:.02f} probability\"\n)\nsamples_per_job = max(1, n_samples // effective_n_jobs(n_jobs, config))\ndef reducer(\nresults_it: Iterable[Tuple[NDArray, NDArray]]\n) -&gt; Tuple[NDArray, NDArray]:\nreturn np.concatenate(list(x[0] for x in results_it)).astype(\nnp.float_\n), np.concatenate(list(x[1] for x in results_it)).astype(np.int_)\nseed_sequence = ensure_seed_sequence(seed)\nmap_reduce_seed_sequence, cvxpy_seed = tuple(seed_sequence.spawn(2))\nmap_reduce_job: MapReduceJob[Utility, Tuple[NDArray, NDArray]] = MapReduceJob(\nu,\nmap_func=_group_testing_shapley,\nreduce_func=reducer,\nmap_kwargs=dict(n_samples=samples_per_job, progress=progress),\nconfig=config,\nn_jobs=n_jobs,\n)\nuu, betas = map_reduce_job(seed=map_reduce_seed_sequence)\n# Matrix of estimated differences. See Eqs. (3) and (4) in the paper.\nC = np.zeros(shape=(n, n))\nfor i in range(n):\nfor j in range(i + 1, n):\nC[i, j] = np.dot(uu, betas[:, i] - betas[:, j])\nC *= const.Z / T\ntotal_utility = u(u.data.indices)\n###########################################################################\n# Solution of the constraint problem with cvxpy\nv = cp.Variable(n)\nconstraints = [cp.sum(v) == total_utility]\nfor i in range(n):\nfor j in range(i + 1, n):\nconstraints.append(v[i] - v[j] &lt;= epsilon + C[i, j])\nconstraints.append(v[j] - v[i] &lt;= epsilon - C[i, j])\nproblem = cp.Problem(cp.Minimize(0), constraints)\nsolver = options.pop(\"solver\", cp.SCS)\nproblem.solve(solver=solver, **options)\nif problem.status != \"optimal\":\nlog.warning(f\"cvxpy returned status {problem.status}\")\nvalues = (\nnp.nan * np.ones_like(u.data.indices)\nif not hasattr(v.value, \"__len__\")\nelse v.value\n)\nstatus = Status.Failed\nelse:\nvalues = v.value\nstatus = Status.Converged\nreturn ValuationResult(\nalgorithm=\"group_testing_shapley\",\nstatus=status,\nvalues=values,\ndata_names=u.data.data_names,\nsolver_status=problem.status,\n)\n</code></pre>"},{"location":"api/pydvl/value/shapley/knn/","title":"Knn","text":"<p>This module contains Shapley computations for K-Nearest Neighbours.</p> <p>Todo</p> <p>Implement approximate KNN computation for sublinear complexity</p>"},{"location":"api/pydvl/value/shapley/knn/#pydvl.value.shapley.knn--references","title":"References","text":"<ol> <li> <p>Jia, R. et al., 2019. Efficient Task-Specific Data Valuation for Nearest Neighbor Algorithms. In: Proceedings of the VLDB Endowment, Vol. 12, No. 11, pp. 1610\u20131623.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/knn/#pydvl.value.shapley.knn.knn_shapley","title":"<code>knn_shapley(u, *, progress=True)</code>","text":"<p>Computes exact Shapley values for a KNN classifier.</p> <p>This implements the method described in (Jia, R. et al., 2019)<sup>1</sup>. It exploits the local structure of K-Nearest Neighbours to reduce the number of calls to the utility function to a constant number per index, thus reducing computation time to \\(O(n)\\).</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility with a KNN model to extract parameters from. The object will not be modified nor used other than to call get_params()</p> <p> TYPE: <code>Utility</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> RAISES DESCRIPTION <code>TypeError</code> <p>If the model in the utility is not a sklearn.neighbors.KNeighborsClassifier.</p> <p>New in version 0.1.0</p> Source code in <code>src/pydvl/value/shapley/knn.py</code> <pre><code>def knn_shapley(u: Utility, *, progress: bool = True) -&gt; ValuationResult:\n\"\"\"Computes exact Shapley values for a KNN classifier.\n    This implements the method described in (Jia, R. et al., 2019)&lt;sup&gt;&lt;a href=\"#jia_efficient_2019a\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    It exploits the local structure of K-Nearest Neighbours to reduce the number\n    of calls to the utility function to a constant number per index, thus\n    reducing computation time to $O(n)$.\n    Args:\n        u: Utility with a KNN model to extract parameters from. The object\n            will not be modified nor used other than to call [get_params()](\n            &lt;https://scikit-learn.org/stable/modules/generated/sklearn.base.BaseEstimator.html#sklearn.base.BaseEstimator.get_params&gt;)\n        progress: Whether to display a progress bar.\n    Returns:\n        Object with the data values.\n    Raises:\n        TypeError: If the model in the utility is not a\n            [sklearn.neighbors.KNeighborsClassifier][].\n    !!! tip \"New in version 0.1.0\"\n    \"\"\"\nif not isinstance(u.model, KNeighborsClassifier):\nraise TypeError(\"KNN Shapley requires a K-Nearest Neighbours model\")\ndefaults: Dict[str, Union[int, str]] = {\n\"algorithm\": \"ball_tree\" if u.data.dim &gt;= 20 else \"kd_tree\",\n\"metric\": \"minkowski\",\n\"p\": 2,\n}\ndefaults.update(u.model.get_params())\n# HACK: NearestNeighbors doesn't support this. There will be more...\ndel defaults[\"weights\"]\nn_neighbors: int = int(defaults[\"n_neighbors\"])\ndefaults[\"n_neighbors\"] = len(u.data)  # We want all training points sorted\nassert n_neighbors &lt; len(u.data)\n# assert data.target_dim == 1\nnns = NearestNeighbors(**defaults).fit(u.data.x_train)\n# closest to farthest\n_, indices = nns.kneighbors(u.data.x_test)\nvalues: NDArray[np.float_] = np.zeros_like(u.data.indices, dtype=np.float_)\nn = len(u.data)\nyt = u.data.y_train\niterator = enumerate(zip(u.data.y_test, indices), start=1)\nfor j, (y, ii) in maybe_progress(iterator, progress):\nvalue_at_x = int(yt[ii[-1]] == y) / n\nvalues[ii[-1]] += (value_at_x - values[ii[-1]]) / j\nfor i in range(n - 2, n_neighbors, -1):  # farthest to closest\nvalue_at_x = (\nvalues[ii[i + 1]] + (int(yt[ii[i]] == y) - int(yt[ii[i + 1]] == y)) / i\n)\nvalues[ii[i]] += (value_at_x - values[ii[i]]) / j\nfor i in range(n_neighbors, -1, -1):  # farthest to closest\nvalue_at_x = (\nvalues[ii[i + 1]]\n+ (int(yt[ii[i]] == y) - int(yt[ii[i + 1]] == y)) / n_neighbors\n)\nvalues[ii[i]] += (value_at_x - values[ii[i]]) / j\nreturn ValuationResult(\nalgorithm=\"knn_shapley\",\nstatus=Status.Converged,\nvalues=values,\ndata_names=u.data.data_names,\n)\n</code></pre>"},{"location":"api/pydvl/value/shapley/montecarlo/","title":"Montecarlo","text":"<p>Monte Carlo approximations to Shapley Data values.</p> <p>Warning</p> <p>You probably want to use the common interface provided by compute_shapley_values() instead of directly using the functions in this module.</p> <p>Because exact computation of Shapley values requires \\(\\mathcal{O}(2^n)\\) re-trainings of the model, several Monte Carlo approximations are available. The first two sample from the powerset of the training data directly: combinatorial_montecarlo_shapley() and owen_sampling_shapley(). The latter uses a reformulation in terms of a continuous extension of the utility.</p> <p>Alternatively, employing another reformulation of the expression above as a sum over permutations, one has the implementation in permutation_montecarlo_shapley(), or using an early stopping strategy to reduce computation truncated_montecarlo_shapley().</p> <p>Also see</p> <p>It is also possible to use group_testing_shapley() to reduce the number of evaluations of the utility. The method is however typically outperformed by others in this module.</p> <p>Also see</p> <p>Additionally, you can consider grouping your data points using GroupedDataset and computing the values of the groups instead. This is not to be confused with \"group testing\" as implemented in group_testing_shapley(): any of the algorithms mentioned above, including Group Testing, can work to valuate groups of samples as units.</p>"},{"location":"api/pydvl/value/shapley/montecarlo/#pydvl.value.shapley.montecarlo--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/montecarlo/#pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley","title":"<code>permutation_montecarlo_shapley(u, done, *, truncation=NoTruncation(), n_jobs=1, config=ParallelConfig(), progress=False, seed=None)</code>","text":"<p>Computes an approximate Shapley value by sampling independent permutations of the index set, approximating the sum:</p> \\[ v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} \\tilde{w}( | \\sigma_{:i} | )[u(\\sigma_{:i} \\cup \\{i\\}) \u2212 u(\\sigma_{:i})], \\] <p>where \\(\\sigma_{:i}\\) denotes the set of indices in permutation sigma before the position where \\(i\\) appears (see [[data-valuation]] for details).</p> <p>This implements the method described in (Ghorbani and Zou, 2019)<sup>1</sup> with a double stopping criterion.</p> <p>.. todo::    Think of how to add Robin-Gelman or some other more principled stopping    criterion.</p> <p>Instead of naively implementing the expectation, we sequentially add points to coalitions from a permutation and incrementally compute marginal utilities. We stop computing marginals for a given permutation based on a TruncationPolicy. (Ghorbani and Zou, 2019)<sup>1</sup> mention two policies: one that stops after a certain fraction of marginals are computed, implemented in FixedTruncation, and one that stops if the last computed utility (\"score\") is close to the total utility using the standard deviation of the utility as a measure of proximity, implemented in BootstrapTruncation.</p> <p>We keep sampling permutations and updating all shapley values until the StoppingCriterion returns <code>True</code>.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>function checking whether computation must stop.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>truncation</code> <p>An optional callable which decides whether to interrupt processing a permutation and set all subsequent marginals to zero. Typically used to stop computation when the marginal is small.</p> <p> TYPE: <code>TruncationPolicy</code> DEFAULT: <code>NoTruncation()</code> </p> <code>n_jobs</code> <p>number of jobs across which to distribute the computation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Seed</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> Source code in <code>src/pydvl/value/shapley/montecarlo.py</code> <pre><code>@deprecated(\ntarget=True,\ndeprecated_in=\"0.7.0\",\nremove_in=\"0.8.0\",\nargs_mapping=dict(\ncoordinator_update_period=None, worker_update_period=None, progress=None\n),\n)\ndef permutation_montecarlo_shapley(\nu: Utility,\ndone: StoppingCriterion,\n*,\ntruncation: TruncationPolicy = NoTruncation(),\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\nseed: Seed = None,\n) -&gt; ValuationResult:\nr\"\"\"Computes an approximate Shapley value by sampling independent\n    permutations of the index set, approximating the sum:\n    $$\n    v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)}\n    \\tilde{w}( | \\sigma_{:i} | )[u(\\sigma_{:i} \\cup \\{i\\}) \u2212 u(\\sigma_{:i})],\n    $$\n    where $\\sigma_{:i}$ denotes the set of indices in permutation sigma before\n    the position where $i$ appears (see [[data-valuation]] for details).\n    This implements the method described in (Ghorbani and Zou, 2019)&lt;sup&gt;&lt;a href=\"#ghorbani_data_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;\n    with a double stopping criterion.\n    .. todo::\n       Think of how to add Robin-Gelman or some other more principled stopping\n       criterion.\n    Instead of naively implementing the expectation, we sequentially add points\n    to coalitions from a permutation and incrementally compute marginal utilities.\n    We stop computing marginals for a given permutation based on a\n    [TruncationPolicy][pydvl.value.shapley.truncated.TruncationPolicy].\n    (Ghorbani and Zou, 2019)&lt;sup&gt;&lt;a href=\"#ghorbani_data_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;\n    mention two policies: one that stops after a certain\n    fraction of marginals are computed, implemented in\n    [FixedTruncation][pydvl.value.shapley.truncated.FixedTruncation],\n    and one that stops if the last computed utility (\"score\") is close to the\n    total utility using the standard deviation of the utility as a measure of\n    proximity, implemented in\n    [BootstrapTruncation][pydvl.value.shapley.truncated.BootstrapTruncation].\n    We keep sampling permutations and updating all shapley values\n    until the [StoppingCriterion][pydvl.value.stopping.StoppingCriterion] returns\n    `True`.\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: function checking whether computation must stop.\n        truncation: An optional callable which decides whether to interrupt\n            processing a permutation and set all subsequent marginals to\n            zero. Typically used to stop computation when the marginal is small.\n        n_jobs: number of jobs across which to distribute the computation.\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        progress: Whether to display a progress bar.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n    Returns:\n        Object with the data values.\n    \"\"\"\nalgorithm = \"permutation_montecarlo_shapley\"\nparallel_backend = init_parallel_backend(config)\nu = parallel_backend.put(u)\nmax_workers = effective_n_jobs(n_jobs, config)\nn_submitted_jobs = 2 * max_workers  # number of jobs in the executor's queue\nseed_sequence = ensure_seed_sequence(seed)\nresult = ValuationResult.zeros(algorithm=algorithm)\npbar = tqdm(disable=not progress, total=100, unit=\"%\")\nwith init_executor(\nmax_workers=max_workers, config=config, cancel_futures=CancellationPolicy.ALL\n) as executor:\npending: set[Future] = set()\nwhile True:\npbar.n = 100 * done.completion()\npbar.refresh()\ncompleted, pending = wait(\npending, timeout=config.wait_timeout, return_when=FIRST_COMPLETED\n)\nfor future in completed:\nresult += future.result()\n# we could check outside the loop, but that means more\n# submissions if the stopping criterion is unstable\nif done(result):\nreturn result\n# Ensure that we always have n_submitted_jobs in the queue or running\nn_remaining_slots = n_submitted_jobs - len(pending)\nseeds = seed_sequence.spawn(n_remaining_slots)\nfor i in range(n_remaining_slots):\nfuture = executor.submit(\n_permutation_montecarlo_one_step,\nu,\ntruncation,\nalgorithm,\nseed=seeds[i],\n)\npending.add(future)\n</code></pre>"},{"location":"api/pydvl/value/shapley/montecarlo/#pydvl.value.shapley.montecarlo.combinatorial_montecarlo_shapley","title":"<code>combinatorial_montecarlo_shapley(u, done, *, n_jobs=1, config=ParallelConfig(), progress=False, seed=None)</code>","text":"<p>Computes an approximate Shapley value using the combinatorial definition:</p> \\[v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}} \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)]\\] <p>This consists of randomly sampling subsets of the power set of the training indices in u.data, and computing their marginal utilities. See Data valuation for details.</p> <p>Note that because sampling is done with replacement, the approximation is poor even for \\(2^{m}\\) subsets with \\(m&gt;n\\), even though there are \\(2^{n-1}\\) subsets for each \\(i\\). Prefer permutation_montecarlo_shapley().</p> <p>Parallelization is done by splitting the set of indices across processes and computing the sum over subsets \\(S \\subseteq N \\setminus \\{i\\}\\) separately.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion for the computation.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>n_jobs</code> <p>number of parallel jobs across which to distribute the computation. Each worker receives a chunk of indices</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> Source code in <code>src/pydvl/value/shapley/montecarlo.py</code> <pre><code>def combinatorial_montecarlo_shapley(\nu: Utility,\ndone: StoppingCriterion,\n*,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\nseed: Optional[Seed] = None,\n) -&gt; ValuationResult:\nr\"\"\"Computes an approximate Shapley value using the combinatorial\n    definition:\n    $$v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}}\n    \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)]$$\n    This consists of randomly sampling subsets of the power set of the training\n    indices in [u.data][pydvl.utils.utility.Utility], and computing their\n    marginal utilities. See [Data valuation][computing-data-values] for details.\n    Note that because sampling is done with replacement, the approximation is\n    poor even for $2^{m}$ subsets with $m&gt;n$, even though there are $2^{n-1}$\n    subsets for each $i$. Prefer\n    [permutation_montecarlo_shapley()][pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley].\n    Parallelization is done by splitting the set of indices across processes and\n    computing the sum over subsets $S \\subseteq N \\setminus \\{i\\}$ separately.\n    Args:\n        u: Utility object with model, data, and scoring function\n        done: Stopping criterion for the computation.\n        n_jobs: number of parallel jobs across which to distribute the\n            computation. Each worker receives a chunk of\n            [indices][pydvl.utils.dataset.Dataset.indices]\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n    Returns:\n        Object with the data values.\n    \"\"\"\nmap_reduce_job: MapReduceJob[NDArray, ValuationResult] = MapReduceJob(\nu.data.indices,\nmap_func=_combinatorial_montecarlo_shapley,\nreduce_func=lambda results: reduce(operator.add, results),\nmap_kwargs=dict(u=u, done=done, progress=progress),\nn_jobs=n_jobs,\nconfig=config,\n)\nreturn map_reduce_job(seed=seed)\n</code></pre>"},{"location":"api/pydvl/value/shapley/naive/","title":"Naive","text":""},{"location":"api/pydvl/value/shapley/naive/#pydvl.value.shapley.naive.permutation_exact_shapley","title":"<code>permutation_exact_shapley(u, *, progress=True)</code>","text":"<p>Computes the exact Shapley value using the formulation with permutations:</p> \\[v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} [u(\\sigma_{i-1} \\cup {i}) \u2212 u(\\sigma_{i})].\\] <p>See Data valuation for details.</p> <p>When the length of the training set is &gt; 10 this prints a warning since the computation becomes too expensive. Used mostly for internal testing and simple use cases. Please refer to the Monte Carlo approximations for practical applications.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> Source code in <code>src/pydvl/value/shapley/naive.py</code> <pre><code>def permutation_exact_shapley(u: Utility, *, progress: bool = True) -&gt; ValuationResult:\nr\"\"\"Computes the exact Shapley value using the formulation with permutations:\n    $$v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} [u(\\sigma_{i-1} \\cup {i}) \u2212 u(\\sigma_{i})].$$\n    See [Data valuation][computing-data-values] for details.\n    When the length of the training set is &gt; 10 this prints a warning since the\n    computation becomes too expensive. Used mostly for internal testing and\n    simple use cases. Please refer to the [Monte Carlo\n    approximations][pydvl.value.shapley.montecarlo] for practical applications.\n    Args:\n        u: Utility object with model, data, and scoring function\n        progress: Whether to display progress bars for each job.\n    Returns:\n        Object with the data values.\n    \"\"\"\nn = len(u.data)\n# Note that the cache in utility saves most of the refitting because we\n# use frozenset for the input.\nif n &gt; 10:\nwarnings.warn(\nf\"Large dataset! Computation requires {n}! calls to utility()\",\nRuntimeWarning,\n)\nvalues = np.zeros(n)\nfor p in maybe_progress(\npermutations(u.data.indices),\nprogress,\ndesc=\"Permutation\",\ntotal=math.factorial(n),\n):\nfor i, idx in enumerate(p):\nvalues[idx] += u(p[: i + 1]) - u(p[:i])\nvalues /= math.factorial(n)\nreturn ValuationResult(\nalgorithm=\"permutation_exact_shapley\",\nstatus=Status.Converged,\nvalues=values,\ndata_names=u.data.data_names,\n)\n</code></pre>"},{"location":"api/pydvl/value/shapley/naive/#pydvl.value.shapley.naive.combinatorial_exact_shapley","title":"<code>combinatorial_exact_shapley(u, *, n_jobs=1, config=ParallelConfig(), progress=False)</code>","text":"<p>Computes the exact Shapley value using the combinatorial definition.</p> \\[v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}} \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)].\\] <p>See Data valuation for details.</p> <p>Note</p> <p>If the length of the training set is &gt; n_jobs*20 this prints a warning because the computation is very expensive. Used mostly for internal testing and simple use cases. Please refer to the Monte Carlo approximations for practical applications.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> Source code in <code>src/pydvl/value/shapley/naive.py</code> <pre><code>def combinatorial_exact_shapley(\nu: Utility,\n*,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\n) -&gt; ValuationResult:\nr\"\"\"Computes the exact Shapley value using the combinatorial definition.\n    $$v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}} \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)].$$\n    See [Data valuation][computing-data-values] for details.\n    !!! Note\n        If the length of the training set is &gt; n_jobs*20 this prints a warning\n        because the computation is very expensive. Used mostly for internal testing\n        and simple use cases. Please refer to the\n        [Monte Carlo][pydvl.value.shapley.montecarlo] approximations for practical\n        applications.\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_jobs: Number of parallel jobs to use\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n    Returns:\n        Object with the data values.\n    \"\"\"\n# Arbitrary choice, will depend on time required, caching, etc.\nif len(u.data) // n_jobs &gt; 20:\nwarnings.warn(\nf\"Large dataset! Computation requires 2^{len(u.data)} calls to model.fit()\"\n)\ndef reduce_fun(results: List[NDArray]) -&gt; NDArray:\nreturn np.array(results).sum(axis=0)  # type: ignore\nmap_reduce_job: MapReduceJob[NDArray, NDArray] = MapReduceJob(\nu.data.indices,\nmap_func=_combinatorial_exact_shapley,\nmap_kwargs=dict(u=u, progress=progress),\nreduce_func=reduce_fun,\nn_jobs=n_jobs,\nconfig=config,\n)\nvalues = map_reduce_job()\nreturn ValuationResult(\nalgorithm=\"combinatorial_exact_shapley\",\nstatus=Status.Converged,\nvalues=values,\ndata_names=u.data.data_names,\n)\n</code></pre>"},{"location":"api/pydvl/value/shapley/owen/","title":"Owen","text":""},{"location":"api/pydvl/value/shapley/owen/#pydvl.value.shapley.owen--references","title":"References","text":"<ol> <li> <p>Okhrati, R., Lipani, A., 2021. A Multilinear Sampling Algorithm to Estimate Shapley Values. In: 2020 25th International Conference on Pattern Recognition (ICPR), pp. 7992\u20137999. IEEE.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/owen/#pydvl.value.shapley.owen.owen_sampling_shapley","title":"<code>owen_sampling_shapley(u, n_samples, max_q, *, method=OwenAlgorithm.Standard, n_jobs=1, config=ParallelConfig(), progress=False, seed=None)</code>","text":"<p>Owen sampling of Shapley values as described in (Okhrati and Lipani, 2021)<sup>1</sup>.</p> <p>This function computes a Monte Carlo approximation to</p> \\[v_u(i) = \\int_0^1 \\mathbb{E}_{S \\sim P_q(D_{\\backslash \\{i\\}})} [u(S \\cup \\{i\\}) - u(S)]\\] <p>using one of two methods. The first one, selected with the argument <code>mode = OwenAlgorithm.Standard</code>, approximates the integral with:</p> \\[\\hat{v}_u(i) = \\frac{1}{Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m \\cup \\{i\\}) - u(S^{(q_j)}_m)],\\] <p>where \\(q_j = \\frac{j}{Q} \\in [0,1]\\) and the sets \\(S^{(q_j)}\\) are such that a sample \\(x \\in S^{(q_j)}\\) if a draw from a \\(Ber(q_j)\\) distribution is 1.</p> <p>The second method, selected with the argument <code>mode = OwenAlgorithm.Antithetic</code>, uses correlated samples in the inner sum to reduce the variance:</p> \\[\\hat{v}_u(i) = \\frac{1}{2 Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m \\cup \\{i\\}) - u(S^{(q_j)}_m) + u((S^{(q_j)}_m)^c \\cup \\{i\\}) - u((S^{( q_j)}_m)^c)],\\] <p>where now \\(q_j = \\frac{j}{2Q} \\in [0,\\frac{1}{2}]\\), and \\(S^c\\) is the complement of \\(S\\).</p> <p>Note</p> <p>The outer integration could be done instead with a quadrature rule.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object holding data, model and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>n_samples</code> <p>Numer of sets to sample for each value of q</p> <p> TYPE: <code>int</code> </p> <code>max_q</code> <p>Number of subdivisions for q \u2208 [0,1] (the element sampling probability) used to approximate the outer integral.</p> <p> TYPE: <code>int</code> </p> <code>method</code> <p>Selects the algorithm to use, see the description. Either OwenAlgorithm.Full for \\(q \\in [0,1]\\) or OwenAlgorithm.Halved for \\(q \\in [0,0.5]\\) and correlated samples</p> <p> TYPE: <code>OwenAlgorithm</code> DEFAULT: <code>Standard</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use. Each worker receives a chunk of the total of <code>max_q</code> values for q.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>New in version 0.3.0</p> <p>Changed in version 0.5.0</p> <p>Support for parallel computation and enable antithetic sampling.</p> Source code in <code>src/pydvl/value/shapley/owen.py</code> <pre><code>def owen_sampling_shapley(\nu: Utility,\nn_samples: int,\nmax_q: int,\n*,\nmethod: OwenAlgorithm = OwenAlgorithm.Standard,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\nseed: Optional[Seed] = None\n) -&gt; ValuationResult:\nr\"\"\"Owen sampling of Shapley values as described in\n    (Okhrati and Lipani, 2021)&lt;sup&gt;&lt;a href=\"#okhrati_multilinear_2021\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    This function computes a Monte Carlo approximation to\n    $$v_u(i) = \\int_0^1 \\mathbb{E}_{S \\sim P_q(D_{\\backslash \\{i\\}})}\n    [u(S \\cup \\{i\\}) - u(S)]$$\n    using one of two methods. The first one, selected with the argument ``mode =\n    OwenAlgorithm.Standard``, approximates the integral with:\n    $$\\hat{v}_u(i) = \\frac{1}{Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m\n    \\cup \\{i\\}) - u(S^{(q_j)}_m)],$$\n    where $q_j = \\frac{j}{Q} \\in [0,1]$ and the sets $S^{(q_j)}$ are such that a\n    sample $x \\in S^{(q_j)}$ if a draw from a $Ber(q_j)$ distribution is 1.\n    The second method, selected with the argument ``mode =\n    OwenAlgorithm.Antithetic``, uses correlated samples in the inner sum to\n    reduce the variance:\n    $$\\hat{v}_u(i) = \\frac{1}{2 Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m\n    \\cup \\{i\\}) - u(S^{(q_j)}_m) + u((S^{(q_j)}_m)^c \\cup \\{i\\}) - u((S^{(\n    q_j)}_m)^c)],$$\n    where now $q_j = \\frac{j}{2Q} \\in [0,\\frac{1}{2}]$, and $S^c$ is the\n    complement of $S$.\n    !!! Note\n        The outer integration could be done instead with a quadrature rule.\n    Args:\n        u: [Utility][pydvl.utils.utility.Utility] object holding data, model\n            and scoring function.\n        n_samples: Numer of sets to sample for each value of q\n        max_q: Number of subdivisions for q \u2208 [0,1] (the element sampling\n            probability) used to approximate the outer integral.\n        method: Selects the algorithm to use, see the description. Either\n            [OwenAlgorithm.Full][pydvl.value.shapley.owen.OwenAlgorithm] for\n            $q \\in [0,1]$ or\n            [OwenAlgorithm.Halved][pydvl.value.shapley.owen.OwenAlgorithm] for\n            $q \\in [0,0.5]$ and correlated samples\n        n_jobs: Number of parallel jobs to use. Each worker receives a chunk\n            of the total of `max_q` values for q.\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n    Returns:\n        Object with the data values.\n    !!! tip \"New in version 0.3.0\"\n    !!! tip \"Changed in version 0.5.0\"\n        Support for parallel computation and enable antithetic sampling.\n    \"\"\"\nmap_reduce_job: MapReduceJob[NDArray, ValuationResult] = MapReduceJob(\nu.data.indices,\nmap_func=_owen_sampling_shapley,\nreduce_func=lambda results: reduce(operator.add, results),\nmap_kwargs=dict(\nu=u,\nmethod=OwenAlgorithm(method),\nn_samples=n_samples,\nmax_q=max_q,\nprogress=progress,\n),\nn_jobs=n_jobs,\nconfig=config,\n)\nreturn map_reduce_job(seed=seed)\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/","title":"Truncated","text":""},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.TruncationPolicy","title":"<code>TruncationPolicy()</code>","text":"<p>             Bases: <code>ABC</code></p> <p>A policy for deciding whether to stop computing marginals in a permutation.</p> <p>Statistics are kept on the number of calls and truncations as <code>n_calls</code> and <code>n_truncations</code> respectively.</p> ATTRIBUTE DESCRIPTION <code>n_calls</code> <p>Number of calls to the policy.</p> <p> TYPE: <code>int</code> </p> <code>n_truncations</code> <p>Number of truncations made by the policy.</p> <p> TYPE: <code>int</code> </p> <p>Todo</p> <p>Because the policy objects are copied to the workers, the statistics are not accessible from the coordinating process. We need to add methods for this.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self):\nself.n_calls: int = 0\nself.n_truncations: int = 0\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.TruncationPolicy.reset","title":"<code>reset()</code>  <code>abstractmethod</code>","text":"<p>Reset the policy to a state ready for a new permutation.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>@abc.abstractmethod\ndef reset(self):\n\"\"\"Reset the policy to a state ready for a new permutation.\"\"\"\n...\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.TruncationPolicy.__call__","title":"<code>__call__(idx, score)</code>","text":"<p>Check whether the computation should be interrupted.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Position in the permutation currently being computed.</p> <p> TYPE: <code>int</code> </p> <code>score</code> <p>Last utility computed.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>bool</code> <p><code>True</code> if the computation should be interrupted.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __call__(self, idx: int, score: float) -&gt; bool:\n\"\"\"Check whether the computation should be interrupted.\n    Args:\n        idx: Position in the permutation currently being computed.\n        score: Last utility computed.\n    Returns:\n        `True` if the computation should be interrupted.\n    \"\"\"\nret = self._check(idx, score)\nself.n_calls += 1\nself.n_truncations += 1 if ret else 0\nreturn ret\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.NoTruncation","title":"<code>NoTruncation</code>","text":"<p>             Bases: <code>TruncationPolicy</code></p> <p>A policy which never interrupts the computation.</p>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.FixedTruncation","title":"<code>FixedTruncation(u, fraction)</code>","text":"<p>             Bases: <code>TruncationPolicy</code></p> <p>Break a permutation after computing a fixed number of marginals.</p> <p>The experiments in Appendix B of (Ghorbani and Zou, 2019)<sup>1</sup> show that when the training set size is large enough, one can simply truncate the iteration over permutations after a fixed number of steps. This happens because beyond a certain number of samples in a training set, the model becomes insensitive to new ones. Alas, this strongly depends on the data distribution and the model and there is no automatic way of estimating this number.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>fraction</code> <p>Fraction of marginals in a permutation to compute before stopping (e.g. 0.5 to compute half of the marginals).</p> <p> TYPE: <code>float</code> </p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self, u: Utility, fraction: float):\nsuper().__init__()\nif fraction &lt;= 0 or fraction &gt; 1:\nraise ValueError(\"fraction must be in (0, 1]\")\nself.max_marginals = len(u.data) * fraction\nself.count = 0\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.RelativeTruncation","title":"<code>RelativeTruncation(u, rtol)</code>","text":"<p>             Bases: <code>TruncationPolicy</code></p> <p>Break a permutation if the marginal utility is too low.</p> <p>This is called \"performance tolerance\" in (Ghorbani and Zou, 2019)<sup>1</sup>.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>rtol</code> <p>Relative tolerance. The permutation is broken if the last computed utility is less than <code>total_utility * rtol</code>.</p> <p> TYPE: <code>float</code> </p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self, u: Utility, rtol: float):\nsuper().__init__()\nself.rtol = rtol\nlogger.info(\"Computing total utility for permutation truncation.\")\nself.total_utility = u(u.data.indices)\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.BootstrapTruncation","title":"<code>BootstrapTruncation(u, n_samples, sigmas=1)</code>","text":"<p>             Bases: <code>TruncationPolicy</code></p> <p>Break a permutation if the last computed utility is close to the total utility, measured as a multiple of the standard deviation of the utilities.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_samples</code> <p>Number of bootstrap samples to use to compute the variance of the utilities.</p> <p> TYPE: <code>int</code> </p> <code>sigmas</code> <p>Number of standard deviations to use as a threshold.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1</code> </p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self, u: Utility, n_samples: int, sigmas: float = 1):\nsuper().__init__()\nself.n_samples = n_samples\nlogger.info(\"Computing total utility for permutation truncation.\")\nself.total_utility = u(u.data.indices)\nself.count: int = 0\nself.variance: float = 0\nself.mean: float = 0\nself.sigmas: float = sigmas\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.truncated_montecarlo_shapley","title":"<code>truncated_montecarlo_shapley(u, *, done, truncation, config=ParallelConfig(), n_jobs=1, coordinator_update_period=10, worker_update_period=5)</code>","text":"<p>Warning</p> <p>This method is deprecated and only a wrapper for permutation_montecarlo_shapley.</p> <p>Todo</p> <p>Think of how to add Robin-Gelman or some other more principled stopping criterion.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Check on the results which decides when to stop sampling permutations.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>truncation</code> <p>callable that decides whether to stop computing marginals for a given permutation.</p> <p> TYPE: <code>TruncationPolicy</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>n_jobs</code> <p>Number of permutation monte carlo jobs to run concurrently.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <p>Returns:     Object with the data values.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>@deprecated(\ntarget=True,\ndeprecated_in=\"0.7.0\",\nremove_in=\"0.8.0\",\nargs_mapping=dict(coordinator_update_period=None, worker_update_period=None),\n)\ndef truncated_montecarlo_shapley(\nu: Utility,\n*,\ndone: StoppingCriterion,\ntruncation: TruncationPolicy,\nconfig: ParallelConfig = ParallelConfig(),\nn_jobs: int = 1,\ncoordinator_update_period: int = 10,\nworker_update_period: int = 5,\n) -&gt; ValuationResult:\n\"\"\"\n    !!! Warning\n        This method is deprecated and only a wrapper for\n        [permutation_montecarlo_shapley][pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley].\n    !!! Todo\n        Think of how to add Robin-Gelman or some other more principled stopping\n        criterion.\n    Args:\n        u: Utility object with model, data, and scoring function\n        done: Check on the results which decides when to stop sampling\n            permutations.\n        truncation: callable that decides whether to stop computing marginals\n            for a given permutation.\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        n_jobs: Number of permutation monte carlo jobs to run concurrently.\n    Returns:\n        Object with the data values.\n    \"\"\"\nfrom pydvl.value.shapley.montecarlo import permutation_montecarlo_shapley\nreturn cast(\nValuationResult,\npermutation_montecarlo_shapley(\nu, done=done, truncation=truncation, config=config, n_jobs=n_jobs\n),\n)\n</code></pre>"},{"location":"api/pydvl/value/shapley/types/","title":"Types","text":""},{"location":"api/pydvl/value/shapley/types/#pydvl.value.shapley.types.ShapleyMode","title":"<code>ShapleyMode</code>","text":"<p>             Bases: <code>str</code>, <code>Enum</code></p> <p>Supported algorithms for the computation of Shapley values.</p> <p>Todo</p> <p>Make algorithms register themselves here.</p>"},{"location":"examples/data_oob/","title":"Data OOB","text":"<p>This notebook introduces the Data-OOB method, an implementation based on a publication from Kwon and Zou \"Data-OOB: Out-of-bag Estimate as a Simple and Efficient Data Value\" ICML 2023 , using pyDVL.</p> <p>The objective of this paper is mainly to overcome the computational bottleneck of shapley-based data valuation methods that require to fit a significant number of models to accurately estimate marginal contributions. The algorithms computes data values from out of bag estimates using a bagging model.</p> <p>The value can be interpreted as a partition of the OOB estimate, which is originally introduced to estimate the prediction error. This OOB estimate is given as:</p> \\[ \\sum_{i=1}^n\\frac{\\sum_{b=1}^{B}\\mathbb{1}(w_{bi}=0)T(y_i, \\hat{f}_b(x_i))}{\\sum_{b=1}^{B} \\mathbb{1} (w_{bi}=0)} \\] <pre>\n<code>   age  fnlwgt  education-num  capital-gain  capital-loss  hours-per-week  \\\n0   39   77516             13          2174             0              40   \n1   50   83311             13             0             0              13   \n2   38  215646              9             0             0              40   \n3   53  234721              7             0             0              40   \n4   28  338409             13             0             0              40   \n\n  income  \n0  &lt;=50K  \n1  &lt;=50K  \n2  &lt;=50K  \n3  &lt;=50K  \n4  &lt;=50K  \n</code>\n</pre> <pre><code>oob_values = compute_data_oob(utility, n_est=1000, max_samples=0.95)\n</code></pre>"},{"location":"examples/data_oob/#bagging-for-data-valuation","title":"Bagging for data valuation","text":""},{"location":"examples/data_oob/#setup","title":"Setup","text":"<p>We begin by importing the main libraries and setting some defaults.</p>   If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/data_oob/#variance","title":"Variance","text":"<p>The variance it the weak learner variance. It is computed with Welford's online algorithm.</p>"},{"location":"examples/data_oob/#point-removal-experiments","title":"Point removal experiments","text":"<p>The standard procedure for the evaluation of data valuation schemes is the point removal experiment. The objective is to measure the evolution of performance when the best/worst points are removed from the training set. </p>"},{"location":"examples/influence_imagenet/","title":"For CNNs","text":"If you are reading this in the documentation, some boilerplate has been omitted for convenience.   <pre><code>%load_ext autoreload\n</code></pre> <pre><code>%autoreload\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nimport torch\nfrom torch import nn\nfrom notebook_support import (\nplot_sample_images,\nplot_lowest_highest_influence_images,\nplot_losses,\ncorrupt_imagenet,\nload_preprocess_imagenet,\nplot_corrupted_influences_distribution,\ncompute_mean_corrupted_influences,\nTrainingManager,\nMODEL_PATH,\nnew_resnet_model,\n)\ndefault_figsize = (7, 7)\nplt.rcParams[\"figure.figsize\"] = default_figsize\nplt.rcParams[\"font.size\"] = 12\nplt.rcParams[\"xtick.labelsize\"] = 12\nplt.rcParams[\"ytick.labelsize\"] = 10\nhessian_reg = 1e4 if os.environ.get(\"CI\") else 1e-3\nrandom_state = 42\nnp.random.seed(random_state)\nDEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n</code></pre> <pre><code>from pydvl.influence.general import compute_influences\nfrom pydvl.reporting.plots import plot_influence_distribution_by_label\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, f1_score\n</code></pre> <pre><code>label_names = {90: \"tables\", 100: \"boats\"}\ntrain_ds, val_ds, test_ds = load_preprocess_imagenet(\ntrain_size=0.8,\ntest_size=0.1,\nkeep_labels=label_names,\ndownsampling_ratio=1,\n)\nprint(\"Normalised image dtype:\", train_ds[\"normalized_images\"][0].dtype)\nprint(\"Label type:\", type(train_ds[\"labels\"][0]))\nprint(\"Image type:\", type(train_ds[\"images\"][0]))\ntrain_ds.info()\n</code></pre> <pre>\n<code>Normalised image dtype: torch.float32\nLabel type: &lt;class 'str'&gt;\nImage type: &lt;class 'PIL.JpegImagePlugin.JpegImageFile'&gt;\n&lt;class 'pandas.core.frame.DataFrame'&gt;\nRangeIndex: 707 entries, 0 to 706\nData columns (total 3 columns):\n #   Column             Non-Null Count  Dtype \n---  ------             --------------  ----- \n 0   normalized_images  707 non-null    object\n 1   labels             707 non-null    object\n 2   images             707 non-null    object\ndtypes: object(3)\nmemory usage: 16.7+ KB\n</code>\n</pre> <p>Let's take a closer look at a few image samples</p> <pre><code>plot_sample_images(train_ds, n_images_per_class=3)\n</code></pre> <p>Let's now further pre-process the data and prepare for model training. The helper function <code>process_io</code> converts the normalized images into tensors and the labels to the indices 0 and 1 to train the classifier.</p> <pre><code>from typing import Tuple\ndef process_io(df: pd.DataFrame, labels: dict) -&amp;gt; Tuple[torch.Tensor, torch.Tensor]:\nx = df[\"normalized_images\"]\ny = df[\"labels\"]\nds_label_to_model_label = {\nds_label: idx for idx, ds_label in enumerate(labels.values())\n}\nx_nn = torch.stack(x.tolist()).to(DEVICE)\ny_nn = torch.tensor([ds_label_to_model_label[yi] for yi in y], device=DEVICE)\nreturn x_nn, y_nn\ntrain_x, train_y = process_io(train_ds, label_names)\nval_x, val_y = process_io(val_ds, label_names)\ntest_x, test_y = process_io(test_ds, label_names)\n</code></pre> <pre><code>model_ft = new_resnet_model(output_size=len(label_names))\nmgr = TrainingManager(\n\"model_ft\",\nmodel_ft,\nnn.CrossEntropyLoss(),\ntrain_x,\ntrain_y,\nval_x,\nval_y,\nMODEL_PATH,\n)\n# Set use_cache=False to retrain the model\ntrain_loss, val_loss = mgr.train(n_epochs=50, use_cache=True)\n</code></pre> <pre>\n<code>Cached model found, loading...\n</code>\n</pre> <pre><code>plot_losses(train_loss, val_loss)\n</code></pre> <p>The confusion matrix and \\(F_1\\) score look good, especially considering the low resolution of the images and their complexity (they contain different objects)</p> <pre><code>pred_y_test = np.argmax(model_ft(test_x).detach(), axis=1)\nmodel_score = f1_score(test_y, pred_y_test, average=\"weighted\")\ncm = confusion_matrix(test_y, pred_y_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=label_names.values())\nprint(\"f1_score of model:\", model_score)\ndisp.plot();\n</code></pre> <pre>\n<code>f1_score of model: 0.8468272032336833\n</code>\n</pre> <pre><code>influences = compute_influences(\nmodel=mgr.model,\nloss=mgr.loss,\nx=train_x,\ny=train_y,\nx_test=test_x,\ny_test=test_y,\nhessian_regularization=hessian_reg,\ninversion_method=\"cg\",\ninfluence_type=\"up\",\nprogress=True,\n)\n</code></pre> <pre><code>test_image_idx = 42\nmodel_label_to_ds_label = {\nidx: ds_label for idx, ds_label in enumerate(label_names.values())\n}\npredicted_label = model_label_to_ds_label[\nnp.argmax(model_ft(test_x[test_image_idx].unsqueeze(0)).detach(), axis=1).item()\n]\ntrue_label = test_ds[\"labels\"][test_image_idx]\nplt.rcParams[\"figure.figsize\"] = (3, 3)\nplt.imshow(test_ds[\"images\"][test_image_idx])\nplt.axis(\"off\")\nplt.title(f\"Predicted: {predicted_label} - True: {true_label}\")\nplt.show()\n</code></pre> <p>Now we plot the histogram of the influence that all training images have on the image selected above, separated by their label.</p> <pre><code>plt.rcParams[\"figure.figsize\"] = default_figsize\nplot_influence_distribution_by_label(\ninfluences[test_image_idx],\ntrain_ds[\"labels\"].values,\ntitle_extra=f\"over index {test_image_idx}\",\n)\n</code></pre> <p>Rather unsurprisingly, the training samples that have the same label as the test image have, on average, a higher influence on the classifier's output for it. Let's then take them and visualize those with the highest and lowest influence:</p> <pre><code>images_with_same_label = train_ds[\"labels\"] == test_ds[\"labels\"][test_image_idx]\ninfluence_values_with_same_label = influences[test_image_idx][images_with_same_label]\nimages_same_label = train_ds[\"images\"][images_with_same_label].values\nplot_lowest_highest_influence_images(\ninfluence_values_with_same_label, subset_images=images_same_label, num_to_plot=3\n)\n</code></pre> <p>Looking at the images, it is difficult to explain why those on the right are more influential than those on the left. At first sight, the choice seems to be random (or at the very least noisy). Let's dig in a bit more by looking at average influences:</p> <pre><code>avg_influences = np.mean(influences, axis=0)\n</code></pre> <p>Once again, let's plot the histogram of influence values by label.</p> <pre><code>plot_influence_distribution_by_label(\navg_influences, train_ds[\"labels\"].values, \"over all test samples\"\n)\n</code></pre> <p>Next, for each class (you can change value by changing label key) we can have a look at the top and bottom images by average influence, i.e. we can show the images that have the highest and lowest average influence over all test images.</p> <pre><code>label = \"tables\"\nimg_with_selected_label = train_ds[\"labels\"] == label\nif_selected_label = avg_influences[img_with_selected_label]\nimges_same_label = train_ds[\"images\"][img_with_selected_label].values\nplot_lowest_highest_influence_images(if_selected_label, imges_same_label, num_to_plot=3)\n</code></pre> <p>Once again, it is not easy to explain why the images on the left have a lower influence than the ones on the right. One could argue that in order to predict that there is a dining table in the image it is beneficial to clearly see both the chairs and the table itself, a feature missing in some samples on the left. Also, color seems to be a discriminant: houses with a blue painting could get confused with the water around a boat. Of course, this is debatable and different people could come up with other explanations a posteriori.</p> <pre><code>corrupted_model = new_resnet_model(output_size=len(label_names))\ncorrupted_dataset, corrupted_indices = corrupt_imagenet(\ndataset=train_ds,\nfraction_to_corrupt=0.1,\navg_influences=avg_influences,\n)\ncorrupted_train_x, corrupted_train_y = process_io(corrupted_dataset, label_names)\nmgr = TrainingManager(\n\"corrupted_model\",\ncorrupted_model,\nnn.CrossEntropyLoss(),\ncorrupted_train_x,\ncorrupted_train_y,\nval_x,\nval_y,\nMODEL_PATH,\n)\ntraining_loss, validation_loss = mgr.train(n_epochs=50, use_cache=True)\n</code></pre> <pre>\n<code>Cached model found, loading...\n</code>\n</pre> <pre><code>plot_losses(training_loss, validation_loss)\n</code></pre> <pre><code>pred_y_test = np.argmax(corrupted_model(test_x).detach(), axis=1)\nmodel_score = f1_score(test_y, pred_y_test, average=\"weighted\")\nprint(\"F1 score of model with corrupted data:\", model_score)\n</code></pre> <pre>\n<code>F1 score of model with corrupted data: 0.8164795918367347\n</code>\n</pre> <p>Interestingly, despite being trained on a corrupted dataset, the model has a fairly high \\(F_1\\) score. Let's now calculate the influence of the corrupted training data points over the test data points.</p> <pre><code>influences = compute_influences(\nmodel=mgr.model,\nloss=mgr.loss,\nx=corrupted_train_x,\ny=corrupted_train_y,\nx_test=test_x,\ny_test=test_y,\nhessian_regularization=hessian_reg,\ninversion_method=\"cg\",\ninfluence_type=\"up\",\nprogress=True,\n)\n</code></pre> <pre>\n<code>Split Gradient:   0%|          | 0/98 [00:00&lt;?, ?it/s]</code>\n</pre> <pre>\n<code>Conjugate gradient:   0%|          | 0/98 [00:00&lt;?, ?it/s]</code>\n</pre> <pre>\n<code>Split Gradient:   0%|          | 0/707 [00:00&lt;?, ?it/s]</code>\n</pre> <p>As before, since we are interested in the average influence on the test dataset, we take the average of influences across rows, and then plot the highest and lowest influences for a chosen label</p> <pre><code>avg_corrupted_influences = np.mean(influences, axis=0)\n</code></pre> <pre><code>label = \"boats\"\nimg_with_selected_label = corrupted_dataset[\"labels\"] == label\nif_selected_label = avg_corrupted_influences[img_with_selected_label]\nimges_same_label = corrupted_dataset[\"images\"][img_with_selected_label].values\nplot_lowest_highest_influence_images(if_selected_label, imges_same_label, num_to_plot=3)\n</code></pre> <p>As expected, the samples with lowest (negative) influence for the label \"boats\" are those that have been corrupted: all the images on the left are tables! We can compare the average influence of corrupted data with non-corrupted ones</p> <pre><code>plot_corrupted_influences_distribution(\ncorrupted_dataset, corrupted_indices, avg_corrupted_influences\n)\n</code></pre> <pre><code>compute_mean_corrupted_influences(\ncorrupted_dataset, corrupted_indices, avg_corrupted_influences\n)\n</code></pre> label avg_non_corrupted_infl avg_corrupted_infl score_diff 0 boats 0.945390 -0.890972 1.836362 1 tables -1.092637 -2.757206 1.664569 <p>And indeed corrupted data have a more negative influence on average than clean ones!</p> <p>Despite this being a useful property, influence functions are known to be unreliable for tasks of data valuation, especially in deep learning where the fundamental assumption of the theory (convexity) is grossly violated. A lot of factors (e.g. the size of the network, the training process or the Hessian regularization term) can interfere with the computation, to the point that often the results that we obtain cannot be trusted. This has been extensively studied in the recent paper:</p> <p>Basu, S., P. Pope, and S. Feizi. Influence Functions in Deep Learning Are Fragile. International Conference on Learning Representations (ICLR). 2021.</p> <p>Nevertheless, influence functions offer a relatively quick and mathematically rigorous way to evaluate (at first order) the importance of a training point for a model's prediction.</p>"},{"location":"examples/influence_imagenet/#influence-functions-for-neural-networks","title":"Influence functions for neural networks","text":"<p>This notebook explores the use of influence functions for convolutional neural networks. In the first part we will investigate the usefulness, or lack thereof, of influence functions for the interpretation of a classifier's outputs.</p> <p>For our study we choose a pre-trained ResNet18, fine-tuned on the tiny-imagenet dataset. This dataset was created for a Stanford course on Deep Learning for Computer Vision, and is a subset of the famous ImageNet with 200 classes instead of 1000, and images down-sampled to a lower resolution of 64x64 pixels.</p> <p>After tuning the last layers of the network, we will use pyDVL to find the most and the least influential training images for the test set. This can sometimes be used to explain inference errors, or to direct efforts during data collection, although we will face inconclusive results with our model and data. This illustrates well-known issues of influence functions for neural networks.</p> <p>However, in the final part of the notebook we will see that influence functions are an effective tool for finding anomalous or corrupted data points.</p> <p>We conclude with an appendix with some basic theoretical concepts used.</p>"},{"location":"examples/influence_imagenet/#imports-and-setup","title":"Imports and setup","text":""},{"location":"examples/influence_imagenet/#loading-and-preprocessing-the-dataset","title":"Loading and preprocessing the dataset","text":"<p>We pick two classes arbitrarily to work with: 90 and 100, corresponding respectively to dining tables, and boats in Venice (you can of course select any other two classes, or more of them, although that would imply longer training times and some modifications in the notebook below). The dataset is loaded with <code>load_preprocess_imagenet()</code>, which returns three pandas <code>DataFrames</code> with training, validation and test sets respectively. Each dataframe has three columns: normalized images, labels and the original images. Note that you can load a subset of the data decreasing downsampling_ratio.</p>"},{"location":"examples/influence_imagenet/#model-definition-and-training","title":"Model definition and training","text":"<p>We use a ResNet18 from <code>torchvision</code> with final layers modified for binary classification.</p> <p>Training for influence computation is facilitated by <code>:class:~pydvl.influence.model_wrappers.torch_wrappers.TorchModel</code>, a convenience wrapper around torch models which is part of pyDVL. We wrap this with a simple class <code>TrainingManager</code> which transparently handles persistence after training. The latter is not part of the main pyDVL package but just a way to reduce clutter in this notebook.</p> <p>We train the model for 50 epochs and save the results. Then we plot the train and validation loss curves.</p>"},{"location":"examples/influence_imagenet/#influence-computation","title":"Influence computation","text":"<p>Let's now calculate influences! The main method is <code>:func:~pydvl.influence.general.compute_influences</code>, which takes a trained <code>nn.Model</code>, the training loss, some input dataset with labels (which typically is the training data, or a subset of it) and some test data.</p> <p>Other important parameters are the Hessian regularization term, which should be chosen as small as possible for the computation to converge (further details on why this is important can be found in the Appendix).</p> <p>Since Resnet18 is quite big, we pick conjugate gradient (<code>cg</code>) as the method for inversion of the Hessian. A naive computation would require a lot of memory. Finally, the influence type will be <code>up</code>. The other option, <code>perturbation</code>, is beyond the scope of this notebook, but more info can be found in the notebook using the Wine dataset or in the documentation for pyDVL.</p> <p>The output of <code>calculate_influences</code> is a matrix of size <code>test_set_length</code> x <code>training_set_length</code>. Each row represents a test data point, and each column a training data point, so that entry \\((i,j)\\) represents the influence of training point \\(j\\) on test point \\(i\\).</p>"},{"location":"examples/influence_imagenet/#analysing-influences","title":"Analysing influences","text":"<p>With the computed influences we can study single images or all of them together:</p>"},{"location":"examples/influence_imagenet/#influence-on-a-single-test-image","title":"Influence on a single test image","text":"<p>Let's take any image in the test set:</p>"},{"location":"examples/influence_imagenet/#analysing-the-average-influence-on-test-samples","title":"Analysing the average influence on test samples","text":"<p>By averaging across the rows of the influence matrix, we obtain the average influence of each training sample on the whole test set:</p>"},{"location":"examples/influence_imagenet/#detecting-corrupted-data","title":"Detecting corrupted data","text":"<p>After facing the shortcomings of influence functions for explaining decisions, we move to an application with clear-cut results. Influences can be successfully used to detect corrupted or mislabeled samples, making them an effective tool to \"debug\" training data.</p> <p>We begin by training a new model (with the same architecture as before) on a dataset with some corrupted labels. The method <code>get_corrupted_imagenet</code> will take the training dataset and corrupt a certain fraction of the labels by flipping them. We use the same number of epochs and optimizer as before.</p>"},{"location":"examples/influence_imagenet/#theory-of-influence-functions-for-neural-networks","title":"Theory of influence functions for neural networks","text":"<p>In this appendix we will briefly go through the basic ideas of influence functions adapted for neural networks as introduced in Koh, Pang Wei, and Percy Liang. \"Understanding Black-box Predictions via Influence Functions\" International conference on machine learning. PMLR, 2017.</p> <p>Note however that this paper departs from the standard and established theory and notation for influence functions. For a rigorous introduction to the topic we recommend classical texts like Hampel, Frank R., Elvezio M. Ronchetti, Peter J. Rousseeuw, and Werner A. Stahel. Robust Statistics: The Approach Based on Influence Functions. 1st edition. Wiley Series in Probability and Statistics. New York: Wiley-Interscience, 2005. https://doi.org/10.1002/9781118186435.</p>"},{"location":"examples/influence_imagenet/#upweighting-points","title":"Upweighting points","text":"<p>Let's start by considering some input space \\(\\mathcal{X}\\) to a model (e.g. images) and an output space \\(\\mathcal{Y}\\) (e.g. labels). Let's take \\(z_i = (x_i, y_i)\\) to be the \\(i\\)-th training point, and \\(\\theta\\) to be the (potentially highly) multi-dimensional parameters of the neural network (i.e. \\(\\theta\\) is a big array with very many parameters). We will indicate with \\(L(z, \\theta)\\) the loss of the model for point \\(z\\) and parameters \\(\\theta\\). When training the model we minimize the loss over all points, i.e. the optimal parameters are calculated through gradient descent on the following formula: $$ \\hat{\\theta} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_i, \\theta) $$ where \\(n\\) is the total number of training data points.</p> <p>For notational  convenience, let's define  $$ \\hat{\\theta}{-z} = \\arg \\min\\theta \\frac{1}{n}\\sum_{z_i \\ne z} L(z_i, \\theta) \\ , $$ i.e. \\(\\hat{\\theta}_{-z}\\) are the model parameters that minimize the total loss when \\(z\\) is not in the training dataset.</p> <p>In order to check the impact of each training point on the model, we would need to calculate \\(\\hat{\\theta}_{-z}\\) for each \\(z\\) in the training dataset, thus re-training the model at least ~\\(n\\) times (more if model training is noisy). This is computationally very expensive, especially for big neural networks. To circumvent this problem, we can just calculate a first order approximation of \\(\\hat{\\theta}\\). This can be done through single backpropagation and without re-training the full model.</p> <p>Let's define $$ \\hat{\\theta}{\\epsilon, z} = \\arg \\min\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_i, \\theta) + \\epsilon L(z, \\theta) \\ , $$ which is the optimal \\(\\hat{\\theta}\\) if we were to up-weigh \\(z\\) by an amount \\(\\epsilon\\).</p> <p>From a classical result (a simple derivation is available in Appendix A of Koh and Liang's paper), we know that: $$ \\frac{d \\ \\hat{\\theta}{\\epsilon, z}}{d \\epsilon} \\Big| = -H_{\\hat{\\theta}}^{-1} \\nabla_\\theta L(z, \\hat{\\theta}) $$ where \\(H_{\\hat{\\theta}} = \\frac{1}{n} \\sum_{i=1}^n \\nabla_\\theta^2 L(z_i, \\hat{\\theta})\\) is the Hessian of \\(L\\). Importantly, notice that this expression is only valid when \\(\\hat{\\theta}\\) is a minimum of \\(L\\), or otherwise \\(H_{\\hat{\\theta}}\\) cannot be inverted!</p>"},{"location":"examples/influence_imagenet/#approximating-the-influence-of-a-point","title":"Approximating the influence of a point","text":"<p>We will define the influence of training point \\(z\\) on test point \\(z_{\\text{test}}\\) as \\(\\mathcal{I}(z, z_{\\text{test}}) =  L(z_{\\text{test}}, \\hat{\\theta}_{-z}) - L(z_{\\text{test}}, \\hat{\\theta})\\) (notice that it is higher for points \\(z\\) which positively impact the model score, since if they are excluded, the loss is higher). In practice, however, we will always use the infinitesimal approximation \\(\\mathcal{I}_{up}(z, z_{\\text{test}})\\), defined as $$  \\mathcal{I}{up}(z, z}) = - \\frac{d L(z_{\\text{test}}, \\hat{\\theta}{\\epsilon, z})}{d \\epsilon} \\Big| $$</p> <p>Using the chain rule and the results calculated above, we thus have:</p> \\[  \\mathcal{I}_{up}(z, z_{\\text{test}}) = - \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ \\frac{d \\hat{\\theta}_{\\epsilon, z}}{d \\epsilon} \\Big|_{\\epsilon=0} = \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ H_{\\hat{\\theta}}^{-1} \\ \\nabla_\\theta L(z, \\hat{\\theta}) \\] <p>In order to calculate this expression we need the gradient and the Hessian of the loss wrt. the model parameters \\(\\hat{\\theta}\\). This can be easily done through a single backpropagation pass.</p>"},{"location":"examples/influence_imagenet/#regularizing-the-hessian","title":"Regularizing the Hessian","text":"<p>One very important assumption that we make when approximating influence is that \\(\\hat{\\theta}\\) is at least a local minimum of the loss. However, we clearly cannot guarantee this except for convex models, and despite good apparent convergence, \\(\\hat{\\theta}\\) might be located in a region with flat curvature or close to a saddle point. In particular, the Hessian might have vanishing eigenvalues making its direct inversion impossible.</p> <p>To circumvent this problem, instead of inverting the true Hessian \\(H_{\\hat{\\theta}}\\), one can invert a small perturbation thereof: \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\), with \\(\\mathbb{I}\\) being the identity matrix. This standard trick ensures that the eigenvalues of \\(H_{\\hat{\\theta}}\\) are bounded away from zero and therefore the matrix is invertible. In order for this regularization not to corrupt the outcome too much, the parameter \\(\\lambda\\) should be as small as possible while still allowing a reliable inversion of \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\).</p>"},{"location":"examples/influence_synthetic/","title":"For mislabeled data","text":"<pre><code>%load_ext autoreload\n</code></pre> <pre><code>%autoreload\n%matplotlib inline\nimport os\nimport random\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\nfrom pydvl.influence import compute_influences, TorchTwiceDifferentiable\nfrom support.shapley import (\nsynthetic_classification_dataset,\ndecision_boundary_fixed_variance_2d,\n)\nfrom support.common import (\nplot_gaussian_blobs,\nplot_losses,\nplot_influences,\n)\nfrom support.torch import (\nfit_torch_model,\nTorchLogisticRegression,\n)\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\nfrom torch.optim import AdamW, lr_scheduler\nfrom torch.utils.data import DataLoader\n</code></pre> <pre>\n<code>/Users/fabio/miniconda3/envs/pydvl_env/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</code>\n</pre> <pre><code>plt.rcParams[\"figure.figsize\"] = (16, 8)\nplt.rcParams[\"font.size\"] = 12\nplt.rcParams[\"xtick.labelsize\"] = 12\nplt.rcParams[\"ytick.labelsize\"] = 10\n</code></pre> <pre><code>random_state = 24\nis_CI = os.environ.get(\"CI\")\n</code></pre> <pre><code>num_samples = 10000\nnum_features = 2\nsigma = 0.2\nmeans = np.asarray([[0.0, 0.0], [1.0, 1.0]])\n</code></pre> <pre><code>random.seed(random_state)\nnp.random.seed(random_state)\n</code></pre> <p>The following code snippet generates the aforementioned dataset.</p> <pre><code>train_data, val_data, test_data = synthetic_classification_dataset(\nmeans, sigma, num_samples, train_size=0.7, test_size=0.2\n)\n# In CI we only use a subset of the training set\nif is_CI:\ntrain_data = (train_data[0][:10], train_data[1][:10])\n</code></pre> <p>Given the simplicity of the dataset, we can calculate exactly the optimal decision boundary(that which maximizes our accuracy). The following code maps a continuous line of z values to a 2-dimensional vector in feature space (More details are in the appendix to this notebook.)</p> <pre><code>decision_boundary_fn = decision_boundary_fixed_variance_2d(means[0], means[1])\ndecision_boundary = decision_boundary_fn(np.linspace(-1.5, 1.5, 100))\n</code></pre> <pre><code>plot_gaussian_blobs(\ntrain_data,\ntest_data,\nxlabel=\"$x_0$\",\nylabel=\"$x_1$\",\nlegend_title=\"$y - labels$\",\nline=decision_boundary,\ns=10,\nsuptitle=\"Plot of train-test data\",\n)\n</code></pre> <p>Note that there are samples which go across the optimal decision boundary and will be wrongly labelled. The optimal decision boundary can not discriminate these as the mislabelling is a consequence of the presence of random noise.</p> <pre><code>model = TorchLogisticRegression(num_features)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\nnum_epochs = 50\nlr = 0.05\nweight_decay = 0.05\nbatch_size = 256\ntrain_data_loader = DataLoader(\nlist(zip(train_data[0], train_data[1].astype(float))),\nbatch_size=batch_size,\nshuffle=True,\n)\nval_data_loader = DataLoader(\nlist(zip(val_data[0], val_data[1].astype(float))),\nbatch_size=batch_size,\nshuffle=True,\n)\noptimizer = AdamW(params=model.parameters(), lr=lr, weight_decay=weight_decay)\nscheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs)\nlosses = fit_torch_model(\nmodel=model,\ntraining_data=train_data_loader,\nval_data=val_data_loader,\nloss=F.binary_cross_entropy,\noptimizer=optimizer,\nscheduler=scheduler,\nnum_epochs=num_epochs,\n)\n</code></pre> <pre>\n<code>Model fitting:   0%|          | 0/50 [00:00&lt;?, ?it/s]Model fitting: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 50/50 [00:02&lt;00:00, 19.41it/s]\n</code>\n</pre> <p>And let's check that the model is not overfitting</p> <pre><code>plot_losses(losses)\n</code></pre> <p>A look at the confusion matrix also shows good results</p> <pre><code>model.eval()\npred_probabilities = model(test_data[0]).detach()\npred_y_test = [1 if prob &amp;gt; 0.5 else 0 for prob in pred_probabilities]\ncm = confusion_matrix(test_data[1], pred_y_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm)\ndisp.plot();\n</code></pre> <p>It is important that the model converges to a point near the optimum, since the influence values assume that we are at a minimum (or close) in the loss landscape. The function</p> \\[I(x_1, y_1, x_2, y_2) \\colon \\mathbb{R}^d \\times \\mathbb{R}^d \\to \\mathbb{R}\\] <p>measures the influence of the data point \\(x_1\\) onto \\(x_2\\) conditioned on the training targets \\(y_1\\) and \\(y_2\\) trough some model parameters \\(\\theta\\). If the loss function L is differentiable, we can take \\(I\\) to be</p> <p>$$ I(x_1, x_2) = \\nabla_\\theta\\; L(x_1, y_1) ^\\mathsf{T} \\; H_\\theta^{-1} \\; \\nabla_\\theta \\; L(x_2, y_2) $$ See \"Understanding Black-box Predictions via Influence Functions\" for a detailed derivation of this formula</p> <p>Let's take a subset of the training data points, which we will calculate the influence values of. </p> <pre><code>x = train_data[0][:100]\ny = train_data[1][:100]\n</code></pre> <p>In pyDVL, the influence of the training points on the test points can be calculated with the following</p> <pre><code>train_data_loader = DataLoader(list(zip(x, y.astype(float))), batch_size=batch_size)\ntest_data_loader = DataLoader(\nlist(zip(test_data[0], test_data[1].astype(float))), batch_size=batch_size\n)\ninfluence_values = compute_influences(\ndifferentiable_model=TorchTwiceDifferentiable(model, F.binary_cross_entropy),\ntraining_data=train_data_loader,\ntest_data=test_data_loader,\ninfluence_type=\"up\",\ninversion_method=\"direct\",  # use 'cg' for big models\n)\n</code></pre> <p>The above explicitly constructs the Hessian. This can often be computationally expensive and conjugate gradient approximate calculation should be used for bigger models. </p> <p>With the influence type 'up', training influences have shape [NxM] where N is the number of test samples and M is the number of training samples. They therefore associate to each training sample its influence on each test sample.  Influence type 'perturbation', instead, return an array of shape  [NxMxF], where F is the number of features in input, ie. the length of x. </p> <p>In our case, in order to have a value of the total average influence of a point we can just average across training samples.</p> <pre><code>mean_train_influences = np.mean(influence_values.numpy(), axis=0)\n</code></pre> <p>Let's plot the results (adjust colorbar_limits for better color gradient)</p> <pre><code>plot_influences(\nx,\nmean_train_influences,\nline=decision_boundary,\nxlabel=\"$x_0$\",\nylabel=\"$x_1$\",\nsuptitle=\"Influences of input points\",\nlegend_title=\"influence values\",\n# colorbar_limits=(-0.3,),\n);\n</code></pre> <p>We can see that, as we approach the separation line, the influences tend to move away from zero, i.e. the points become more decisive for model training, some in a positive way, some negative.</p> <p>As a further test, let's introduce some labelling errors into \\(y\\) and see how the distribution of the influences changes. Let's flip the first 10 labels and calculate influences</p> <pre><code>y_corrupted = np.copy(y)\ny_corrupted[:10] = [1 - yi for yi in y[:10]]\ntrain_corrupted_data_loader = DataLoader(\nlist(zip(x, y_corrupted.astype(float))), batch_size=batch_size\n)\ninfluence_values = compute_influences(\ndifferentiable_model=TorchTwiceDifferentiable(model, F.binary_cross_entropy),\ntraining_data=train_corrupted_data_loader,\ntest_data=test_data_loader,\ninfluence_type=\"up\",\ninversion_method=\"direct\",\n)\nmean_train_influences = np.mean(influence_values.numpy(), axis=0)\n</code></pre> <pre><code>print(\"Average mislabelled data influence:\", np.mean(mean_train_influences[:10]))\nprint(\"Average correct data influence:\", np.mean(mean_train_influences[10:]))\n</code></pre> <pre>\n<code>Average mislabelled data influence: -0.8225848370029777\nAverage correct data influence: 0.011277048916970962\n</code>\n</pre> <pre><code>plot_influences(\nx,\nmean_train_influences,\ncorrupted_indices=np.array(range(10)),\nline=decision_boundary,\nxlabel=\"$x_0$\",\nylabel=\"$x_1$\",\nsuptitle=\"Influences of input points with corrupted data\",\nlegend_title=\"influence values\",\n# colorbar_limits=(-0.3,),\n);\n</code></pre> <p>Red circles indicate the points which have been corrupted. We can see that the mislabelled data have a more negative average influence on the model, especially those that are farther away from the decision boundary.</p> <p>The \"direct\" method that we have used above involves the inversion of the Hessian matrix of the model. If a model has \\(n\\) training points and \\(\\theta  \\in \\mathbb{R}^p\\) parameters, this requires \\(O(n \\ p^2 + p^3)\\) operations, which for larger models, like neural networks, becomes quickly unfeasible. Conjugate gradient avoids the explicit computation of the Hessian via a technique called implicit Hessian-vector products (HVPs), which typically takes \\(O(n \\ p)\\) operations.</p> <p>In the next cell we will use conjugate gradient to compute the influence factors. Since logistic regression is a very simple model, \"cg\" actually slows computation with respect to the direct method, which in this case is a much better choice. Nevertheless, we are able to verify that the influences calculated with \"cg\" are the same (to a minor error) as those calculated directly.</p> <pre><code>influence_values = compute_influences(\ndifferentiable_model=TorchTwiceDifferentiable(model, F.binary_cross_entropy),\ntraining_data=train_corrupted_data_loader,\ntest_data=test_data_loader,\ninfluence_type=\"up\",\ninversion_method=\"cg\",\nprogress=True,\n)\nmean_train_influences = np.mean(influence_values.numpy(), axis=0)\nprint(\"Average mislabelled data influence:\", np.mean(mean_train_influences[:10]))\nprint(\"Average correct data influence:\", np.mean(mean_train_influences[10:]))\n</code></pre> <pre>\n<code>Batch Test Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 8/8 [00:00&lt;00:00, 17.89it/s]\nBatch Train Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 308.47it/s]\nConjugate gradient: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 2000/2000 [00:16&lt;00:00, 118.24it/s]\nBatch Split Input Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 44.89it/s]</code>\n</pre> <pre>\n<code>Average mislabelled data influence: -0.82248804123547\nAverage correct data influence: 0.01127580743952819\n</code>\n</pre> <pre>\n<code>\n</code>\n</pre> <p>Averages are very similar to the ones calculated through direct method. Same is true for the plot</p> <pre><code>plot_influences(\nx,\nmean_train_influences,\ncorrupted_indices=np.array(range(10)),\nline=decision_boundary,\nxlabel=\"$x_0$\",\nylabel=\"$x_1$\",\nsuptitle=\"Influences of input points with corrupted data\",\nlegend_title=\"influence values\",\n# colorbar_limits=(-0.1, 0.1),\n);\n</code></pre>"},{"location":"examples/influence_synthetic/#influence-functions-for-data-mislabeling","title":"Influence functions for data mislabeling","text":"<p>In this notebook, we will take a closer look at the theory of influence functions with the help of a synthetic dataset. Data mislabeling occurs whenever some examples from a usually big dataset are wrongly-labeled. In real-life this happens fairly often, e.g. as a consequence of human error, or noise in the data. </p> <p>Let's consider a classification problem with the following notation:</p> \\[ \\begin{align*} x_i &amp;\\in \\mathbb{R}^d \\\\ y_i &amp;\\in \\{0, 1\\} \\\\ \\forall i &amp;\\in [ N ] \\end{align*} \\] <p>In other words, we have a dataset containing \\(N\\) samples, each with label 1 or 0. As typical example you can think of y indicating whether a patient has a disease based on some feature representation \\(x\\). </p> <p>Let's now introduce a toy model that will help us delve into the theory and practical utility of influence functions. We will assume that \\(y\\) is a Bernoulli binary random variable while the input \\(x\\) is d-dimensional Gaussian distribution which depends on the label \\(y\\). More precisely:</p> \\[ y_i \\sim \\text{Ber}\\left (0.5 \\right) \\\\ x_i \\sim \\mathcal{N}\\left ((1 - y_i) \\mu_1 + y_i \\mu_2, \\sigma^2 I \\right), \\] <p>with fixed means and diagonal covariance. Implementing the sampling scheme in python is straightforward and can be achieved by first sampling \\(y\\) and afterward \\(x\\).</p>"},{"location":"examples/influence_synthetic/#imports","title":"Imports","text":""},{"location":"examples/influence_synthetic/#constants","title":"Constants","text":""},{"location":"examples/influence_synthetic/#dataset","title":"Dataset","text":""},{"location":"examples/influence_synthetic/#plotting-the-dataset","title":"Plotting the dataset","text":"<p>Let's plot the dataset is plotted with their respective labels and the optimal decision line</p>"},{"location":"examples/influence_synthetic/#training-the-model","title":"Training the model","text":"<p>We will now train a logistic regression model on the training data. This can be done with the following</p>"},{"location":"examples/influence_synthetic/#calculating-influences","title":"Calculating influences","text":""},{"location":"examples/influence_synthetic/#inversion-through-conjugate-gradient","title":"Inversion through conjugate gradient","text":""},{"location":"examples/influence_synthetic/#appendix-calculating-the-decision-boundary","title":"Appendix: Calculating the decision boundary","text":"<p>For obtaining the optimal discriminator one has to solve the equation</p> \\[p(x|y=0)=p(x|y=1)\\] <p>and determine the solution set \\(X\\). Let's take the following probabilities</p> \\[ \\begin{align*} p(x|y=0)&amp;=\\mathcal{N}\\left (\\mu_1, \\sigma^2 I \\right) \\\\ p(x|y=1)&amp;=\\mathcal{N}\\left (\\mu_2, \\sigma^2 I \\right) \\end{align*} \\] <p>For a single fixed diagonal variance parameterized by \\(\\sigma\\), the optimal discriminator lays at points which are equidistant from the means of the two distributions, i.e.</p> \\[ \\begin{align*} \\| x - \\mu_1 \\|^2 &amp;= \\| x - \\mu_2 \\|^2 \\\\ \\| \\mu_1 \\|^2 -2 x^\\mathsf{T} \\mu_1 &amp;= \\| \\mu_2 \\|^2 -2 x^\\mathsf{T} \\mu_2 \\\\ \\implies 0 &amp;= 2 (\\mu_2 - \\mu_1)^\\mathsf{T} x + \\| \\mu_1 \\|^2 - \\| \\mu_2 \\|^2 \\\\ 0 &amp;= \\mu_1^\\mathsf{T}x - \\mu_2^\\mathsf{T}x - \\frac{1}{2} \\mu_1^\\mathsf{T} \\mu_1 + \\frac{1}{2} \\mu_2^\\mathsf{T} \\mu_2 \\end{align*} \\] <p>This is just the implicit description of the line. Solving for the explicit form can be achieved by enforcing a functional form \\(f(z) = x = a z + b\\) with \\(z \\in \\mathbb{R}\\) onto \\(x\\). After the term is inserted in the previous equation</p> \\[ 0 = (\\mu_2 - \\mu_1)^\\mathsf{T} (az + b) + \\frac{1}{2} \\| \\mu_1 \\|^2 - \\| \\mu_2 \\|^2 \\] <p>We can write \\(a\\) since, by symmetry, it is expected to be explicitly orthogonal to \\(\\mu_2 - \\mu_1\\). Then, solving for \\(b\\), the solution can be found to be</p> \\[ f(z) = \\underbrace{\\begin{bmatrix} 0 &amp; 1 \\\\ -1 &amp; 0 \\end{bmatrix} (\\mu_2 - \\mu_1)}_a z + \\underbrace{\\frac{\\mu_1 + \\mu_2}{2}}_b \\]"},{"location":"examples/influence_wine/","title":"For outlier detection","text":"<p>Let's start by loading the imports, the dataset and splitting it into train, validation and test sets. We will use a large test set to have a less noisy estimate of the average influence.</p> <pre><code>%load_ext autoreload\n</code></pre> <pre><code>%autoreload\n%matplotlib inline\nimport os\nimport random\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nfrom support.common import plot_losses\nfrom support.torch import TorchMLP, fit_torch_model\nfrom pydvl.influence import compute_influences, TorchTwiceDifferentiable\nfrom support.shapley import load_wine_dataset\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, f1_score\nfrom torch.optim import Adam, lr_scheduler\nfrom torch.utils.data import DataLoader, TensorDataset\n</code></pre> <pre><code>plt.rcParams[\"figure.figsize\"] = (16, 8)\nplt.rcParams[\"font.size\"] = 12\nplt.rcParams[\"xtick.labelsize\"] = 12\nplt.rcParams[\"ytick.labelsize\"] = 10\n</code></pre> <pre><code>random_state = 24\nis_CI = os.environ.get(\"CI\")\n</code></pre> <pre><code>random.seed(random_state)\nnp.random.seed(random_state)\n</code></pre> <pre><code>training_data, val_data, test_data, feature_names = load_wine_dataset(\ntrain_size=0.3, test_size=0.6\n)\n# In CI we only use a subset of the training set\nif is_CI:\ntrain_data = (training_data[0][:10], training_data[1][:10])\n</code></pre> <p>We will corrupt some of the training points by flipping their labels</p> <pre><code>num_corrupted_idxs = 10\ntraining_data[1][:num_corrupted_idxs] = torch.tensor(\n[(val + 1) % 3 for val in training_data[1][:num_corrupted_idxs]]\n)\n</code></pre> <p>and let's wrap it in a pytorch data loader</p> <pre><code>training_data_loader = DataLoader(\nTensorDataset(*training_data), batch_size=32, shuffle=False\n)\nval_data_loader = DataLoader(TensorDataset(*val_data), batch_size=32, shuffle=False)\ntest_data_loader = DataLoader(TensorDataset(*test_data), batch_size=32, shuffle=False)\n</code></pre> <pre><code>feature_dimension = 13\nnum_classes = 3\nnetwork_size = [16, 16]\nlayers_size = [feature_dimension, *network_size, num_classes]\nnum_epochs = 300\nlr = 0.005\nweight_decay = 0.01\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nnn_model = TorchMLP(layers_size)\nnn_model.to(device)\noptimizer = Adam(params=nn_model.parameters(), lr=lr, weight_decay=weight_decay)\nscheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs)\nlosses = fit_torch_model(\nmodel=nn_model,\ntraining_data=training_data_loader,\nval_data=val_data_loader,\nloss=F.cross_entropy,\noptimizer=optimizer,\nscheduler=scheduler,\nnum_epochs=num_epochs,\n)\n</code></pre> <pre>\n<code>Model fitting: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 300/300 [00:00&lt;00:00, 307.77it/s]\n</code>\n</pre> <p>Let's check that the training has found a stable minimum by plotting the training and validation loss</p> <pre><code>plot_losses(losses)\n</code></pre> <p>Since it is a classification problem, let's also take a look at the confusion matrix on the test set</p> <pre><code>nn_model.eval()\npred_y_test = np.argmax(nn_model(test_data[0]).detach(), axis=1)\ncm = confusion_matrix(test_data[1], pred_y_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm)\ndisp.plot();\n</code></pre> <p>And let's compute the f1 score of the model</p> <pre><code>f1_score(test_data[1], pred_y_test, average=\"weighted\")\n</code></pre> <pre>\n<code>0.9906846833902615</code>\n</pre> <p>Let's now move to calculating influences of each point on the total score.</p> <pre><code>train_influences = compute_influences(\nTorchTwiceDifferentiable(nn_model, F.cross_entropy),\ntraining_data=training_data_loader,\ntest_data=test_data_loader,\ninfluence_type=\"up\",\ninversion_method=\"direct\",\nhessian_regularization=0.1,\nprogress=True,\n)\n</code></pre> <pre>\n<code>Batch Test Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:00&lt;00:00, 67.10it/s]\nMVP: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 547/547 [00:00&lt;00:00, 742.01it/s] \nBatch Split Input Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 2/2 [00:00&lt;00:00, 85.02it/s]\n</code>\n</pre> <p>the returned matrix, train_influences, has a quantity of columns equal to the points in the training set, and a number of rows equal to the points in the test set. At each element \\(a_{i,j}\\) it stores the influence that training point \\(j\\) has on the classification of test point \\(i\\).</p> <p>If we take the average across every column of the influences matrix, we obtain an estimate of the overall influence of a training point on the total accuracy of the network.</p> <pre><code>mean_train_influences = np.mean(train_influences.numpy(), axis=0)\n</code></pre> <p>The following histogram shows that there are big differences in score within the training set (notice the log-scale on the y axis).</p> <pre><code>_, ax = plt.subplots()\nax.hist(mean_train_influences[num_corrupted_idxs:], label=\"normal\")\nax.hist(mean_train_influences[:num_corrupted_idxs], label=\"corrupted\", bins=5)\nax.set_title(\"Influece scores distribution\")\nax.set_xlabel(\"influece score\")\nax.set_ylabel(\"number of points\")\nax.legend()\nplt.show()\n</code></pre> <p>We can see that the corrupted points tend to have a negative effect on the model, as expected</p> <pre><code>print(\n\"Average influence of corrupted points: \",\nnp.mean(mean_train_influences[:num_corrupted_idxs]),\n)\nprint(\n\"Average influence of other points: \",\nnp.mean(mean_train_influences[num_corrupted_idxs:]),\n)\n</code></pre> <pre>\n<code>Average influence of corrupted points:  -0.05317057\nAverage influence of other points:  0.034408495\n</code>\n</pre> <p>We have seen how to calculate the influence of single training points on each test point using influence_type 'up'. Using influence_type 'perturbation' we can also calculate the influence of the input features of each point. In the next cell we will calculate the average influence of each feature on training and test points, and ultimately assess which are the most relevant to model performance.</p> <pre><code>feature_influences = compute_influences(\nTorchTwiceDifferentiable(nn_model, F.cross_entropy),\ntraining_data=training_data_loader,\ntest_data=test_data_loader,\ninfluence_type=\"perturbation\",\ninversion_method=\"direct\",\nhessian_regularization=1,\nprogress=True,\n)\n</code></pre> <pre>\n<code>Batch Test Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:00&lt;00:00, 61.20it/s]\nMVP: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 547/547 [00:00&lt;00:00, 1265.72it/s]\nBatch Influence Perturbation: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 2/2 [00:03&lt;00:00,  1.66s/it]\n</code>\n</pre> <pre><code>mean_feature_influences = np.mean(feature_influences.numpy(), axis=(0, 1))\n_, ax = plt.subplots()\nax.bar(feature_names, mean_feature_influences)\nax.set_xlabel(\"training features\")\nax.set_ylabel(\"influence values\")\nax.set_title(\"Average feature influence\")\nplt.xticks(rotation=60)\nplt.show()\n</code></pre> <p>The calculation of the Hessian matrix (necessary to calculate the influences) can be quite numerically challenging, but there are some techniques to speed up its calculation. PyDVL allows to use the full method (\"direct\") or the conjugate gradient method (\"cg\"). The first one should be used only for very small networks (like our current example), while for bigger ones \"cg\" is advisable.</p> <pre><code>cg_train_influences = compute_influences(\nTorchTwiceDifferentiable(nn_model, F.cross_entropy),\ntraining_data=training_data_loader,\ntest_data=test_data_loader,\ninfluence_type=\"up\",\ninversion_method=\"cg\",\nhessian_regularization=0.1,\nprogress=True,\n)\nmean_cg_train_influences = np.mean(cg_train_influences.numpy(), axis=0)\n</code></pre> <pre>\n<code>Batch Test Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:00&lt;00:00, 81.02it/s]\nBatch Train Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 2/2 [00:00&lt;00:00, 535.33it/s]\nConjugate gradient: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 107/107 [00:04&lt;00:00, 22.66it/s]\nBatch Split Input Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 2/2 [00:00&lt;00:00, 98.91it/s]\n</code>\n</pre> <p>Let's compare the results obtained through conjugate gradient with those from the direct method</p> <pre><code>print(\nf\"Percentage error of cg over direct method:{np.mean(np.abs(mean_cg_train_influences - mean_train_influences)/np.abs(mean_train_influences))*100} %\"\n)\n</code></pre> <pre>\n<code>Percentage error of cg over direct method:1.5124550145628746e-05 %\n</code>\n</pre> <p>This was a quick introduction to the pyDVL interface for influence functions. Despite their speed and simplicity, influence functions are known to be a very noisy estimator of data quality, as pointed out in the paper \"Influence functions in deep learning are fragile\". The size of the network, the weight decay, the inversion method used for calculating influences, the size of the test set: they all add up to the total amount of noise. Experiments may therefore give quantitative and qualitatively different results if not averaged across several realisations. Shapley values, on the contrary, have shown to be a more robust, but this comes at the cost of high computational requirements. PyDVL employs several parallelization and caching techniques to optimize such calculations.</p>"},{"location":"examples/influence_wine/#influence-functions-for-outlier-detection","title":"Influence functions for outlier detection","text":"<p>This notebook shows how to calculate influences on a NN model using pyDVL for an arbitrary dataset, and how this can be used to find anomalous or corrupted data points.</p> <p>It uses the wine dataset from sklearn: given a set of 13 different input parameters regarding a particular bottle, each related to some physical property (e.g. concentration of magnesium, malic acidity, alcoholic percentage, etc.), the model will need to predict to which of 3 classes the wine belongs to. For more details, please refer to the sklearn documentation.</p>"},{"location":"examples/influence_wine/#imports","title":"Imports","text":""},{"location":"examples/influence_wine/#constants","title":"Constants","text":""},{"location":"examples/influence_wine/#dataset","title":"Dataset","text":""},{"location":"examples/influence_wine/#fit-a-neural-network-to-the-data","title":"Fit a neural network to the data","text":"<p>We will train a 2-layer neural network. PyDVL has some convenience wrappers to initialize a pytorch NN. If you already have a model loaded and trained, you can skip this section.</p>"},{"location":"examples/influence_wine/#calculating-influences-for-small-neural-networks","title":"Calculating influences for small neural networks","text":"<p>The following cell calculates the influences of each training data point on the neural network.  Neural networks have typically a very bumpy parameter space, which, during training, is explored until the configuration that minimises the loss is found. There is an important assumption in influence functions that the model lays at a (at least local) minimum of such loss, and if this is not fulfilled many issues can arise. In order to avoid this scenario, a regularisation term should be used whenever dealing with big and noisy models. </p>"},{"location":"examples/influence_wine/#influence-of-training-features","title":"Influence of training features","text":""},{"location":"examples/influence_wine/#speeding-up-influences-for-big-models","title":"Speeding up influences for big models","text":""},{"location":"examples/least_core_basic/","title":"Least Core","text":"<p>We will be using the following functions and classes from pyDVL.</p> <pre><code>%autoreload\nfrom pydvl.utils import (\nDataset,\nUtility,\n)\nfrom pydvl.value import compute_least_core_values, LeastCoreMode, ValuationResult\nfrom pydvl.reporting.plots import shaded_mean_std\nfrom pydvl.reporting.scores import compute_removal_score\n</code></pre> <pre><code>X, y = make_classification(\nn_samples=200,\nn_features=50,\nn_informative=25,\nn_classes=3,\nrandom_state=random_state,\n)\n</code></pre> <pre><code>full_dataset = Dataset.from_arrays(\nX, y, stratify_by_target=True, random_state=random_state\n)\nsmall_dataset = Dataset.from_arrays(\nX,\ny,\nstratify_by_target=True,\ntrain_size=10,\nrandom_state=random_state,\n)\n</code></pre> <pre><code>model = LogisticRegression(max_iter=500, solver=\"liblinear\")\n</code></pre> <pre><code>model.fit(full_dataset.x_train, full_dataset.y_train)\nprint(\nf\"Training accuracy: {100 * model.score(full_dataset.x_train, full_dataset.y_train):0.2f}%\"\n)\nprint(\nf\"Testing accuracy: {100 * model.score(full_dataset.x_test, full_dataset.y_test):0.2f}%\"\n)\n</code></pre> <pre>\n<code>Training accuracy: 86.25%\nTesting accuracy: 70.00%\n</code>\n</pre> <pre><code>model.fit(small_dataset.x_train, small_dataset.y_train)\nprint(\nf\"Training accuracy: {100 * model.score(small_dataset.x_train, small_dataset.y_train):0.2f}%\"\n)\nprint(\nf\"Testing accuracy: {100 * model.score(small_dataset.x_test, small_dataset.y_test):0.2f}%\"\n)\n</code></pre> <pre>\n<code>Training accuracy: 100.00%\nTesting accuracy: 47.89%\n</code>\n</pre> <pre><code>utility = Utility(model=model, data=small_dataset)\n</code></pre> <pre><code>exact_values = compute_least_core_values(\nu=utility,\nmode=LeastCoreMode.Exact,\nprogress=True,\n)\n</code></pre> <pre>\n<code>  0%|          | 0/1023 [00:00&lt;?, ?it/s]</code>\n</pre> <pre><code>exact_values_df = exact_values.to_dataframe(column=\"exact_value\").T\nexact_values_df = exact_values_df[sorted(exact_values_df.columns)]\n</code></pre> <pre><code>budget_array = np.linspace(200, 2 ** len(small_dataset), num=10, dtype=int)\nall_estimated_values_df = []\nall_errors = {budget: [] for budget in budget_array}\nfor budget in tqdm(budget_array):\ndfs = []\nerrors = []\ncolumn_name = f\"estimated_value_{budget}\"\nfor i in range(20):\nvalues = compute_least_core_values(\nu=utility,\nmode=LeastCoreMode.MonteCarlo,\nn_iterations=budget,\nn_jobs=n_jobs,\n)\ndf = (\nvalues.to_dataframe(column=column_name)\n.drop(columns=[f\"{column_name}_stderr\"])\n.T\n)\ndf = df[sorted(df.columns)]\nerror = mean_squared_error(\nexact_values_df.loc[\"exact_value\"].values, df.values.ravel()\n)\nall_errors[budget].append(error)\ndf[\"budget\"] = budget\ndfs.append(df)\nestimated_values_df = pd.concat(dfs)\nall_estimated_values_df.append(estimated_values_df)\nvalues_df = pd.concat(all_estimated_values_df)\nerrors_df = pd.DataFrame(all_errors)\n</code></pre> <pre>\n<code>  0%|          | 0/10 [00:00&lt;?, ?it/s]</code>\n</pre> <p>We can see that the approximation error decreases, on average, as the we increase the budget. </p> <p>Still, the decrease may not always necessarily happen when we increase the number of iterations because of the fact that we sample the subsets with replacement in the Monte Carlo method i.e there may be repeated subsets.</p> <pre><code>utility = Utility(model=model, data=full_dataset)\n</code></pre> <pre><code>method_names = [\"Random\", \"Least Core\"]\nremoval_percentages = np.arange(0, 0.41, 0.05)\n</code></pre> <pre><code>all_scores = []\nfor i in trange(5):\nfor method_name in method_names:\nif method_name == \"Random\":\nvalues = ValuationResult.from_random(size=len(utility.data))\nelse:\nvalues = compute_least_core_values(\nu=utility,\nmode=LeastCoreMode.MonteCarlo,\nn_iterations=25000,\nn_jobs=n_jobs,\n)\nscores = compute_removal_score(\nu=utility,\nvalues=values,\npercentages=removal_percentages,\nremove_best=True,\n)\nscores[\"method_name\"] = method_name\nall_scores.append(scores)\nscores_df = pd.DataFrame(all_scores)\n</code></pre> <pre>\n<code>  0%|          | 0/5 [00:00&lt;?, ?it/s]</code>\n</pre> <p>We can clearly see that removing the most valuable data points, as given by the Least Core method, leads to, on average, a decrease in the model's performance and that the method outperforms random removal of data points.</p> <pre><code>all_scores = []\nfor i in trange(5):\nfor method_name in method_names:\nif method_name == \"Random\":\nvalues = ValuationResult.from_random(size=len(utility.data))\nelse:\nvalues = compute_least_core_values(\nu=utility,\nmode=LeastCoreMode.MonteCarlo,\nn_iterations=25000,\nn_jobs=n_jobs,\n)\nscores = compute_removal_score(\nu=utility,\nvalues=values,\npercentages=removal_percentages,\n)\nscores[\"method_name\"] = method_name\nall_scores.append(scores)\nscores_df = pd.DataFrame(all_scores)\n</code></pre> <pre>\n<code>  0%|          | 0/5 [00:00&lt;?, ?it/s]</code>\n</pre> <p>We can clearly see that removing the least valuable data points, as given by the Least Core method, leads to, on average, an increase in the model's performance and that the method outperforms the random removal of data points.</p>"},{"location":"examples/least_core_basic/#least-core-for-data-valuation","title":"Least Core for Data Valuation","text":"<p>This notebook introduces Least Core methods for the computation of data values using pyDVL.</p> <p>Shapley values define a fair way of distributing the worth of the whole training set when every data point is part of it. But they do not consider the question of stability of subsets: Could some data points obtain a higher payoff if they formed smaller subsets? It is argued that this might be relevant if data providers are paid based on data value, since Shapley values can incentivise them not to contribute their data to the \"grand coalition\", but instead try to form smaller ones. Whether this is of actual practical relevance is debatable, but in any case, the least core is an alternative tool available for any task of Data Valuation</p> <p>The Core is another approach to compute data values originating in cooperative game theory that attempts to answer those questions. It is the set of feasible payoffs that cannot be improved upon by a coalition of the participants.</p> <p>Its use for Data Valuation was first described in the paper If You Like Shapley Then You\u2019ll Love the Core by Tom Yan and Ariel D. Procaccia.</p> <p>The Least Core value \\(v\\) of the \\(i\\)-th sample in dataset \\(D\\) wrt. utility \\(u\\) is computed by solving the following Linear Program:</p> \\[ \\begin{array}{lll} \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\ \\text{subject to} &amp; \\displaystyle\\sum_{x_i\\in D} v_u(x_i) = u(D) &amp; \\\\ &amp; \\displaystyle\\sum_{x_i\\in S} v_u(x_i) + e \\geq u(S) &amp;, \\forall S \\subset D, S \\neq \\emptyset \\\\ \\end{array} \\] <p>To illustrate this method we will use a synthetic dataset. We will first use a subset of 10 data point to compute the exact values and use them to assess the Monte Carlo approximation. Afterwards, we will conduct the data removal experiments as described by Ghorbani and Zou in their paper Data Shapley: Equitable Valuation of Data for Machine Learning: We compute the data valuation given different computation budgets and incrementally remove a percentage of the best, respectively worst, data points and observe how that affects the utility.</p>"},{"location":"examples/least_core_basic/#setup","title":"Setup","text":"<p>We begin by importing the main libraries and setting some defaults.</p>   If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/least_core_basic/#dataset","title":"Dataset","text":"<p>We generate a synthetic dataset using the <code>make_classification</code> function from scikit-learn.</p> <p>We sample 200 data points from a 50-dimensional Gaussian distribution with 25 informative features and 25 non-informative features (generated as random linear combinations of the informative features).</p> <p>The 200 samples are uniformly distributed across 3 classes with a small percentage of noise added to the labels to make the task a bit more difficult.</p>"},{"location":"examples/least_core_basic/#estimating-least-core-values","title":"Estimating Least Core Values","text":"<p>In this first section we will use a smaller subset of the dataset containing 10 samples in order to be able to compute exact values in a reasonable amount of time. Afterwards, we will use the Monte Carlo method with a limited budget (maximum number of subsets) to approximate these values.</p>"},{"location":"examples/least_core_basic/#data-removal","title":"Data Removal","text":"<p>We now move on to the data removal experiments using the full dataset.</p> <p>In these experiments, we first rank the data points from most valuable  to least valuable using the values estimated by the Monte Carlo Least Core method. Then, we gradually remove from 5 to 40 percent, by increments of 5 percentage points, of the most valuable/least valuable ones, train the model on this subset and compute its accuracy.</p>"},{"location":"examples/least_core_basic/#remove-best","title":"Remove Best","text":"<p>We start by removing the best data points and seeing how the model's accuracy evolves.</p>"},{"location":"examples/least_core_basic/#remove-worst","title":"Remove Worst","text":"<p>We then proceed to removing the worst data points and seeing how the model's accuracy evolves.</p>"},{"location":"examples/shapley_basic_spotify/","title":"Shapley values","text":"<p>This notebook introduces Shapley methods for the computation of data value using pyDVL.</p> <p>In order to illustrate the practical advantages, we will predict the popularity of songs in the dataset Top Hits Spotify from 2000-2019, and highlight how data valuation can help investigate and boost the performance of the models. In doing so, we will describe the basic usage patterns of pyDVL.</p> <p>Recall that data value is a function of three things:</p> <ol> <li>The dataset.</li> <li>The model.</li> <li>The performance metric or scoring function.</li> </ol> <p>Below we will describe how to instantiate each one of these objects and how to use them for data valuation. Please also see the documentation on data valuation.</p> <p>We will be using the following functions from pyDVL. The main entry point is the function compute_shapley_values(), which provides a facade to all Shapley methods. In order to use it we need the classes Dataset, Utility and Scorer.</p> <pre><code>%autoreload\nfrom pydvl.reporting.plots import plot_shapley\nfrom pydvl.utils.dataset import GroupedDataset\nfrom support.shapley import load_spotify_dataset\nfrom pydvl.value import *\n</code></pre> <pre><code>training_data, val_data, test_data = load_spotify_dataset(\nval_size=0.3, test_size=0.3, target_column=\"popularity\", random_state=random_state\n)\n</code></pre> <pre><code>training_data[0].head()\n</code></pre> artist song duration_ms explicit year danceability energy key loudness mode speechiness acousticness instrumentalness liveness valence tempo genre 1561 Fetty Wap 679 (feat. Remy Boyz) 196693 True 2015 0.618 0.717 7 -5.738 1 0.3180 0.00256 0.000000 0.6250 0.603 190.050 8 1410 Meghan Trainor All About That Bass 187920 True 2015 0.807 0.887 9 -3.726 1 0.0503 0.05730 0.000003 0.1240 0.961 134.052 14 1772 Katy Perry Chained To The Rhythm 237733 False 2017 0.562 0.800 0 -5.404 1 0.1120 0.08140 0.000000 0.1990 0.471 95.029 14 1670 Sigala Sweet Lovin' - Radio Edit 202149 False 2015 0.683 0.910 10 -1.231 1 0.0515 0.05530 0.000005 0.3360 0.674 124.977 15 1780 Liam Payne Strip That Down 204502 False 2017 0.869 0.485 6 -5.595 1 0.0545 0.24600 0.000000 0.0765 0.527 106.028 14 <p>The dataset has many high-level features, some quite intuitive ('duration_ms' or 'tempo'), while others are a bit more cryptic ('valence'?). For information on each feature, please consult the dataset's website.</p> <p>In our analysis, we will use all the columns, except for 'artist' and 'song', to predict the 'popularity' of each song. We will nonetheless keep the information on song and artist in a separate object for future reference.</p> <pre><code>song_name = training_data[0][\"song\"]\nartist = training_data[0][\"artist\"]\ntraining_data[0] = training_data[0].drop([\"song\", \"artist\"], axis=1)\ntest_data[0] = test_data[0].drop([\"song\", \"artist\"], axis=1)\nval_data[0] = val_data[0].drop([\"song\", \"artist\"], axis=1)\n</code></pre> <p>Input and label data are then used to instantiate a Dataset object:</p> <pre><code>dataset = Dataset(*training_data, *val_data)\n</code></pre> <p>The calculation of exact Shapley values is computationally very expensive (exponentially so!) because it requires training the model on every possible subset of the training set. For this reason, PyDVL implements techniques to speed up the calculation, such as Monte Carlo approximations, surrogate models or caching of intermediate results and grouping of data to calculate group Shapley values instead of single data points.</p> <p>In our case, we will group songs by artist and calculate the Shapley value for the artists. Given the pandas Series for 'artist', to group the dataset by it, one does the following:</p> <pre><code>grouped_dataset = GroupedDataset.from_dataset(dataset=dataset, data_groups=artist)\n</code></pre> <pre><code>utility = Utility(\nmodel=GradientBoostingRegressor(n_estimators=3),\ndata=grouped_dataset,\nscorer=Scorer(\"neg_mean_absolute_error\", default=0.0),\n)\nvalues = compute_shapley_values(\nutility,\nmode=ShapleyMode.TruncatedMontecarlo,\n# Stop if the standard error is below 1% of the range of the values (which is ~2),\n# or if the number of updates exceeds 1000\ndone=AbsoluteStandardError(threshold=0.2, fraction=0.9) | MaxUpdates(1000),\ntruncation=RelativeTruncation(utility, rtol=0.01),\nn_jobs=-1,\n)\nvalues.sort(key=\"value\")\ndf = values.to_dataframe(column=\"data_value\", use_names=True)\n</code></pre> <p>The function compute_shapley_values() serves as a common access point to all Shapley methods. For most of them, we must choose a <code>StoppingCriterion</code> with the argument <code>done=</code>. In this case we choose to stop when the ratio of standard error to value is below 0.2 for at least 90% of the training points, or if the number of updates of any index exceeds 1000. The <code>mode</code> argument specifies the Shapley method to use. In this case, we use the Truncated Monte Carlo approximation, which is the fastest of the Monte Carlo methods, owing both to using the permutation definition of Shapley values and the ability to truncate the iteration over a given permutation. We configure this to happen when the contribution of the remaining elements is below 1% of the total utility with the parameter <code>truncation=</code> and the policy RelativeTruncation.</p> <p>Let's take a look at the returned dataframe:</p> <pre><code>df.head()\n</code></pre> data_value data_value_stderr Kendrick Lamar -1.279149 0.091670 BLACKPINK -1.277363 0.177476 Adele -1.241698 0.183732 5 Seconds of Summer -1.228002 0.103377 Flume -1.197065 0.102345 <p>The first thing to notice is that we sorted the results in ascending order of Shapley value. The index holds the labels for each data group: in this case, artist names. The column <code>data_value</code> is just that: the Shapley Data value, and <code>data_value_stderr</code> is its estimated standard error because we are using a Monte Carlo approximation.</p> <p>Let us plot the results. In the next cell we will take the 30 artists with the lowest score and plot their values with 95% Normal confidence intervals. Keep in mind that Monte Carlo Shapley is typically very noisy, and it can take many steps to arrive at a clean estimate.</p> <p>We can immediately see that many artists (groups of samples) have very low, even negative value, which means that they tend to decrease the total score of the model when present in the training set! What happens if we remove them?</p> <p>In the next cell we create a new training set excluding the artists with the lowest scores:</p> <pre><code>low_dvl_artists = df.iloc[:30].index.to_list()\nartist_filter = ~artist.isin(low_dvl_artists)\nX_train_good_dvl = training_data[0][artist_filter]\ny_train_good_dvl = training_data[1][artist_filter]\n</code></pre> <p>Now we will use this \"cleaned\" dataset to retrain the same model and compare its mean absolute error to the one trained on the full dataset. Notice that the score now is calculated using the test set, while in the calculation of the Shapley values we were using the validation set.</p> <pre><code>model_good_data = GradientBoostingRegressor(n_estimators=3).fit(\nX_train_good_dvl, y_train_good_dvl\n)\nerror_good_data = mean_absolute_error(\nmodel_good_data.predict(test_data[0]), test_data[1]\n)\nmodel_all_data = GradientBoostingRegressor(n_estimators=3).fit(\ntraining_data[0], training_data[1]\n)\nerror_all_data = mean_absolute_error(model_all_data.predict(test_data[0]), test_data[1])\nprint(f\"Improvement: {100*(error_all_data - error_good_data)/error_all_data:02f}%\")\n</code></pre> <pre>\n<code>Improvement: 13.940685%\n</code>\n</pre> <p>The score has improved by almost 14%! This is quite an important result, as it shows a consistent process to improve the performance of a model by excluding data points from its training set.</p>   One must however proceed with caution instead of simply throwing away data. For one, `mean_absolute_error` is an estimate of generalization error on unseen data, so the improvement we see on the test set might not be as large upon deployment. It would be advisable to cross-validate this whole process to obtain more conservative estimates. It is also advisable to manually inspect the artists with low value and to try to understand the reason why the model behaves like it does. Finally, remember that **the value depends on the model chosen**! Artists that are detrimental to the Gradient Boosting Regressor might be informative for a different model (although it is likely that the worst ones share some characteristic making them \"bad\" for other regressors).   <p>Let us take all the songs by Rihanna, set their score to 0 and re-calculate the Shapley values.</p> <pre><code>y_train_anomalous = training_data[1].copy(deep=True)\ny_train_anomalous[artist == \"Rihanna\"] = 0\nanomalous_dataset = Dataset(\nx_train=training_data[0],\ny_train=y_train_anomalous,\nx_test=val_data[0],\ny_test=val_data[1],\n)\ngrouped_anomalous_dataset = GroupedDataset.from_dataset(anomalous_dataset, artist)\nanomalous_utility = Utility(\nmodel=GradientBoostingRegressor(n_estimators=3),\ndata=grouped_anomalous_dataset,\nscorer=Scorer(\"neg_mean_absolute_error\", default=0.0),\n)\nvalues = compute_shapley_values(\nanomalous_utility,\nmode=ShapleyMode.TruncatedMontecarlo,\ndone=AbsoluteStandardError(threshold=0.2, fraction=0.9) | MaxUpdates(1000),\nn_jobs=-1,\n)\nvalues.sort(key=\"value\")\ndf = values.to_dataframe(column=\"data_value\", use_names=True)\n</code></pre> <p>Let us now consider the low-value artists (at least for predictive purposes, no claims are made about their artistic value!) and plot the results</p> <p>And Rihanna (our anomalous data group) has moved from top contributor to having negative impact on the performance of the model, as expected!</p> <p>What is going on? A popularity of 0 for Rihanna's songs is inconsistent with listening patterns for other artists. In artificially setting this, we degrade the predictive power of the model.</p> <p>By dropping low-value groups or samples, one can often increase model performance, but by inspecting them, it is possible to identify bogus data sources or acquisition methods.</p>"},{"location":"examples/shapley_basic_spotify/#shapley-for-data-valuation","title":"Shapley for data valuation","text":""},{"location":"examples/shapley_basic_spotify/#setup","title":"Setup","text":"<p>We begin by importing the main libraries and setting some defaults.</p>   If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/shapley_basic_spotify/#loading-and-grouping-the-dataset","title":"Loading and grouping the dataset","text":"<p>pyDVL provides a support function for this notebook, <code>load_spotify_dataset()</code>, which downloads data on songs published after 2014, and splits 30% of data for testing, and 30% of the remaining data for validation. The return value is a triple of training, validation and test data as lists of the form <code>[X_input, Y_label]</code>.</p>"},{"location":"examples/shapley_basic_spotify/#creating-the-utility-and-computing-values","title":"Creating the utility and computing values","text":"<p>Now we can calculate the contribution of each group to the model performance.</p> <p>As a model, we use scikit-learn's GradientBoostingRegressor, but pyDVL can work with any model from sklearn, xgboost or lightgbm. More precisely, any model that implements the protocol pydvl.utils.types.SupervisedModel, which is just the standard sklearn interface of <code>fit()</code>,<code>predict()</code> and <code>score()</code> can be used to construct the utility.</p> <p>The third and final component is the scoring function. It can be anything like accuracy or \\(R^2\\), and is set with a string from the standard sklearn scoring methods. Please refer to that documentation on information on how to define your own scoring function.</p> <p>We group dataset, model and scoring function into an instance of Utility.</p>"},{"location":"examples/shapley_basic_spotify/#evaluation-on-anomalous-data","title":"Evaluation on anomalous data","text":"<p>One interesting test is to corrupt some data and to monitor how their value changes. To do this, we will take one of the artists with the highest value and set the popularity of all their songs to 0.</p>"},{"location":"examples/shapley_knn_flowers/","title":"KNN Shapley","text":"<p>This notebook shows how to calculate Shapley values for the K-Nearest Neighbours algorithm. By making use of the local structure of KNN, it is possible to compute an exact value in almost linear time, as opposed to exponential complexity of exact, model-agnostic Shapley.</p> <p>The main idea is to exploit the fact that adding or removing points beyond the k-ball doesn't influence the score. Because the algorithm then essentially only needs to do a search it runs in \\(\\mathcal{O}(N \\log N)\\) time.</p> <p>By further using approximate nearest neighbours, it is possible to achieve \\((\\epsilon,\\delta)\\)-approximations in sublinear time. However, this is not implemented in pyDVL yet.</p> <p>We refer to the original paper that pyDVL implements for details: Jia, Ruoxi, David Dao, Boxin Wang, Frances Ann Hubis, Nezihe Merve Gurel, Bo Li, Ce Zhang, Costas Spanos, and Dawn Song. Efficient Task-Specific Data Valuation for Nearest Neighbor Algorithms. Proceedings of the VLDB Endowment 12, no. 11 (1 July 2019): 1610\u201323.</p> <p>The main entry point is the function compute_shapley_values(), which provides a facade to all Shapley methods. In order to use it we need the classes Dataset, Utility and Scorer, all of which can be imported from <code>pydvl.value</code>:</p> <pre><code>from pydvl.value import *\n</code></pre> <pre><code>sklearn_dataset = sk.datasets.load_iris()\ndata = Dataset.from_sklearn(sklearn_dataset)\nknn = sk.neighbors.KNeighborsClassifier(n_neighbors=5)\nutility = Utility(knn, data)\n</code></pre> <pre><code>shapley_values = compute_shapley_values(utility, mode=ShapleyMode.KNN, progress=True)\nshapley_values.sort(key=\"value\")\nvalues = shapley_values.values\n</code></pre> <pre>\n<code>0it [00:00, ?it/s]</code>\n</pre> <p>If we now look at  the distribution of Shapley values for each class, we see that each has samples with both high and low scores. This is expected, because an accurate model uses information of all classes.</p> <pre><code>corrupted_data = deepcopy(data)\nn_corrupted = 10\ncorrupted_data.y_train[:n_corrupted] = (corrupted_data.y_train[:n_corrupted] + 1) % 3\nknn = sk.neighbors.KNeighborsClassifier(n_neighbors=5)\ncontaminated_values = compute_shapley_values(\nUtility(knn, corrupted_data), mode=ShapleyMode.KNN\n)\n</code></pre> <p>Taking the average corrupted value and comparing it to non-corrupted ones, we notice that on average anomalous points have a much lower score, i.e. they tend to be much less valuable to the model.</p> <p>To do this, first we make sure that we access the results by data index with a call to <code>ValuationResult.sort()</code>, then we split the values into two groups: corrupted and non-corrupted. Note how we access property <code>values</code> of the <code>ValuationResult</code> object. This is a numpy array of values, sorted however the object was sorted. Finally, we compute the quantiles of the two groups and compare them. We see that the corrupted mean is in the lowest percentile of the value distribution, while the correct mean is in the 70th percentile.</p> <pre><code>contaminated_values.sort(\nkey=\"index\"\n)  # This is redundant, but illustrates sorting, which is in-place\ncorrupted_shapley_values = contaminated_values.values[:n_corrupted]\ncorrect_shapley_values = contaminated_values.values[n_corrupted:]\nmean_corrupted = np.mean(corrupted_shapley_values)\nmean_correct = np.mean(correct_shapley_values)\npercentile_corrupted = np.round(100 * np.mean(values &amp;lt; mean_corrupted), 0)\npercentile_correct = np.round(100 * np.mean(values &amp;lt; mean_correct), 0)\nprint(\nf\"The corrupted mean is at percentile {percentile_corrupted:.0f} of the value distribution.\"\n)\nprint(\nf\"The correct mean is percentile {percentile_correct:.0f} of the value distribution.\"\n)\n</code></pre> <pre>\n<code>The corrupted mean is at percentile 1 of the value distribution.\nThe correct mean is percentile 71 of the value distribution.\n</code>\n</pre> <p>This is confirmed if we plot the distribution of Shapley values and circle corrupt points in red. They all tend to have low Shapley scores, regardless of their position in space and assigned label:</p>"},{"location":"examples/shapley_knn_flowers/#knn-shapley","title":"KNN Shapley","text":""},{"location":"examples/shapley_knn_flowers/#setup","title":"Setup","text":"<p>We begin by importing the main libraries and setting some defaults.</p>   If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/shapley_knn_flowers/#building-a-dataset-and-a-utility","title":"Building a Dataset and a Utility","text":"<p>We use the sklearn iris dataset and wrap it into a pydvl.utils.dataset.Dataset calling the factory pydvl.utils.dataset.Dataset.from_sklearn(). This automatically creates a train/test split for us which will be used to compute the utility.</p> <p>We then create a model and instantiate a Utility using data and model. The model needs to implement the protocol pydvl.utils.types.SupervisedModel, which is just the standard sklearn interface of <code>fit()</code>,<code>predict()</code> and <code>score()</code>. In constructing the <code>Utility</code> one can also choose a scoring function, but we pick the default which is just the model's <code>knn.score()</code>.</p>"},{"location":"examples/shapley_knn_flowers/#computing-values","title":"Computing values","text":"<p>Calculating the Shapley values is straightforward. We just call compute_shapley_values() with the utility object we created above. The function returns a ValuationResult. This object contains the values themselves, data indices and labels.</p>"},{"location":"examples/shapley_knn_flowers/#inspecting-the-results","title":"Inspecting the results","text":"<p>Let us first look at the labels' distribution as a function of petal and sepal length:</p>"},{"location":"examples/shapley_knn_flowers/#corrupting-labels","title":"Corrupting labels","text":"<p>To test how informative values are, we can corrupt some training labels and see how their Shapley values change with respect to the non-corrupted points.</p>"},{"location":"examples/shapley_utility_learning/","title":"Data utility learning","text":"<p>This notebook introduces Data Utility Learning, a method of approximating Data Shapley values by learning to estimate the utility function.</p> <p>The idea is to employ a model to learn the performance of the learning algorithm of interest on unseen data combinations (i.e. subsets of the dataset). The method was originally described in Wang, Tianhao, Yu Yang, and Ruoxi Jia. Improving Cooperative Game Theory-Based Data Valuation via Data Utility Learning. arXiv, 2022.</p> <p>Warning: Work on Data Utility Learning is preliminary. It remains to be seen when or whether it can be put effectively into application. For this further testing and benchmarking are required.</p> <p>Recall the definition of Shapley value \\(v_u(i)\\) for data point \\(i\\):</p> \\[\\begin{equation} v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}} \\binom{n-1}{|S|}^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)] , \\tag{1} \\label{eq:shapley-def} \\end{equation}\\] <p>where \\(N\\) is the set of all indices in the training set and \\(u\\) is the utility.</p> <p>In Data Utility Learning, to avoid the exponential cost of computing this sum, one learns a surrogate model for \\(u\\). We start by sampling so-called utility samples to form a training set \\(S_\\mathrm{train}\\) for our utility model. Each utility sample is a tuple consisting of a subset of indices \\(S_j\\) in the dataset and its utility \\(u(S_j)\\):</p> \\[\\mathcal{S}_\\mathrm{train} = \\{(S_j, u(S_j): j = 1 , ..., m_\\mathrm{train}\\}\\] <p>where \\(m_\\mathrm{train}\\) denotes the training budget for the learned utility function.</p> <p>The subsets are then transformed into boolean vectors \\(\\phi\\) in which a \\(1\\) at index \\(k\\) means that the \\(k\\)-th sample of the dataset is present in the subset:</p> \\[S_j \\mapsto \\phi_j \\in \\{ 0, 1 \\}^{N}\\] <p>We fit a regression model \\(\\tilde{u}\\), called data utility model, on the transformed utility samples \\(\\phi (\\mathcal{S}_\\mathrm{train}) := \\{(\\phi(S_j), u(S_j): j = 1 , ..., m_\\mathrm{train}\\}\\) and use it to predict instead of computing the utility for any \\(S_j \\notin \\mathcal{S}_\\mathrm{train}\\). We abuse notation and identify \\(\\tilde{u}\\) with the composition \\(\\tilde{u} \\circ \\phi : N \\rightarrow \\mathbb{R}\\).</p> <p>The main assumption is that it is much faster to fit and use \\(\\tilde{u}\\) than it is to compute \\(u\\) and that for most \\(i\\), \\(v_\\tilde{u}(i) \\approx v_u(i)\\) in some sense.</p> <p>As is the case with all other Shapley methods, the main entry point is the function compute_shapley_values(), which provides a facade to all algorithms in this family. We use it with the usual classes Dataset and Utility. In addition, we must import the core class for learning a utility, DataUtilityLearning.</p> <pre><code>%autoreload\nfrom pydvl.utils import DataUtilityLearning, top_k_value_accuracy\nfrom pydvl.reporting.plots import shaded_mean_std\nfrom pydvl.value import *\n</code></pre> <pre><code>dataset = Dataset.from_sklearn(\nload_iris(), train_size=0.1, random_state=random_state, stratify_by_target=True\n)\n</code></pre> <p>We verify that, as in the paper, if we fit a Support-Vector Classifier to the training data, we obtain an accuracy of around 92%:</p> <pre><code>model = LinearSVC()\nmodel.fit(dataset.x_train, dataset.y_train)\nprint(f\"Mean accuracy: {100 * model.score(dataset.x_test, dataset.y_test):0.2f}%\")\n</code></pre> <pre>\n<code>Mean accuracy: 92.59%\n</code>\n</pre> <pre><code>computation_times = {}\n</code></pre> <pre><code>utility = Utility(model=model, data=dataset)\n</code></pre> <pre><code>start_time = time.monotonic()\nresult = compute_shapley_values(\nu=utility,\nmode=ShapleyMode.CombinatorialExact,\nn_jobs=-1,\nprogress=False,  # Does not display correctly in a notebook\n)\ncomputation_time = time.monotonic() - start_time\ncomputation_times[\"exact\"] = computation_time\ndf = result.to_dataframe(column=\"exact\").drop(columns=[\"exact_stderr\"])\n</code></pre> <p>We now estimate the Data Shapley values using the DataUtilityLearning wrapper. This class wraps a Utility and delegates calls to it, up until a given budget. Every call yields a utility sample which is saved under the hood for training of the given utility model. Once the budget is exhausted, <code>DataUtilityLearning</code> fits the model to the utility samples and all subsequent calls use the learned model to predict the wrapped utility instead of delegating to it.</p> <p>For the utility model we follow the paper and use a fully connected neural network. To train it we use a total of <code>training_budget</code> utility samples. We repeat this multiple times for each training budget.</p>   Note how we use a MonteCarlo approximation instead of `combinatorial_exact` as before. This is because the exact computation samples subsets in a particular order, from the lowest size to the largest. Because the training budget for the model to learn the utility is around 1/4th of the total number of subsets, this would mean that we would never see utility samples for the larger sizes and the model would be biased (try it!)   <pre><code>mlp_kwargs = dict(\nhidden_layer_sizes=(20, 10),\nactivation=\"relu\",\nsolver=\"adam\",\nlearning_rate_init=0.001,\nbatch_size=32,\nmax_iter=800,\n)\nprint(\nf\"Doing {n_runs} runs for each of {len(training_budget_values)} different training budgets.\"\n)\npbar = tqdm(\nproduct(range(n_runs), training_budget_values),\ntotal=n_runs * len(training_budget_values),\n)\nfor idx, budget in pbar:\npbar.set_postfix_str(f\"Run {idx} for training budget: {budget}\")\ndul_utility = DataUtilityLearning(\nu=utility, training_budget=budget, model=MLPRegressor(**mlp_kwargs)\n)\nstart_time = time.monotonic()\n# DUL will kick in after training_budget calls to utility\nresult = compute_shapley_values(\nu=dul_utility,\nmode=ShapleyMode.PermutationMontecarlo,\ndone=MaxUpdates(300),\nn_jobs=-1,\n)\ncomputation_time = time.monotonic() - start_time\nif budget in computation_times:\ncomputation_times[budget].append(computation_time)\nelse:\ncomputation_times[budget] = [computation_time]\ndul_df = result.to_dataframe(column=f\"{budget}_{idx}\").drop(\ncolumns=[f\"{budget}_{idx}_stderr\"]\n)\ndf = pd.concat([df, dul_df], axis=1)\ncomputation_times_df = pd.DataFrame(computation_times)\n</code></pre> <pre>\n<code>Doing 10 runs for each of 10 different training budgets.\n</code>\n</pre> <pre>\n<code>  0%|          | 0/100 [00:00&lt;?, ?it/s]</code>\n</pre> <p>Next we compute the \\(l_1\\) error for the different training budgets across all runs and plot mean and standard deviation. We obtain results analogous to Figure 1 of the paper, verifying that the method indeed works for estimating the Data Shapley values (at least in this context).</p> <p>In the plot we also display the mean and standard deviation of the computation time taken for each training budget. </p> <pre><code>errors = np.zeros((len(training_budget_values), n_runs), dtype=float)\naccuracies = np.zeros((len(training_budget_values), n_runs), dtype=float)\ntop_k = 3\nfor i, budget in enumerate(training_budget_values):\nfor j in range(n_runs):\ny_true = df[\"exact\"].values\ny_estimated = df[f\"{budget}_{j}\"].values\nerrors[i, j] = np.linalg.norm(y_true - y_estimated, ord=2)\naccuracies[i, j] = top_k_value_accuracy(y_true, y_estimated, k=top_k)\nerror_from_mean = np.linalg.norm(df[\"exact\"].values - df[\"exact\"].values.mean(), ord=2)\n</code></pre> <p>Let us next look at how well the ranking of values resulting from using the surrogate \\(\\tilde{u}\\) matches the ranking by the exact values. For this we fix \\(k=3\\) and consider the \\(k\\) samples with the highest value according to \\(\\tilde{u}\\) and \\(u\\):</p> <p>Finally, for each sample, we look at the distance of the estimates to the exact value across runs. Boxes are centered at the 50th percentile with wiskers at the 25th and 75th. We plot relative distances, as a percentage. We observe a general tendency to underestimate the value:</p> <pre><code>highest_value_index = df.index[df[\"exact\"].argmax()]\ny_train_corrupted = dataset.y_train.copy()\ny_train_corrupted[highest_value_index] = (\ny_train_corrupted[highest_value_index] + 1\n) % 3\ncorrupted_dataset = Dataset(\nx_train=dataset.x_train,\ny_train=y_train_corrupted,\nx_test=dataset.x_test,\ny_test=dataset.y_test,\n)\n</code></pre> <p>We retrain the model on the new dataset and verify that the accuracy decreases:</p> <pre><code>model = LinearSVC()\nmodel.fit(dataset.x_train, y_train_corrupted)\nprint(f\"Mean accuracy: {100 * model.score(dataset.x_test, dataset.y_test):0.2f}%\")\n</code></pre> <pre>\n<code>Mean accuracy: 82.96%\n</code>\n</pre> <p>Finally, we recompute the values of all samples using the exact method and the best training budget previously obtained and then plot the resulting scores.</p> <pre><code>best_training_budget = training_budget_values[errors.mean(axis=1).argmin()]\nutility = Utility(\nmodel=LinearSVC(),\ndata=corrupted_dataset,\n)\nresult = compute_shapley_values(\nu=utility,\nmode=ShapleyMode.CombinatorialExact,\nn_jobs=-1,\nprogress=False,\n)\ndf_corrupted = result.to_dataframe(column=\"exact\").drop(columns=[\"exact_stderr\"])\ndul_utility = DataUtilityLearning(\nu=utility, training_budget=best_training_budget, model=MLPRegressor(**mlp_kwargs)\n)\nresult = compute_shapley_values(\nu=dul_utility,\nmode=ShapleyMode.PermutationMontecarlo,\ndone=MaxUpdates(300),\nn_jobs=-1,\n)\ndul_df = result.to_dataframe(column=\"estimated\").drop(columns=[\"estimated_stderr\"])\ndf_corrupted = pd.concat([df_corrupted, dul_df], axis=1)\n</code></pre> <p>We can see in the figure that both methods assign the lowest value to the sample with the corrupted label.</p>  As mentioned above, despite the previous results, this work is preliminary and the usefulness of Data Utility Learning remains to be tested in practice."},{"location":"examples/shapley_utility_learning/#data-utility-learning","title":"Data Utility Learning","text":""},{"location":"examples/shapley_utility_learning/#setup","title":"Setup","text":"<p>We begin by importing the main libraries and setting some defaults.</p>   If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/shapley_utility_learning/#dataset","title":"Dataset","text":"<p>Following the paper, we take 15 samples (10%) from the Iris dataset and compute their Data Shapley values by using all the remaining samples as test set for computing the utility, which in this case is accuracy.</p>"},{"location":"examples/shapley_utility_learning/#data-shapley","title":"Data Shapley","text":"<p>We start by defining the utility using the model and computing the exact Data Shapley values by definition \\(\\ref{eq:shapley-def}\\).</p>"},{"location":"examples/shapley_utility_learning/#evaluation-on-anomalous-data","title":"Evaluation on anomalous data","text":"<p>One interesting way to assess the Data Utility Learning approach is to corrupt some data and monitor how the value changes. To do this, we will take the sample with the highest score and change its label.</p>"},{"location":"getting-started/first-steps/","title":"Getting started","text":"<p>Warning</p> <p>Make sure you have read Installing pyDVL before using the library.  In particular read about how caching and parallelization work, since they might require additional setup.</p>"},{"location":"getting-started/first-steps/#main-concepts","title":"Main concepts","text":"<p>pyDVL aims to be a repository of production-ready, reference implementations of algorithms for data valuation and influence functions. Even though we only briefly introduce key concepts in the documentation, the following sections  should be enough to get you started.</p> <ul> <li>Basics of data valuation for key objects and usage patterns for Shapley value   computation and related methods.</li> <li>Computing Influence Values for instructions on how to compute influence functions.</li> </ul>"},{"location":"getting-started/first-steps/#running-the-examples","title":"Running the examples","text":"<p>If you are somewhat familiar with the concepts of data valuation, you can start by browsing our worked-out examples illustrating pyDVL's capabilities either:</p> <ul> <li>In the examples under Basics of data valuation and Computing Influence Values.</li> <li>Using binder notebooks, deployed from each   example's page.</li> <li>Locally, by starting a jupyter server at the root of the project. You will   have to install jupyter first manually since it's not a dependency of the   library.</li> </ul>"},{"location":"getting-started/first-steps/#advanced-usage","title":"Advanced usage","text":"<p>Besides the do's and don'ts of data valuation itself, which are the subject of the examples and the documentation of each method, there are two main things to keep in mind when using pyDVL.</p>"},{"location":"getting-started/first-steps/#caching","title":"Caching","text":"<p>pyDVL uses memcached to cache the computation of the utility function and speed up some computations (see the installation guide).</p> <p>Caching of the utility function is disabled by default. When it is enabled it takes into account the data indices passed as argument and the utility function wrapped into the Utility object. This means that care must be taken when reusing the same utility function with different data, see the documentation for the caching module for more information.</p> <p>In general, caching won't play a major role in the computation of Shapley values because the probability of sampling the same subset twice, and hence needing the same utility function computation, is very low. However, it can be very useful when comparing methods that use the same utility function, or when running multiple experiments with the same data.</p> <p>When is the cache really necessary?</p> <p>Crucially, semi-value computations with the PermutationSampler require caching to be enabled, or they will take twice as long as the direct implementation in compute_shapley_values.</p>"},{"location":"getting-started/first-steps/#parallelization","title":"Parallelization","text":"<p>pyDVL supports joblib for local parallelization (within one machine) and ray for distributed parallelization (across multiple machines).</p> <p>The former works out of the box but for the latter you will need to provide a running cluster (or run ray in local mode).</p> <p>As of v0.7.0 pyDVL does not allow requesting resources per task sent to the cluster, so you will need to make sure that each worker has enough resources to handle the tasks it receives. A data valuation task using game-theoretic methods will typically make a copy of the whole model and dataset to each worker, even if the re-training only happens on a subset of the data. This means that you should make sure that each worker has enough memory to handle the whole dataset.</p>"},{"location":"getting-started/installation/","title":"Installing pyDVL","text":"<p>To install the latest release use:</p> <pre><code>pip install pyDVL\n</code></pre> <p>To use all features of influence functions use instead:</p> <pre><code>pip install pyDVL[influence]\n</code></pre> <p>This includes a dependency on PyTorch (Version 2.0 and above) and thus is left out by default.</p> <p>In case that you have a supported version of CUDA installed (v11.2 to 11.8 as of this writing), you can enable eigenvalue computations for low-rank approximations with CuPy on the GPU by using:</p> <pre><code>pip install pyDVL[cupy]\n</code></pre> <p>If you use a different version of CUDA, please install CuPy manually.</p> <p>In order to check the installation you can use:</p> <pre><code>python -c \"import pydvl; print(pydvl.__version__)\"\n</code></pre> <p>You can also install the latest development version from TestPyPI:</p> <pre><code>pip install pyDVL --index-url https://test.pypi.org/simple/\n</code></pre>"},{"location":"getting-started/installation/#dependencies","title":"Dependencies","text":"<p>pyDVL requires Python &gt;= 3.8, Memcached for caching and Ray for parallelization in a cluster (locally it uses joblib). Additionally, the Influence functions module requires PyTorch (see Installing pyDVL).</p> <p>ray is used to distribute workloads across nodes in a cluster (it can be used locally as well, but for this we recommend joblib instead). Please follow the instructions in their documentation to set up the cluster. Once you have a running cluster, you can use it by passing the address of the head node to parallel methods via ParallelConfig.</p>"},{"location":"getting-started/installation/#setting-up-the-cache","title":"Setting up the cache","text":"<p>memcached is an in-memory key-value store accessible over the network. pyDVL uses it to cache the computation of the utility function and speed up some computations (in particular, semi-value computations with the PermutationSampler but other methods may benefit as well).</p> <p>You can either install it as a package or run it inside a docker container (the simplest). For installation instructions, refer to the Getting started section in memcached's wiki. Then you can run it with:</p> <pre><code>memcached -u user\n</code></pre> <p>To run memcached inside a container in daemon mode instead, do:</p> <pre><code>docker container run -d --rm -p 11211:11211 memcached:latest\n</code></pre> <p>Using the cache</p> <p>Continue reading about the cache in the First Steps and the documentation for the caching module.</p>"},{"location":"influence/","title":"The influence function","text":""},{"location":"influence/#the-influence-function","title":"The influence function","text":"<p>Warning</p> <p>The code in the package pydvl.influence is experimental. Package structure and basic API are bound to change before v1.0.0</p> <p>The influence function (IF) is a method to quantify the effect (influence) that each training point has on the parameters of a model, and by extension on any function thereof. In particular, it allows to estimate how much each training sample affects the error on a test point, making the IF useful for understanding and debugging models.</p> <p>Alas, the influence function relies on some assumptions that can make their application difficult. Yet another drawback is that they require the computation of the inverse of the Hessian of the model wrt. its parameters, which is intractable for large models like deep neural networks. Much of the recent research tackles this issue using approximations, like a Neuman series  (Agarwal et al., 2017)<sup>1</sup>, with the most successful solution using a low-rank approximation that iteratively finds increasing eigenspaces of the Hessian  (Schioppa et al., 2021)<sup>2</sup>.</p> <p>pyDVL implements several methods for the efficient computation of the IF for machine learning. In the examples we document some of the difficulties that can arise when using the IF.</p>"},{"location":"influence/#construction","title":"Construction","text":"<p>First introduced in the context of robust statistics in (Hampel, 1974)<sup>3</sup>, the IF was popularized in the context of machine learning in  (Koh and Liang, 2017)<sup>4</sup>.</p> <p>Following their formulation, consider an input space \\(\\mathcal{X}\\) (e.g. images) and an output space \\(\\mathcal{Y}\\) (e.g. labels). Let's take \\(z_i = (x_i, y_i)\\), for \\(i \\in  \\{1,...,n\\}\\) to be the \\(i\\)-th training point, and \\(\\theta\\) to be the (potentially highly) multi-dimensional parameters of a model (e.g. \\(\\theta\\) is a big array with all of a neural network's parameters, including biases and/or dropout rates). We will denote with \\(L(z, \\theta)\\) the loss of the model for point \\(z\\) when the parameters are \\(\\theta.\\)</p> <p>To train a model, we typically minimize the loss over all \\(z_i\\), i.e. the optimal parameters are</p> \\[\\hat{\\theta} = \\arg \\min_\\theta \\sum_{i=1}^n L(z_i, \\theta).\\] <p>In practice, lack of convexity means that one doesn't really obtain the minimizer of the loss, and the training is stopped when the validation loss stops decreasing.</p> <p>For notational convenience, let's define</p> \\[\\hat{\\theta}_{-z} = \\arg \\min_\\theta \\sum_{z_i \\ne z} L(z_i, \\theta), \\] <p>i.e. \\(\\hat{\\theta}_{-z}\\) are the model parameters that minimize the total loss when \\(z\\) is not in the training dataset.</p> <p>In order to compute the impact of each training point on the model, we would need to calculate \\(\\hat{\\theta}_{-z}\\) for each \\(z\\) in the training dataset, thus re-training the model at least ~\\(n\\) times (more if model training is stochastic). This is computationally very expensive, especially for big neural networks. To circumvent this problem, we can just calculate a first order approximation of \\(\\hat{\\theta}\\). This can be done through single backpropagation and without re-training the full model.</p> <p>pyDVL supports two ways of computing the empirical influence function, namely up-weighting of samples and perturbation influences. The choice is done by the parameter <code>influence_type</code> in the main entry point  compute_influences.</p>"},{"location":"influence/#approximating-the-influence-of-a-point","title":"Approximating the influence of a point","text":"<p>Let's define</p> \\[\\hat{\\theta}_{\\epsilon, z} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_i, \\theta) + \\epsilon L(z, \\theta), \\] <p>which is the optimal \\(\\hat{\\theta}\\) when we up-weight \\(z\\) by an amount \\(\\epsilon \\gt 0\\).</p> <p>From a classical result (a simple derivation is available in Appendix A of  (Koh and Liang, 2017)<sup>4</sup>), we know that:</p> \\[\\frac{d \\ \\hat{\\theta}_{\\epsilon, z}}{d \\epsilon} \\Big|_{\\epsilon=0} = -H_{\\hat{\\theta}}^{-1} \\nabla_\\theta L(z, \\hat{\\theta}), \\] <p>where \\(H_{\\hat{\\theta}} = \\frac{1}{n} \\sum_{i=1}^n \\nabla_\\theta^2 L(z_i, \\hat{\\theta})\\) is the Hessian of \\(L\\). These quantities are also knows as influence factors.</p> <p>Importantly, notice that this expression is only valid when \\(\\hat{\\theta}\\) is a minimum of \\(L\\), or otherwise \\(H_{\\hat{\\theta}}\\) cannot be inverted! At the same time, in machine learning full convergence is rarely achieved, so direct Hessian inversion is not possible. Approximations need to be developed that circumvent the problem of inverting the Hessian of the model in all those (frequent) cases where it is not positive definite.</p> <p>The influence of training point \\(z\\) on test point \\(z_{\\text{test}}\\) is defined as:</p> \\[\\mathcal{I}(z, z_{\\text{test}}) =  L(z_{\\text{test}}, \\hat{\\theta}_{-z}) - L(z_{\\text{test}}, \\hat{\\theta}). \\] <p>Notice that \\(\\mathcal{I}\\) is higher for points \\(z\\) which positively impact the model score, since the loss is higher when they are excluded from training. In practice, one needs to rely on the following infinitesimal approximation:</p> \\[\\mathcal{I}_{up}(z, z_{\\text{test}}) = - \\frac{d L(z_{\\text{test}}, \\hat{\\theta}_{\\epsilon, z})}{d \\epsilon} \\Big|_{\\epsilon=0} \\] <p>Using the chain rule and the results calculated above, we get:</p> \\[\\mathcal{I}_{up}(z, z_{\\text{test}}) = - \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ \\frac{d \\hat{\\theta}_{\\epsilon, z}}{d \\epsilon} \\Big|_{\\epsilon=0} = \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ H_{\\hat{\\theta}}^{-1} \\ \\nabla_\\theta L(z, \\hat{\\theta}) \\] <p>All the resulting factors are gradients of the loss wrt. the model parameters \\(\\hat{\\theta}\\). This can be easily computed through one or more backpropagation passes.</p>"},{"location":"influence/#perturbation-definition-of-the-influence-score","title":"Perturbation definition of the influence score","text":"<p>How would the loss of the model change if, instead of up-weighting an individual point \\(z\\), we were to up-weight only a single feature of that point? Given \\(z = (x, y)\\), we can define \\(z_{\\delta} = (x+\\delta, y)\\), where \\(\\delta\\) is a vector of zeros except for a 1 in the position of the feature we want to up-weight. In order to approximate the effect of modifying a single feature of a single point on the model score we can define</p> \\[\\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_{i}, \\theta) + \\epsilon L(z_{\\delta}, \\theta) - \\epsilon L(z, \\theta), \\] <p>Similarly to what was done above, we up-weight point \\(z_{\\delta}\\), but then we also remove the up-weighting for all the features that are not modified by \\(\\delta\\). From the calculations in the previous section, it is then easy to see that</p> \\[\\frac{d \\ \\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z}}{d \\epsilon} \\Big|_{\\epsilon=0} = -H_{\\hat{\\theta}}^{-1} \\nabla_\\theta \\Big( L(z_{\\delta}, \\hat{\\theta}) - L(z, \\hat{\\theta}) \\Big) \\] <p>and if the feature space is continuous and as \\(\\delta \\to 0\\) we can write</p> \\[\\frac{d \\ \\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z}}{d \\epsilon} \\Big|_{\\epsilon=0} = -H_{\\hat{\\theta}}^{-1} \\ \\nabla_x \\nabla_\\theta L(z, \\hat{\\theta}) \\delta + \\mathcal{o}(\\delta) \\] <p>The influence of each feature of \\(z\\) on the loss of the model can therefore be estimated through the following quantity:</p> \\[\\mathcal{I}_{pert}(z, z_{\\text{test}}) = - \\lim_{\\delta \\to 0} \\ \\frac{1}{\\delta} \\frac{d L(z_{\\text{test}}, \\hat{\\theta}_{\\epsilon, \\ z_{\\delta}, \\ -z})}{d \\epsilon} \\Big|_{\\epsilon=0} \\] <p>which, using the chain rule and the results calculated above, is equal to</p> \\[\\mathcal{I}_{pert}(z, z_{\\text{test}}) = - \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ \\frac{d \\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z}}{d \\epsilon} \\Big|_{\\epsilon=0} = \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ H_{\\hat{\\theta}}^{-1} \\ \\nabla_x \\nabla_\\theta L(z, \\hat{\\theta}) \\] <p>The perturbation definition of the influence score is not straightforward to understand, but it has a simple interpretation: it tells how much the loss of the model changes when a certain feature of point z is up-weighted. A positive perturbation influence score indicates that the feature might have a positive effect on the accuracy of the model.</p> <p>It is worth noting that the perturbation influence score is a very rough estimate of the impact of a point on the models loss and it is subject to large approximation errors. It can nonetheless be used to build training-set attacks, as done in (Koh and Liang, 2017)<sup>4</sup>.</p>"},{"location":"influence/#computation","title":"Computation","text":"<p>The main entry point of the library for influence calculation is compute_influences. Given a pre-trained pytorch model with a loss, first an instance of TorchTwiceDifferentiable needs to be created:</p> <pre><code>from pydvl.influence import TorchTwiceDifferentiable\nwrapped_model = TorchTwiceDifferentiable(model, loss, device)\n</code></pre> <p>The device specifies where influence calculation will be run. </p> <p>Given training and test data loaders, the influence of each training point on each test point can be calculated via:</p> <pre><code>from pydvl.influence import compute_influences\nfrom torch.utils.data import DataLoader\ntraining_data_loader = DataLoader(...)\ntest_data_loader = DataLoader(...)\ncompute_influences(\nwrapped_model,\ntraining_data_loader,\ntest_data_loader,\n)\n</code></pre> <p>The result is a tensor with one row per test point and one column per training point. Thus, each entry \\((i, j)\\) represents the influence of training point \\(j\\) on test point \\(i\\). A large positive influence indicates that training point \\(j\\) tends to improve the performance of the model on test point \\(i\\), and vice versa, a large negative influence indicates that training point \\(j\\) tends to worsen the performance of the model on test point \\(i\\).</p>"},{"location":"influence/#perturbation-influences","title":"Perturbation influences","text":"<p>The method of empirical influence computation can be selected in compute_influences with the parameter <code>influence_type</code>:</p> <pre><code>from pydvl.influence import compute_influences\ncompute_influences(\nwrapped_model,\ntraining_data_loader,\ntest_data_loader,\ninfluence_type=\"perturbation\",\n)\n</code></pre> <p>The result is a tensor with at least three dimensions. The first two dimensions are the same as in the case of <code>influence_type=up</code> case, i.e. one row per test point and one column per training point. The remaining dimensions are the same as the number of input features in the data. Therefore, each entry in the tensor represents the influence of each feature of each training point on each test point.</p>"},{"location":"influence/#approximate-matrix-inversion","title":"Approximate matrix inversion","text":"<p>In almost every practical application it is not possible to construct, even less invert the complete Hessian in memory. pyDVL offers several approximate algorithms to invert it by setting the parameter <code>inversion_method</code> of compute_influences.</p> <pre><code>from pydvl.influence import compute_influences\ncompute_influences(\nwrapped_model,\ntraining_data_loader,\ntest_data_loader,\ninversion_method=\"cg\"\n)\n</code></pre> <p>Each inversion method has its own set of parameters that can be tuned to improve the final result. These parameters can be passed directly to compute_influences as keyword arguments. For example, the following code sets the maximum number of iterations for conjugate gradient to \\(100\\) and the minimum relative error to \\(0.01\\):</p> <pre><code>from pydvl.influence import compute_influences\ncompute_influences(\nwrapped_model,\ntraining_data_loader,\ntest_data_loader,\ninversion_method=\"cg\",\nhessian_regularization=1e-4,\nmaxiter=100,\nrtol=0.01\n)\n</code></pre>"},{"location":"influence/#hessian-regularization","title":"Hessian regularization","text":"<p>Additionally, and as discussed in the introduction, in machine learning training rarely converges to a global minimum of the loss. Despite good apparent convergence, \\(\\hat{\\theta}\\) might be located in a region with flat curvature or close to a saddle point. In particular, the Hessian might have vanishing eigenvalues making its direct inversion impossible. Certain methods, such as the Arnoldi method are robust against these problems, but most are not.</p> <p>To circumvent this problem, many approximate methods can be implemented. The simplest adds a small hessian perturbation term, i.e. \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\), with \\(\\mathbb{I}\\) being the identity matrix. This standard trick ensures that the eigenvalues of \\(H_{\\hat{\\theta}}\\) are bounded away from zero and therefore the matrix is invertible. In order for this regularization not to corrupt the outcome too much, the parameter \\(\\lambda\\) should be as small as possible while still allowing a reliable inversion of \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\).</p> <pre><code>from pydvl.influence import compute_influences\ncompute_influences(\nwrapped_model,\ntraining_data_loader,\ntest_data_loader,\ninversion_method=\"cg\",\nhessian_regularization=1e-4\n)\n</code></pre>"},{"location":"influence/#influence-factors","title":"Influence factors","text":"<p>The compute_influences method offers a fast way to obtain the influence scores given a model and a dataset. Nevertheless, it is often more convenient to inspect and save some of the intermediate results of influence calculation for later use.</p> <p>The influence factors(refer to the previous section for a definition) are typically the most computationally demanding part of influence calculation. They can be obtained via the compute_influence_factors function, saved, and later used for influence calculation on different subsets of the training dataset.</p> <pre><code>from pydvl.influence import compute_influence_factors\ninfluence_factors = compute_influence_factors(\nwrapped_model,\ntraining_data_loader,\ntest_data_loader,\ninversion_method=\"cg\"\n)\n</code></pre> <p>The result is an object of type  InverseHvpResult, which holds the calculated influence factors (<code>influence_factors.x</code>) and a dictionary with the info on the inversion process (<code>influence_factors.info</code>).</p>"},{"location":"influence/#methods-for-inverse-hvp-calculation","title":"Methods for inverse HVP calculation","text":"<p>In order to calculate influence values, pydvl implements several methods for the calculation of the inverse Hessian vector product (iHVP). More precisely, given a model, training data and a tensor \\(b\\), the function solve_hvp will find \\(x\\) such that \\(H x = b\\), with \\(H\\) is the hessian of model.</p> <p>Many different inversion methods can be selected via the parameter <code>inversion_method</code> of compute_influences.</p> <p>The following subsections will offer more detailed explanations for each method.</p>"},{"location":"influence/#direct-inversion","title":"Direct inversion","text":"<p>With <code>inversion_method = \"direct\"</code> pyDVL will calculate the inverse Hessian using the direct matrix inversion. This means that the Hessian will first be explicitly created and then inverted. This method is the most accurate, but also the most computationally demanding. It is therefore not recommended for large datasets or models with many parameters.</p> <pre><code>import torch\nfrom pydvl.influence.inversion import solve_hvp\nb = torch.Tensor(...)\nsolve_hvp(\n\"direct\",\nwrapped_model,\ntraining_data_loader,\nb,\n)\n</code></pre> <p>The result, an object of type  InverseHvpResult, which holds two objects: <code>influence_factors.x</code> and <code>influence_factors.info</code>. The first one is the inverse Hessian vector product, while the second one is a dictionary with the info on the inversion process. For this method, the info consists of the Hessian matrix itself.</p>"},{"location":"influence/#conjugate-gradient","title":"Conjugate Gradient","text":"<p>This classical procedure for solving linear systems of equations is an iterative method that does not require the explicit inversion of the Hessian. Instead, it only requires the calculation of Hessian-vector products, making it a good choice for large datasets or models with many parameters. It is nevertheless much slower to converge than the direct inversion method and not as accurate. More info on the theory of conjugate gradient can be found on Wikipedia.</p> <p>In pyDVL, you can select conjugate gradient with <code>inversion_method = \"cg\"</code>, like this:</p> <pre><code>from pydvl.influence.inversion import solve_hvp\nsolve_hvp(\n\"cg\",\nwrapped_model,\ntraining_data_loader,\nb,\nx0=None,\nrtol=1e-7,\natol=1e-7,\nmaxiter=None,\n)\n</code></pre> <p>The additional optional parameters <code>x0</code>, <code>rtol</code>, <code>atol</code>, and <code>maxiter</code> are passed to the solve_batch_cg function, and are respecively the initial guess for the solution, the relative tolerance, the absolute tolerance, and the maximum number of iterations.</p> <p>The resulting InverseHvpResult holds the solution of the iHVP, <code>influence_factors.x</code>, and some info on the inversion process <code>influence_factors.info</code>. More specifically, for each batch this will contain the number of iterations, a boolean indicating if the inversion converged, and the residual of the inversion.</p>"},{"location":"influence/#linear-time-stochastic-second-order-approximation-lissa","title":"Linear time Stochastic Second-Order Approximation (LiSSA)","text":"<p>The LiSSA method is a stochastic approximation of the inverse Hessian vector product. Compared to conjugate gradient it is faster but less accurate and typically suffers from instability.</p> <p>In order to find the solution of the HVP, LiSSA iteratively approximates the inverse of the Hessian matrix with the following update:</p> \\[H^{-1}_{j+1} b = b + (I - d) \\ H - \\frac{H^{-1}_j b}{s},\\] <p>where \\(d\\) and \\(s\\) are a dampening and a scaling factor, which are essential for the convergence of the method and they need to be chosen carefully, and I  is the identity matrix. More info on the theory of LiSSA can be found in the  original paper (Agarwal et al., 2017)<sup>1</sup>.</p> <p>In pyDVL, you can select LiSSA with <code>inversion_method = \"lissa\"</code>, like this:</p> <pre><code>from pydvl.influence.inversion import solve_hvp\nsolve_hvp(\n\"lissa\",\nwrapped_model,\ntraining_data_loader,\nb,\nmaxiter=1000,\ndampen=0.0,\nscale=10.0,\nh0=None,\nrtol=1e-4,\n)\n</code></pre> <p>with the additional optional parameters <code>maxiter</code>, <code>dampen</code>, <code>scale</code>, <code>h0</code>, and <code>rtol</code>, which are passed to the solve_lissa function, being the maximum number of iterations, the dampening factor, the scaling factor, the initial guess for the solution and the relative tolerance, respectively.</p> <p>The resulting InverseHvpResult holds the solution of the iHVP, <code>influence_factors.x</code>, and, within <code>influence_factors.info</code>, the maximum percentage error and the mean percentage error of the approximation.</p>"},{"location":"influence/#arnoldi-solver","title":"Arnoldi solver","text":"<p>The Arnoldi method is a Krylov subspace method for approximating dominating eigenvalues and eigenvectors. Under a low rank assumption on the Hessian at a minimizer (which is typically observed for deep neural networks), this approximation captures the essential action of the Hessian. More concretely, for \\(Hx=b\\) the solution is approximated by</p> \\[x \\approx V D^{-1} V^T b\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors. See also  (Schioppa et al., 2021)<sup>2</sup>.</p> <p>In pyDVL, you can use Arnoldi with <code>inversion_method = \"arnoldi\"</code>, as follows:</p> <pre><code>from pydvl.influence.inversion import solve_hvp\nsolve_hvp(\n\"arnoldi\",\nwrapped_model,\ntraining_data_loader,\nb,\nhessian_perturbation=0.0,\nrank_estimate=10,\ntol=1e-6,\neigen_computation_on_gpu=False \n)\n</code></pre> <p>For the parameters, check solve_arnoldi. The resulting InverseHvpResult holds the solution of the iHVP, <code>influence_factors.x</code>, and, within <code>influence_factors.info</code>, the computed eigenvalues and eigenvectors.</p> <ol> <li> <p>Agarwal, N., Bullins, B., Hazan, E., 2017. Second-Order Stochastic Optimization for Machine Learning in Linear Time. JMLR 18, 1\u201340.\u00a0\u21a9\u21a9</p> </li> <li> <p>Schioppa, A., Zablotskaia, P., Vilar, D., Sokolov, A., 2021. Scaling Up Influence Functions. Presented at the AAAI-22, arXiv. https://doi.org/10.48550/arXiv.2112.03052 \u21a9\u21a9</p> </li> <li> <p>Hampel, F.R., 1974. The Influence Curve and Its Role in Robust Estimation. J. Am. Stat. Assoc. 69, 383\u2013393. https://doi.org/10.2307/2285666 \u21a9</p> </li> <li> <p>Koh, P.W., Liang, P., 2017. Understanding Black-box Predictions via Influence Functions, in: Proceedings of the 34th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 1885\u20131894.\u00a0\u21a9\u21a9\u21a9</p> </li> </ol>"},{"location":"value/","title":"Data valuation","text":"<p>Note</p> <p>If you want to jump right into the steps to compute values, skip ahead to Computing data values.</p> <p>Data valuation is the task of assigning a number to each element of a training set which reflects its contribution to the final performance of some model trained on it. Some methods attempt to be model-agnostic, but in most cases the model is an integral part of the method. In these cases, this number not an intrinsic property of the element of interest, but typically a function of three factors:</p> <ol> <li> <p>The dataset \\(D\\), or more generally, the distribution it was sampled    from (with this we mean that value would ideally be the (expected)    contribution of a data point to any random set \\(D\\) sampled from the same    distribution).</p> </li> <li> <p>The algorithm \\(\\mathcal{A}\\) mapping the data \\(D\\) to some estimator \\(f\\)    in a model class \\(\\mathcal{F}\\). E.g. MSE minimization to find the parameters    of a linear model.</p> </li> <li> <p>The performance metric of interest \\(u\\) for the problem. When value depends on    a model, it must be measured in some way which uses it. E.g. the \\(R^2\\) score or    the negative MSE over a test set.</p> </li> </ol> <p>pyDVL collects algorithms for the computation of data values in this sense, mostly those derived from cooperative game theory. The methods can be found in the package pydvl.value , with support from modules pydvl.utils.dataset and pydvl.utils.utility, as detailed below.</p> <p>Warning</p> <p>Be sure to read the section on the difficulties using data values.</p> <p>There are three main families of methods for data valuation: game-theoretic,  influence-based and intrinsic. As of v0.7.0 pyDVL supports the first two. Here, we focus on game-theoretic concepts and refer to the main documentation on the influence funtion for the second.</p>"},{"location":"value/#game-theoretical-methods","title":"Game theoretical methods","text":"<p>The main contenders in game-theoretic approaches are Shapley values (Ghorbani and Zou, 2019)<sup>1</sup>, (Kwon et al., 2021)<sup>2</sup>,  (Schoch et al., 2022)<sup>3</sup>, their generalization to so-called semi-values by (Kwon and Zou, 2022)<sup>4</sup> and (Wang and Jia, 2022)<sup>5</sup>, and the Core (Yan and Procaccia, 2021)<sup>6</sup>. All of these are implemented in pyDVL.</p> <p>In these methods, data points are considered players in a cooperative game  whose outcome is the performance of the model when trained on subsets  (coalitions) of the data, measured on a held-out valuation set. This  outcome, or utility, must typically be computed for every subset of  the training set, so that an exact computation is \\(\\mathcal{O} (2^n)\\) in the  number of samples \\(n\\), with each iteration requiring a full re-fitting of the  model using a coalition as training set. Consequently, most methods involve  Monte Carlo approximations, and sometimes approximate utilities which are  faster to compute, e.g. proxy models (Wang et al., 2022)<sup>7</sup> or constant-cost approximations like Neural Tangent Kernels (Wu et al., 2022)<sup>8</sup>.</p> <p>The reasoning behind using game theory is that, in order to be useful, an assignment of value, dubbed valuation function, is usually required to fulfil certain requirements of consistency and \"fairness\". For instance, in some applications value should not depend on the order in which data are considered, or it should be equal for samples that contribute equally to any subset of the data (of equal size). When considering aggregated value for (sub-)sets of data there are additional desiderata, like having a value function that does not increase with repeated samples. Game-theoretic methods are all rooted in axioms that by construction ensure different desiderata, but despite their practical usefulness, none of them are either necessary or sufficient for all applications. For instance, SV methods try to equitably distribute all value among all samples, failing to identify repeated ones as unnecessary, with e.g. a zero value.</p>"},{"location":"value/#applications-of-data-valuation","title":"Applications of data valuation","text":"<p>Many applications are touted for data valuation, but the results can be inconsistent. Values have a strong dependency on the training procedure and the performance metric used. For instance, accuracy is a poor metric for imbalanced sets and this has a stark effect on data values. Some models exhibit great variance in some regimes and this again has a detrimental effect on values.</p> <p>Nevertheless, some of the most promising applications are:</p> <ul> <li>Cleaning of corrupted data.</li> <li>Pruning unnecessary or irrelevant data.</li> <li>Repairing mislabeled data.</li> <li>Guiding data acquisition and annotation (active learning).</li> <li>Anomaly detection and model debugging and interpretation.</li> </ul> <p>Additionally, one of the motivating applications for the whole field is that of data markets: a marketplace where data owners can sell their data to interested parties. In this setting, data valuation can be key component to determine the price of data. Algorithm-agnostic methods like LAVA (Just et al., 2023)<sup>9</sup> are particularly well suited for this, as they use the Wasserstein distance between a vendor's data and the buyer's to determine the value of the former. </p> <p>However, this is a complex problem which can face practical banal problems like the fact that data owners may not wish to disclose their data for valuation.</p>"},{"location":"value/#computing-data-values","title":"Computing data values","text":"<p>Using pyDVL to compute data values is a simple process that can be broken down into three steps:</p> <ol> <li>Creating a Dataset object from your data.</li> <li>Creating a Utility which ties your model to    the dataset and a scoring function.</li> <li>Computing values with a method of your choice, e.g. via    compute_shapley_values.</li> </ol>"},{"location":"value/#creating-a-dataset","title":"Creating a Dataset","text":"<p>The first item in the tuple \\((D, \\mathcal{A}, u)\\) characterising data value is the dataset. The class Dataset is a simple convenience wrapper for the train and test splits that is used throughout pyDVL. The test set will be used to evaluate a scoring function for the model.</p> <p>It can be used as follows:</p> <pre><code>import numpy as np\nfrom pydvl.utils import Dataset\nfrom sklearn.model_selection import train_test_split\nX, y = np.arange(100).reshape((50, 2)), np.arange(50)\nX_train, X_test, y_train, y_test = train_test_split(\nX, y, test_size=0.5, random_state=16\n)\ndataset = Dataset(X_train, X_test, y_train, y_test)\n</code></pre> <p>It is also possible to construct Datasets from sklearn toy datasets for illustrative purposes using from_sklearn.</p>"},{"location":"value/#grouping-data","title":"Grouping data","text":"<p>Be it because data valuation methods are computationally very expensive, or because we are interested in the groups themselves, it can be often useful or necessary to group samples to valuate them together. GroupedDataset provides an alternative to Dataset with the same interface which allows this.</p> <p>You can see an example in action in the Spotify notebook, but here's a simple example grouping a pre-existing <code>Dataset</code>. First we construct an array mapping each index in the dataset to a group, then use from_dataset:</p> <pre><code>import numpy as np\nfrom pydvl.utils import GroupedDataset\n# Randomly assign elements to any one of num_groups:\ndata_groups = np.random.randint(0, num_groups, len(dataset))\ngrouped_dataset = GroupedDataset.from_dataset(dataset, data_groups)\n</code></pre>"},{"location":"value/#creating-a-utility","title":"Creating a Utility","text":"<p>In pyDVL we have slightly overloaded the name \"utility\" and use it to refer to an object that keeps track of all three items in \\((D, \\mathcal{A}, u)\\). This will be an instance of Utility which, as mentioned, is a convenient wrapper for the dataset, model and scoring function used for valuation methods.</p> <p>Here's a minimal example:</p> <pre><code>import sklearn as sk\nfrom pydvl.utils import Dataset, Utility\ndataset = Dataset.from_sklearn(sk.datasets.load_iris())\nmodel = sk.svm.SVC()\nutility = Utility(model, dataset)\n</code></pre> <p>The object <code>utility</code> is a callable that data valuation methods will execute with different subsets of training data. Each call will retrain the model on a subset and evaluate it on the test data using a scoring function. By default, Utility will use <code>model.score()</code>, but it is possible to use any scoring function (greater values must be better). In particular, the constructor accepts the same types as argument as sklearn.model_selection.cross_validate: a string, a scorer callable or None for the default.</p> <pre><code>utility = Utility(model, dataset, \"explained_variance\")\n</code></pre> <p><code>Utility</code> will wrap the <code>fit()</code> method of the model to cache its results. This greatly reduces computation times of Monte Carlo methods. Because of how caching is implemented, it is important not to reuse <code>Utility</code> objects for different datasets. You can read more about setting up the cache in the installation guide and the documentation of the caching module.</p>"},{"location":"value/#using-custom-scorers","title":"Using custom scorers","text":"<p>The <code>scoring</code> argument of Utility can be used to specify a custom Scorer object. This is a simple wrapper for a callable that takes a model, and test data and returns a score.</p> <p>More importantly, the object provides information about the range of the score, which is used by some methods by estimate the number of samples necessary, and about what default value to use when the model fails to train.</p> <p>Note</p> <p>The most important property of a <code>Scorer</code> is its default value. Because many models will fail to fit on small subsets of the data, it is important to provide a sensible default value for the score.</p> <p>It is possible to skip the construction of the Scorer when constructing the <code>Utility</code> object. The two following calls are equivalent:</p> <pre><code>from pydvl.utils import Utility, Scorer\nutility = Utility(\nmodel, dataset, \"explained_variance\", score_range=(-np.inf, 1), default_score=0.0\n)\nutility = Utility(\nmodel, dataset, Scorer(\"explained_variance\", range=(-np.inf, 1), default=0.0)\n)\n</code></pre>"},{"location":"value/#learning-the-utility","title":"Learning the utility","text":"<p>Because each evaluation of the utility entails a full retrain of the model with a new subset of the training set, it is natural to try to learn this mapping from subsets to scores. This is the idea behind Data Utility Learning (DUL)  (Wang et al., 2022)<sup>7</sup> and in pyDVL it's as simple as wrapping the <code>Utility</code> inside DataUtilityLearning:</p> <pre><code>from pydvl.utils import Utility, DataUtilityLearning, Dataset\nfrom sklearn.linear_model import LinearRegression, LogisticRegression\nfrom sklearn.datasets import load_iris\ndataset = Dataset.from_sklearn(load_iris())\nu = Utility(LogisticRegression(), dataset, enable_cache=False)\ntraining_budget = 3\nwrapped_u = DataUtilityLearning(u, training_budget, LinearRegression())\n# First 3 calls will be computed normally\nfor i in range(training_budget):\n_ = wrapped_u((i,))\n# Subsequent calls will be computed using the fit model for DUL\nwrapped_u((1, 2, 3))\n</code></pre> <p>As you can see, all that is required is a model to learn the utility itself and the fitting and using of the learned model happens behind the scenes.</p> <p>There is a longer example with an investigation of the results achieved by DUL in a dedicated notebook.</p>"},{"location":"value/#leave-one-out-values","title":"Leave-One-Out values","text":"<p>LOO is the simplest approach to valuation. It assigns to each sample its marginal utility as value:</p> \\[v_u(i) = u(D) \u2212 u(D_{-i}).\\] <p>For notational simplicity, we consider the valuation function as defined over the indices of the dataset \\(D\\), and \\(i \\in D\\) is the index of the sample, \\(D_{-i}\\) is the training set without the sample \\(x_i\\), and \\(u\\) is the utility function.</p> <p>For the purposes of data valuation, this is rarely useful beyond serving as a baseline for benchmarking. Although in some benchmarks it can perform astonishingly well on occasion. One particular weakness is that it does not necessarily correlate with an intrinsic value of a sample: since it is a marginal utility, it is affected by diminishing returns. Often, the training set is large enough for a single sample not to have any significant effect on training performance, despite any qualities it may possess. Whether this is indicative of low value or not depends on each one's goals and definitions, but other methods are typically preferable.</p> <pre><code>from pydvl.value.loo import compute_loo\nvalues = compute_loo(utility, n_jobs=-1)\n</code></pre> <p>The return value of all valuation functions is an object of type ValuationResult. This can be iterated over, indexed with integers, slices and Iterables, as well as converted to a pandas.DataFrame.</p>"},{"location":"value/#problems-of-data-values","title":"Problems of data values","text":"<p>There are a number of factors that affect how useful values can be for your project. In particular, regression can be especially tricky, but the particular nature of every (non-trivial) ML problem can have an effect:</p> <ul> <li> <p>Unbounded utility: Choosing a scorer for a classifier is simple: accuracy   or some F-score provides a bounded number with a clear interpretation. However,   in regression problems most scores, like \\(R^2\\), are not bounded because   regressors can be arbitrarily bad. This leads to great variability in the   utility for low sample sizes, and hence unreliable Monte Carlo approximations   to the values. Nevertheless, in practice it is only the ranking of samples   that matters, and this tends to be accurate (wrt. to the true ranking) despite   inaccurate values.</p> Squashing scores <p>pyDVL offers a dedicated function composition for scorer functions which can be used to squash a score. The following is defined in module score: <pre><code>import numpy as np\nfrom pydvl.utils import compose_score\ndef sigmoid(x: float) -&gt; float:\nreturn float(1 / (1 + np.exp(-x)))\nsquashed_r2 = compose_score(\"r2\", sigmoid, \"squashed r2\")\nsquashed_variance = compose_score(\n\"explained_variance\", sigmoid, \"squashed explained variance\"\n)\n</code></pre> These squashed scores can prove useful in regression problems, but they can also introduce issues in the low-value regime.</p> </li> <li> <p>High variance utility: Classical applications of game theoretic value   concepts operate with deterministic utilities, but in ML we use an evaluation   of the model on a validation set as a proxy for the true risk. Even if the   utility is bounded, if it has high variance then values will also have high   variance, as will their Monte Carlo estimates. One workaround in pyDVL is to   configure the caching system to allow multiple evaluations of the utility for   every index set. A moving average is computed and returned once the standard   error is small, see MemcachedConfig.   (Wang and Jia, 2022)<sup>5</sup> prove that by relaxing one of the Shapley axioms   and considering the general class of semi-values, of which Shapley is an   instance, one can prove that a choice of constant weights is the best one can   do in a utility-agnostic setting. This method, dubbed Data Banzhaf, is   available in pyDVL as   compute_banzhaf_semivalues.</p> </li> <li> <p>Data set size: Computing exact Shapley values is NP-hard, and Monte Carlo   approximations can converge slowly. Massive datasets are thus impractical, at   least with game-theoretical methods. A workaround   is to group samples and investigate their value together. You can do this using   GroupedDataset. There is a fully   worked-out example here. Some algorithms   also provide different sampling strategies to reduce the variance, but due to a   no-free-lunch-type theorem, no single strategy can be optimal for all utilities.</p> </li> <li> <p>Model size: Since every evaluation of the utility entails retraining the   whole model on a subset of the data, large models require great amounts of   computation. But also, they will effortlessly interpolate small to medium   datasets, leading to great variance in the evaluation of performance on the   dedicated validation set. One mitigation for this problem is cross-validation,   but this would incur massive computational cost. As of v.0.7.0 there are no   facilities in pyDVL for cross-validating the utility (note that this would   require cross-validating the whole value computation).</p> </li> </ul> <ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning, in: Proceedings of the 36th International Conference on Machine Learning, PMLR. Presented at the International Conference on Machine Learning (ICML 2019), PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> <li> <p>Kwon, Y., Rivas, M.A., Zou, J., 2021. Efficient Computation and Analysis of Distributional Shapley Values, in: Proceedings of the 24th International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics, PMLR, pp. 793\u2013801.\u00a0\u21a9</p> </li> <li> <p>Schoch, S., Xu, H., Ji, Y., 2022. CS-Shapley: Class-wise Shapley Values for Data Valuation in Classification, in: Proc. Of the Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). Presented at the Advances in Neural Information Processing Systems (NeurIPS 2022).\u00a0\u21a9</p> </li> <li> <p>Kwon, Y., Zou, J., 2022. Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning, in: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022,. Presented at the AISTATS 2022, PMLR.\u00a0\u21a9</p> </li> <li> <p>Wang, J.T., Jia, R., 2022. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning [WWW Document]. https://doi.org/10.48550/arXiv.2205.15466 \u21a9\u21a9</p> </li> <li> <p>Yan, T., Procaccia, A.D., 2021. If You Like Shapley Then You\u2019ll Love the Core, in: Proceedings of the 35th AAAI Conference on Artificial Intelligence, 2021. Presented at the AAAI Conference on Artificial Intelligence, Association for the Advancement of Artificial Intelligence, pp. 5751\u20135759. https://doi.org/10.1609/aaai.v35i6.16721 \u21a9</p> </li> <li> <p>Wang, T., Yang, Y., Jia, R., 2022. Improving Cooperative Game Theory-based Data Valuation via Data Utility Learning. Presented at the International Conference on Learning Representations (ICLR 2022). Workshop on Socially Responsible Machine Learning, arXiv. https://doi.org/10.48550/arXiv.2107.06336 \u21a9\u21a9</p> </li> <li> <p>Wu, Z., Shu, Y., Low, B.K.H., 2022. DAVINZ: Data Valuation using Deep Neural Networks at Initialization, in: Proceedings of the 39th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 24150\u201324176.\u00a0\u21a9</p> </li> <li> <p>Just, H.A., Kang, F., Wang, T., Zeng, Y., Ko, M., Jin, M., Jia, R., 2023. LAVA: Data Valuation without Pre-Specified Learning Algorithms. Presented at the The Eleventh International Conference on Learning Representations (ICLR 2023).\u00a0\u21a9</p> </li> </ol>"},{"location":"value/notation/","title":"Notation for valuation","text":"<p>The following notation is used throughout the documentation:</p> <p>Let \\(D = \\{x_1, \\ldots, x_n\\}\\) be a training set of \\(n\\) samples.</p> <p>The utility function \\(u:\\mathcal{D} \\rightarrow \\mathbb{R}\\) maps subsets of \\(D\\) to real numbers.</p> <p>The value \\(v\\) of the \\(i\\)-th sample in dataset \\(D\\) wrt. utility \\(u\\) is denoted as \\(v_u(x_i)\\) or simply \\(v(i)\\).</p> <p>For any \\(S \\subseteq D\\), we donote by \\(S_{-i}\\) the set of samples in \\(D\\) excluding \\(x_i\\), and \\(S_{+i}\\) denotes the set \\(S\\) with \\(x_i\\) added.</p> <p>The marginal utility of adding sample \\(x_i\\) to a subset \\(S\\) is denoted as \\(\\delta(i) := u(S_{+i}) - u(S)\\).</p> <p>The set \\(D_{-i}^{(k)}\\) contains all subsets of \\(D\\) of size \\(k\\) that do not include sample \\(x_i\\).</p>"},{"location":"value/semi-values/","title":"Semi-values","text":"<p>SV is a particular case of a more general concept called semi-value, which is a generalization to different weighting schemes. A semi-value is any valuation function with the form:</p> \\[ v_\\text{semi}(i) = \\sum_{i=1}^n w(k) \\sum_{S \\subset D_{-i}^{(k)}} [u(S_{+i}) - u(S)], \\] <p>where the coefficients \\(w(k)\\) satisfy the property:</p> \\[\\sum_{k=1}^n w(k) = 1,\\] <p>the set \\(D_{-i}^{(k)}\\) contains all subsets of \\(D\\) of size \\(k\\) that do not include sample \\(x_i\\), \\(S_{+i}\\) is the set \\(S\\) with \\(x_i\\) added, and \\(u\\) is the utility function.</p> <p>Two instances of this are Banzhaf indices (Wang and Jia, 2022)<sup>1</sup>, and Beta Shapley (Kwon and Zou, 2022)<sup>2</sup>, with better numerical and rank stability in certain situations.</p> <p>Note</p> <p>Shapley values are a particular case of semi-values and can therefore also be computed with the methods described here. However, as of version 0.7.0, we recommend using compute_shapley_values instead, in particular because it implements truncation policies for TMCS.</p>"},{"location":"value/semi-values/#beta-shapley","title":"Beta Shapley","text":"<p>For some machine learning applications, where the utility is typically the performance when trained on a set \\(S \\subset D\\), diminishing returns are often observed when computing the marginal utility of adding a new data point.</p> <p>Beta Shapley is a weighting scheme that uses the Beta function to place more weight on subsets deemed to be more informative. The weights are defined as:</p> \\[ w(k) := \\frac{B(k+\\beta, n-k+1+\\alpha)}{B(\\alpha, \\beta)}, \\] <p>where \\(B\\) is the Beta function, and \\(\\alpha\\) and \\(\\beta\\) are parameters that control the weighting of the subsets. Setting both to 1 recovers Shapley values, and setting \\(\\alpha = 1\\), and \\(\\beta = 16\\) is reported in (Kwon and Zou, 2022)<sup>2</sup> to be a good choice for some applications. Beta Shapley values are available in pyDVL through compute_beta_shapley_semivalues:</p> <pre><code>from pydvl.value import *\nutility = Utility(model, data)\nvalues = compute_beta_shapley_semivalues(\nu=utility, done=AbsoluteStandardError(threshold=1e-4), alpha=1, beta=16\n)\n</code></pre> <p>See however the Banzhaf indices section  for an alternative choice of weights which is reported to work better.</p>"},{"location":"value/semi-values/#banzhaf-indices","title":"Banzhaf indices","text":"<p>As noted in the section Problems of Data Values, the Shapley value can be very sensitive to variance in the utility function. For machine learning applications, where the utility is typically the performance when trained on a set \\(S \\subset D\\), this variance is often largest for smaller subsets \\(S\\). It is therefore reasonable to try reducing the relative contribution of these subsets with adequate weights.</p> <p>One such choice of weights is the Banzhaf index, which is defined as the constant:</p> \\[w(k) := 2^{n-1},\\] <p>for all set sizes \\(k\\). The intuition for picking a constant weight is that for any choice of weight function \\(w\\), one can always construct a utility with higher variance where \\(w\\) is greater. Therefore, in a worst-case sense, the best one can do is to pick a constant weight.</p> <p>The authors of (Wang and Jia, 2022)<sup>1</sup> show that Banzhaf indices are more robust to variance in the utility function than Shapley and Beta Shapley values. They are available in pyDVL through compute_banzhaf_semivalues:</p> <pre><code>from pydvl.value import *\nutility = Utility(model, data)\nvalues = compute_banzhaf_semivalues(\nu=utility, done=AbsoluteStandardError(threshold=1e-4), alpha=1, beta=16\n)\n</code></pre>"},{"location":"value/semi-values/#general-semi-values","title":"General semi-values","text":"<p>As explained above, both Beta Shapley and Banzhaf indices are special cases of semi-values. In pyDVL we provide a general method for computing these with any combination of the three ingredients that define a semi-value:</p> <ul> <li>A utility function \\(u\\).</li> <li>A sampling method</li> <li>A weighting scheme \\(w\\).</li> </ul> <p>You can construct any combination of these three ingredients with compute_generic_semivalues. The utility function is the same as for Shapley values, and the sampling method can be any of the types defined in the samplers module. For instance, the following snippet is equivalent to the above:</p> <pre><code>from pydvl.value import *\ndata = Dataset(...)\nutility = Utility(model, data)\nvalues = compute_generic_semivalues(\nsampler=PermutationSampler(data.indices),\nu=utility,\ncoefficient=beta_coefficient(alpha=1, beta=16),\ndone=AbsoluteStandardError(threshold=1e-4),\n)\n</code></pre> <p>Allowing any coefficient can help when experimenting with models which are more sensitive to changes in training set size. However, Data Banzhaf indices are proven to be the most robust to variance in the utility function, in the sense of rank stability, across a range of models and datasets (Wang and Jia, 2022)<sup>1</sup>. </p> <p>Careful with permutation sampling</p> <p>This generic implementation of semi-values allowing for any combination of sampling and weighting schemes is very flexible and, in principle, it recovers the original Shapley value, so that  compute_shapley_values is no longer necessary. However, it loses the optimization in permutation sampling that reuses the utility computation from the last iteration when iterating over a permutation. This doubles the computation requirements (and slightly increases variance) when using permutation sampling, unless the cache is enabled. In addition, as mentioned above, truncation policies are not supported by this generic implementation (as of v0.7.0). For these reasons it is preferable to use compute_shapley_values whenever not computing other semi-values.</p> <ol> <li> <p>Wang, J.T., Jia, R., 2022. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning [WWW Document]. https://doi.org/10.48550/arXiv.2205.15466 \u21a9\u21a9\u21a9</p> </li> <li> <p>Kwon, Y., Zou, J., 2022. Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning, in: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022,. Presented at the AISTATS 2022, PMLR.\u00a0\u21a9\u21a9</p> </li> </ol>"},{"location":"value/shapley/","title":"Shapley value","text":""},{"location":"value/shapley/#shapley-value","title":"Shapley value","text":"<p>The Shapley method is an approach to compute data values originating in cooperative game theory. Shapley values are a common way of assigning payoffs to each participant in a cooperative game (i.e. one in which players can form coalitions) in a way that ensures that certain axioms are fulfilled.</p> <p>pyDVL implements several methods for the computation and approximation of Shapley values. They can all be accessed via the facade function compute_shapley_values. The supported methods are enumerated in ShapleyMode.</p> <p>Empirically, the most useful method is the so-called Truncated Monte Carlo Shapley (Ghorbani and Zou, 2019)<sup>1</sup>, which is a Monte Carlo approximation of the permutation Shapley value.</p>"},{"location":"value/shapley/#combinatorial-shapley","title":"Combinatorial Shapley","text":"<p>The first algorithm is just a verbatim implementation of the definition. As such it returns as exact a value as the utility function allows (see what this means in Problems of Data Values).</p> <p>The value \\(v\\) of the \\(i\\)-th sample in dataset \\(D\\) wrt. utility \\(u\\) is computed as a weighted sum of its marginal utility wrt. every possible coalition of training samples within the training set:</p> \\[ v(i) = \\frac{1}{n} \\sum_{S \\subseteq D_{-i}} \\binom{n-1}{ | S | }^{-1} [u(S_{+i}) \u2212 u(S)] ,\\] <p>where \\(D_{-i}\\) denotes the set of samples in \\(D\\) excluding \\(x_i\\), and \\(S_{+i}\\) denotes the set \\(S\\) with \\(x_i\\) added.</p> <pre><code>from pydvl.value import compute_shapley_values\nvalues = compute_shapley_values(utility, mode=\"combinatorial_exact\")\ndf = values.to_dataframe(column='value')\n</code></pre> <p>We can convert the return value to a pandas.DataFrame. and name the column with the results as <code>value</code>. Please refer to the documentation in shapley and ValuationResult for more information.</p>"},{"location":"value/shapley/#monte-carlo-combinatorial-shapley","title":"Monte Carlo Combinatorial Shapley","text":"<p>Because the number of subsets \\(S \\subseteq D_{-i}\\) is \\(2^{ | D | - 1 }\\), one typically must resort to approximations. The simplest one is done via Monte Carlo sampling of the powerset \\(\\mathcal{P}(D)\\). In pyDVL this simple technique is called \"Monte Carlo Combinatorial\". The method has very poor converge rate and others are preferred, but if desired, usage follows the same pattern:</p> <pre><code>from pydvl.value import compute_shapley_values, MaxUpdates\nvalues = compute_shapley_values(\nutility, mode=\"combinatorial_montecarlo\", done=MaxUpdates(1000)\n)\ndf = values.to_dataframe(column='cmc')\n</code></pre> <p>The DataFrames returned by most Monte Carlo methods will contain approximate standard errors as an additional column, in this case named <code>cmc_stderr</code>.</p> <p>Note the usage of the object MaxUpdates as the stop condition. This is an instance of a StoppingCriterion. Other examples are MaxTime and AbsoluteStandardError.</p>"},{"location":"value/shapley/#owen-sampling","title":"Owen sampling","text":"<p>Owen Sampling (Okhrati and Lipani, 2021)<sup>2</sup> is a practical algorithm based on the combinatorial definition. It uses a continuous extension of the utility from \\(\\{0,1\\}^n\\), where a 1 in position \\(i\\) means that sample \\(x_i\\) is used to train the model, to \\([0,1]^n\\). The ensuing expression for Shapley value uses integration instead of discrete weights:</p> \\[ v_u(i) = \\int_0^1 \\mathbb{E}_{S \\sim P_q(D_{-i})} [u(S_{+i}) - u(S)]. \\] <p>Using Owen sampling follows the same pattern as every other method for Shapley values in pyDVL. First construct the dataset and utility, then call compute_shapley_values:</p> <pre><code>from pydvl.value import compute_shapley_values\nvalues = compute_shapley_values(\nu=utility, mode=\"owen\", n_iterations=4, max_q=200\n)\n</code></pre> <p>There are more details on Owen sampling, and its variant Antithetic Owen Sampling in the documentation for the function doing the work behind the scenes: owen_sampling_shapley.</p> <p>Note that in this case we do not pass a StoppingCriterion to the function, but instead the number of iterations and the maximum number of samples to use in the integration.</p>"},{"location":"value/shapley/#permutation-shapley","title":"Permutation Shapley","text":"<p>An equivalent way of computing Shapley values (<code>ApproShapley</code>) appeared in  (Castro et al., 2009)<sup>3</sup> and is the basis for the method most often used in practice. It uses permutations over indices instead of subsets:</p> \\[ v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} [u(\\sigma_{:i} \\cup \\{x_i\\}) \u2212 u(\\sigma_{:i})], \\] <p>where \\(\\sigma_{:i}\\) denotes the set of indices in permutation sigma before the position where \\(i\\) appears. To approximate this sum (which has \\(\\mathcal{O}(n!)\\) terms!) one uses Monte Carlo sampling of permutations, something which has surprisingly low sample complexity. One notable difference wrt. the combinatorial approach above is that the approximations always fulfill the efficiency axiom of Shapley, namely \\(\\sum_{i=1}^n \\hat{v}_i = u(D)\\) (see  (Castro et al., 2009)<sup>3</sup>, Proposition 3.2).</p> <p>By adding two types of early stopping, the result is the so-called Truncated Monte Carlo Shapley (Ghorbani and Zou, 2019)<sup>1</sup>, which is efficient enough to be useful in applications. The first is simply a convergence criterion, of which there are several to choose from. The second is a criterion to truncate the iteration over single permutations. RelativeTruncation chooses to stop iterating over samples in a permutation when the marginal utility becomes too small.</p> <pre><code>from pydvl.value import compute_shapley_values, MaxUpdates, RelativeTruncation\nvalues = compute_shapley_values(\nu=utility,\nmode=\"permutation_montecarlo\",\ndone=MaxUpdates(1000),\ntruncation=RelativeTruncation(utility, rtol=0.01)\n)\n</code></pre> <p>You can see this method in action in this example using the Spotify dataset.</p>"},{"location":"value/shapley/#exact-shapley-for-knn","title":"Exact Shapley for KNN","text":"<p>It is possible to exploit the local structure of K-Nearest Neighbours to reduce the amount of subsets to consider: because no sample besides the K closest affects the score, most are irrelevant and it is possible to compute a value in linear time. This method was introduced by (Jia et al., 2019)<sup>4</sup>, and can be used in pyDVL with:</p> <pre><code>from pydvl.utils import Dataset, Utility\nfrom pydvl.value import compute_shapley_values\nfrom sklearn.neighbors import KNeighborsClassifier\nmodel = KNeighborsClassifier(n_neighbors=5)\ndata = Dataset(...)\nutility = Utility(model, data)\nvalues = compute_shapley_values(u=utility, mode=\"knn\")\n</code></pre>"},{"location":"value/shapley/#group-testing","title":"Group testing","text":"<p>An alternative approach introduced in (Jia et al., 2019)<sup>4</sup> first approximates the differences of values with a Monte Carlo sum. With</p> \\[\\hat{\\Delta}_{i j} \\approx v_i - v_j,\\] <p>one then solves the following linear constraint satisfaction problem (CSP) to infer the final values:</p> \\[ \\begin{array}{lll} \\sum_{i = 1}^N v_i &amp; = &amp; U (D)\\\\ | v_i - v_j - \\hat{\\Delta}_{i j} | &amp; \\leqslant &amp; \\frac{\\varepsilon}{2 \\sqrt{N}} \\end{array} \\] <p>Warning</p> <p>We have reproduced this method in pyDVL for completeness and benchmarking, but we don't advocate its use because of the speed and memory cost. Despite our best efforts, the number of samples required in practice for convergence can be several orders of magnitude worse than with e.g. TMCS. Additionally, the CSP can sometimes turn out to be infeasible.</p> <p>Usage follows the same pattern as every other Shapley method, but with the addition of an <code>epsilon</code> parameter required for the solution of the CSP. It should be the same value used to compute the minimum number of samples required. This can be done with num_samples_eps_delta, but note that the number returned will be huge! In practice, fewer samples can be enough, but the actual number will strongly depend on the utility, in particular its variance.</p> <pre><code>from pydvl.utils import Dataset, Utility\nfrom pydvl.value import compute_shapley_values\nmodel = ...\ndata = Dataset(...)\nutility = Utility(model, data, score_range=(_min, _max))\nmin_iterations = num_samples_eps_delta(epsilon, delta, n, utility.score_range)\nvalues = compute_shapley_values(\nu=utility, mode=\"group_testing\", n_iterations=min_iterations, eps=eps\n)\n</code></pre> <ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning, in: Proceedings of the 36th International Conference on Machine Learning, PMLR. Presented at the International Conference on Machine Learning (ICML 2019), PMLR, pp. 2242\u20132251.\u00a0\u21a9\u21a9</p> </li> <li> <p>Okhrati, R., Lipani, A., 2021. A Multilinear Sampling Algorithm to Estimate Shapley Values, in: 2020 25th International Conference on Pattern Recognition (ICPR). Presented at the 2020 25th International Conference on Pattern Recognition (ICPR), IEEE, pp. 7992\u20137999. https://doi.org/10.1109/ICPR48806.2021.9412511 \u21a9</p> </li> <li> <p>Castro, J., G\u00f3mez, D., Tejada, J., 2009. Polynomial calculation of the Shapley value based on sampling. Computers   &amp; Operations Research, Selected papers presented at the Tenth International Symposium on Locational Decisions (ISOLDE X) 36, 1726\u20131730. https://doi.org/10.1016/j.cor.2008.04.004 \u21a9\u21a9</p> </li> <li> <p>Jia, R., Dao, D., Wang, B., Hubis, F.A., Gurel, N.M., Li, B., Zhang, C., Spanos, C., Song, D., 2019. Efficient task-specific data valuation for nearest neighbor algorithms. Proc. VLDB Endow. 12, 1610\u20131623. https://doi.org/10.14778/3342263.3342637 \u21a9\u21a9</p> </li> </ol>"},{"location":"value/the-core/","title":"Core values","text":"<p>The Shapley values define a fair way to distribute payoffs amongst all participants when they form a grand coalition. But they do not consider the question of stability: under which conditions do all participants form the grand coalition? Would the participants be willing to form the grand coalition given how the payoffs are assigned, or would some of them prefer to form smaller coalitions?</p> <p>The Core is another approach to computing data values originating in cooperative game theory that attempts to ensure this stability. It is the set of feasible payoffs that cannot be improved upon by a coalition of the participants.</p> <p>It satisfies the following 2 properties:</p> <ul> <li> <p>Efficiency:   The payoffs are distributed such that it is not possible   to make any participant better off   without making another one worse off.   \\(\\(\\sum_{i\\in D} v(i) = u(D)\\,\\)\\)</p> </li> <li> <p>Coalitional rationality:   The sum of payoffs to the agents in any coalition S is at   least as large as the amount that these agents could earn by   forming a coalition on their own.   \\(\\(\\sum_{i \\in S} v(i) \\geq u(S), \\forall S \\subset D\\,\\)\\)</p> </li> </ul> <p>The second property states that the sum of payoffs to the agents in any subcoalition \\(S\\) is at least as large as the amount that these agents could earn by forming a coalition on their own.</p>"},{"location":"value/the-core/#least-core-values","title":"Least Core values","text":"<p>Unfortunately, for many cooperative games the Core may be empty. By relaxing the coalitional rationality property by a subsidy \\(e \\gt 0\\), we are then able to find approximate payoffs:</p> \\[ \\sum_{i\\in S} v(i) + e \\geq u(S), \\forall S \\subset D, S \\neq \\emptyset \\ ,\\] <p>The least core value \\(v\\) of the \\(i\\)-th sample in dataset \\(D\\) wrt. utility \\(u\\) is computed by solving the following Linear Program:</p> \\[ \\begin{array}{lll} \\text{minimize} &amp; e &amp; \\\\ \\text{subject to} &amp; \\sum_{i\\in D} v(i) = u(D) &amp; \\\\ &amp; \\sum_{i\\in S} v(i) + e \\geq u(S) &amp;, \\forall S \\subset D, S \\neq \\emptyset  \\\\ \\end{array} \\]"},{"location":"value/the-core/#exact-least-core","title":"Exact Least Core","text":"<p>This first algorithm is just a verbatim implementation of the definition. As such it returns as exact a value as the utility function allows (see what this means in Problems of Data Values][problems-of-data-values]).</p> <pre><code>from pydvl.value import compute_least_core_values\nvalues = compute_least_core_values(utility, mode=\"exact\")\n</code></pre>"},{"location":"value/the-core/#monte-carlo-least-core","title":"Monte Carlo Least Core","text":"<p>Because the number of subsets \\(S \\subseteq D \\setminus \\{i\\}\\) is \\(2^{ | D | - 1 }\\), one typically must resort to approximations.</p> <p>The simplest approximation consists in using a fraction of all subsets for the constraints. (Yan and Procaccia, 2021)<sup>1</sup> show that a quantity of order \\(\\mathcal{O}((n - \\log \\Delta ) / \\delta^2)\\) is enough to obtain a so-called \\(\\delta\\)-approximate least core with high probability. I.e. the following property holds with probability \\(1-\\Delta\\) over the choice of subsets:</p> \\[ \\mathbb{P}_{S\\sim D}\\left[\\sum_{i\\in S} v(i) + e^{*} \\geq u(S)\\right] \\geq 1 - \\delta, \\] <p>where \\(e^{*}\\) is the optimal least core subsidy.</p> <pre><code>from pydvl.value import compute_least_core_values\nvalues = compute_least_core_values(\nutility, mode=\"montecarlo\", n_iterations=n_iterations\n)\n</code></pre> <p>Note</p> <p>Although any number is supported, it is best to choose <code>n_iterations</code> to be at least equal to the number of data points.</p> <p>Because computing the Least Core values requires the solution of a linear and a quadratic problem after computing all the utility values, we offer the possibility of splitting the latter from the former. This is useful when running multiple experiments: use mclc_prepare_problem to prepare a list of problems to solve, then solve them in parallel with lc_solve_problems.</p> <pre><code>from pydvl.value.least_core import mclc_prepare_problem, lc_solve_problems\nn_experiments = 10\nproblems = [mclc_prepare_problem(utility, n_iterations=n_iterations)\nfor _ in range(n_experiments)]\nvalues = lc_solve_problems(problems)\n</code></pre>"},{"location":"value/the-core/#method-comparison","title":"Method comparison","text":"<p>The TransferLab team reproduced the results of the original paper in a publication for the 2022 MLRC (Benmerzoug and Delgado, 2023)<sup>2</sup>.</p> <p> Best sample removal on binary image classification </p> <p>Roughly speaking, MCLC performs better in identifying high value points, as measured by best-sample removal tasks. In all other aspects, it performs worse or similarly to TMCS at comparable sample budgets. But using an equal number of subsets is more computationally expensive because of the need to solve large linear and quadratic optimization problems.</p> <p> Worst sample removal on binary image classification </p> <p>For these reasons we recommend some variation of SV like TMCS for outlier detection, data cleaning and pruning, and perhaps MCLC for the selection of interesting points to be inspected for the improvement of data collection or model design.</p> <ol> <li> <p>Yan, T., Procaccia, A.D., 2021. If You Like Shapley Then You\u2019ll Love the Core, in: Proceedings of the 35th AAAI Conference on Artificial Intelligence, 2021. Presented at the AAAI Conference on Artificial Intelligence, Association for the Advancement of Artificial Intelligence, pp. 5751\u20135759. https://doi.org/10.1609/aaai.v35i6.16721 \u21a9</p> </li> <li> <p>Benmerzoug, A., Delgado, M. de B., 2023. [Re] If you like Shapley, then you\u2019ll love the core. ReScience C 9. https://doi.org/10.5281/zenodo.8173733 \u21a9</p> </li> </ol>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"The python library for data valuation","text":"<p>pyDVL collects algorithms for data valuation and influence function computation. It runs most of them in parallel either locally or in a cluster and supports distributed caching of results.</p> <p>If you're a first time user of pyDVL, we recommend you to go through the Getting Started and Installing pyDVL guides.</p> <p>Installation</p><p>Steps to install and requirements</p> <p>Data valuation</p><p>Basics of data valuation and description of the main algorithms </p> <p>Influence Function</p><p>An introduction to the influence function and its computation with pyDVL </p> <p>Browse the API</p><p>Full documentation of the API</p>"},{"location":"CHANGELOG/","title":"Changelog","text":""},{"location":"CHANGELOG/#unreleased","title":"Unreleased","text":"<ul> <li>Implementation of Data-OOB by @BastienZim   PR #426</li> <li>Refactoring of parallel module. Old imports will stop working in v0.9.0   PR #421</li> </ul>"},{"location":"CHANGELOG/#070-documentation-and-if-overhaul-new-methods-and-bug-fixes","title":"0.7.0 - \ud83d\udcda\ud83c\udd95 Documentation and IF overhaul, new methods and bug fixes \ud83d\udca5\ud83d\udc1e","text":"<p>This is our first \u03b2 release! We have worked hard to deliver improvements across the board, with a focus on documentation and usability. We have also reworked the internals of the <code>influence</code> module, improved parallelism and handling of randomness.</p>"},{"location":"CHANGELOG/#added","title":"Added","text":"<ul> <li>Implemented solving the Hessian equation via spectral low-rank approximation   PR #365</li> <li>Enabled parallel computation for Leave-One-Out values   PR #406</li> <li>Added more abbreviations to documentation   PR #415</li> <li>Added seed to functions from <code>pydvl.utils.numeric</code>, <code>pydvl.value.shapley</code> and   <code>pydvl.value.semivalues</code>. Introduced new type <code>Seed</code> and conversion function    <code>ensure_seed_sequence</code>.   PR #396</li> <li>Added <code>batch_size</code> parameter to <code>compute_banzhaf_semivalues</code>,   <code>compute_beta_shapley_semivalues</code>, <code>compute_shapley_semivalues</code> and   <code>compute_generic_semivalues</code>.   PR #428</li> </ul>"},{"location":"CHANGELOG/#changed","title":"Changed","text":"<ul> <li>Replaced sphinx with mkdocs for documentation. Major overhaul of documentation   PR #352</li> <li>Made ray an optional dependency, relying on joblib as default parallel backend   PR #408</li> <li>Decoupled <code>ray.init</code> from <code>ParallelConfig</code> PR #373</li> <li>Breaking Changes</li> <li>Signature change: return information about Hessian inversion from     <code>compute_influence_factors</code> PR #375</li> <li>Major changes to IF interface and functionality. Foundation for a framework     abstraction for IF computation.     PR #278 PR #394</li> <li>Renamed <code>semivalues</code> to <code>compute_generic_semivalues</code> PR #413</li> <li>New <code>joblib</code> backend as default instead of ray. Simplify MapReduceJob.     PR #355</li> <li>Bump torch dependency for influence package to 2.0     PR #365</li> </ul>"},{"location":"CHANGELOG/#fixed","title":"Fixed","text":"<ul> <li>Fixes to parallel computation of generic semi-values: properly handle all   samplers and stopping criteria, irrespective of parallel backend.   PR #372</li> <li>Optimises memory usage in IF calculation   PR #375</li> <li>Fix adding valuation results with overlapping indices and different lengths   PR #370</li> <li>Fixed bugs in conjugate gradient and <code>linear_solve</code> PR #358</li> <li>Fix installation of dev requirements for Python3.10   PR #382</li> <li>Improvements to IF documentation   PR #371</li> </ul>"},{"location":"CHANGELOG/#061-bug-fixes-and-small-improvements","title":"0.6.1 - \ud83c\udfd7 Bug fixes and small improvements","text":"<ul> <li>Fix parsing keyword arguments of <code>compute_semivalues</code> dispatch function   PR #333</li> <li>Create new <code>RayExecutor</code> class based on the concurrent.futures API,   use the new class to fix an issue with Truncated Monte Carlo Shapley   (TMCS) starting too many processes and dying, plus other small changes   PR #329</li> <li>Fix creation of GroupedDataset objects using the <code>from_arrays</code>   and <code>from_sklearn</code> class methods    PR #324</li> <li>Fix release job not triggering on CI when a new tag is pushed   PR #331</li> <li>Added alias <code>ApproShapley</code> from Castro et al. 2009 for permutation Shapley   PR #332</li> </ul>"},{"location":"CHANGELOG/#060-new-algorithms-cleanup-and-bug-fixes","title":"0.6.0 - \ud83c\udd95 New algorithms, cleanup and bug fixes \ud83c\udfd7","text":"<ul> <li>Fixes in <code>ValuationResult</code>: bugs around data names, semantics of   <code>empty()</code>, new method <code>zeros()</code> and normalised random values   PR #327</li> <li>New method: Implements generalised semi-values for data valuation,   including Data Banzhaf and Beta Shapley, with configurable sampling strategies   PR #319</li> <li>Adds kwargs parameter to <code>from_array</code> and <code>from_sklearn</code> Dataset and   GroupedDataset class methods   PR #316</li> <li>PEP-561 conformance: added <code>py.typed</code> PR #307</li> <li>Removed default non-negativity constraint on least core subsidy   and added instead a <code>non_negative_subsidy</code> boolean flag.   Renamed <code>options</code> to <code>solver_options</code> and pass it as dict.   Change default least-core solver to SCS with 10000 max_iters.   PR #304</li> <li>Cleanup: removed unnecessary decorator <code>@unpackable</code> PR #233</li> <li>Stopping criteria: fixed problem with <code>StandardError</code> and enable proper   composition of index convergence statuses. Fixed a bug with <code>n_jobs</code> in   <code>truncated_montecarlo_shapley</code>.   PR #300 and   PR #305</li> <li>Shuffling code around to allow for simpler user imports, some cleanup and   documentation fixes.   PR #284</li> <li>Bug fix: Warn instead of raising an error when <code>n_iterations</code>   is less than the size of the dataset in Monte Carlo Least Core   PR #281</li> </ul>"},{"location":"CHANGELOG/#050-fixes-nicer-interfaces-and-more-breaking-changes","title":"0.5.0 - \ud83d\udca5 Fixes, nicer interfaces and... more breaking changes \ud83d\ude12","text":"<ul> <li>Fixed parallel and antithetic Owen sampling for Shapley values. Simplified   and extended tests.   PR #267</li> <li>Added <code>Scorer</code> class for a cleaner interface. Fixed minor bugs around   Group-Testing Shapley, added more tests and switched to cvxpy for the solver.   PR #264</li> <li>Generalised stopping criteria for valuation algorithms. Improved classes   <code>ValuationResult</code> and <code>Status</code> with more operations. Some minor issues fixed.   PR #252</li> <li>Fixed a bug whereby <code>compute_shapley_values</code> would only spawn one process when   using <code>n_jobs=-1</code> and Monte Carlo methods.   PR #270</li> <li>Bugfix in <code>RayParallelBackend</code>: wrong semantics for <code>kwargs</code>.   PR #268</li> <li>Splitting of problem preparation and solution in Least-Core computation.   Umbrella function for LC methods.   PR #257 </li> <li>Operations on <code>ValuationResult</code> and <code>Status</code> and some cleanup   PR #248</li> <li>Bug fix and minor improvements: Fixes bug in TMCS with remote Ray cluster,   raises an error for dummy sequential parallel backend with TMCS, clones model   inside <code>Utility</code> before fitting by default, with flag <code>clone_before_fit</code>    to disable it, catches all warnings in <code>Utility</code> when <code>show_warnings</code> is    <code>False</code>. Adds Miner and Gloves toy games utilities   PR #247</li> </ul>"},{"location":"CHANGELOG/#040-new-algorithms-and-more-breaking-changes","title":"0.4.0 - \ud83c\udfed\ud83d\udca5 New algorithms and more breaking changes","text":"<ul> <li>GH action to mark issues as stale   PR #201</li> <li>Disabled caching of Utility values as well as repeated evaluations by default   PR #211</li> <li>Test and officially support Python version 3.9 and 3.10    PR #208</li> <li>Breaking change: Introduces a class ValuationResult to gather and inspect   results from all valuation algorithms   PR #214</li> <li>Fixes bug in Influence calculation with multidimensional input and adds new   example notebook   PR #195</li> <li>Breaking change: Passes the input to <code>MapReduceJob</code> at initialization,   removes <code>chunkify_inputs</code> argument from <code>MapReduceJob</code>, removes <code>n_runs</code>   argument from <code>MapReduceJob</code>, calls the parallel backend's <code>put()</code> method for   each generated chunk in <code>_chunkify()</code>, renames ParallelConfig's <code>num_workers</code>   attribute to <code>n_local_workers</code>, fixes a bug in <code>MapReduceJob</code>'s chunkification   when <code>n_runs</code> &gt;= <code>n_jobs</code>, and defines a sequential parallel backend to run   all jobs in the current thread   PR #232</li> <li>New method: Implements exact and monte carlo Least Core for data valuation,   adds <code>from_arrays()</code> class method to the <code>Dataset</code> and <code>GroupedDataset</code>   classes, adds <code>extra_values</code> argument to <code>ValuationResult</code>, adds   <code>compute_removal_score()</code> and <code>compute_random_removal_score()</code> helper functions   PR #237</li> <li>New method: Group Testing Shapley for valuation, from Jia et al. 2019 PR #240</li> <li>Fixes bug in ray initialization in <code>RayParallelBackend</code> class   PR #239</li> <li>Implements \"Egalitarian Least Core\", adds cvxpy as a   dependency and uses it instead of scipy as optimizer   PR #243</li> </ul>"},{"location":"CHANGELOG/#030-breaking-changes","title":"0.3.0 - \ud83d\udca5 Breaking changes","text":"<ul> <li>Simplified and fixed powerset sampling and testing   PR #181</li> <li>Simplified and fixed publishing to PyPI from CI   PR #183</li> <li>Fixed bug in release script and updated contributing docs.   PR #184</li> <li>Added Pull Request template   PR #185</li> <li>Modified Pull Request template to automatically link PR to issue   PR ##186</li> <li>First implementation of Owen Sampling, squashed scores, better testing   PR #194</li> <li>Improved documentation on caching, Shapley, caveats of values, bibtex   PR #194</li> <li>Breaking change: Rearranging of modules to accommodate for new methods   PR #194</li> </ul>"},{"location":"CHANGELOG/#020-better-docs","title":"0.2.0 - \ud83d\udcda Better docs","text":"<p>Mostly API documentation and notebooks, plus some bugfixes.</p>"},{"location":"CHANGELOG/#added_1","title":"Added","text":"<p>In PR #161: - Support for $$ math in sphinx docs. - Usage of sphinx extension for external links (introducing new directives like   <code>:gh:</code>, <code>:issue:</code> and <code>:tfl:</code> to construct standardised links to external   resources). - Only update auto-generated documentation files if there are changes. Some   minor additions to <code>update_docs.py</code>. - Parallelization of exact combinatorial Shapley. - Integrated KNN shapley into the main interface <code>compute_shapley_values</code>.</p>"},{"location":"CHANGELOG/#changed_1","title":"Changed","text":"<p>In PR #161: - Improved main docs and Shapley notebooks. Added or fixed many docstrings,   readme and documentation for contributors. Typos, grammar and style in code,   documentation and notebooks. - Internal renaming and rearranging in the parallelization and caching modules.</p>"},{"location":"CHANGELOG/#fixed_1","title":"Fixed","text":"<ul> <li>Bug in random matrix generation   PR #161.</li> <li>Bugs in MapReduceJob's <code>_chunkify</code> and <code>_backpressure</code> methods   PR #176.</li> </ul>"},{"location":"CHANGELOG/#010-first-release","title":"0.1.0 - \ud83c\udf89 first release","text":"<p>This is very first release of pyDVL.</p> <p>It contains:</p> <ul> <li> <p>Data Valuation Methods:</p> </li> <li> <p>Leave-One-Out</p> </li> <li>Influence Functions</li> <li>Shapley:<ul> <li>Exact Permutation and Combinatorial</li> <li>Montecarlo Permutation and Combinatorial</li> <li>Truncated Montecarlo Permutation</li> </ul> </li> <li>Caching of results with Memcached</li> <li>Parallelization of computations with Ray</li> <li>Documentation</li> <li>Notebooks containing examples of different use cases</li> </ul>"},{"location":"api/pydvl/","title":"API","text":""},{"location":"api/pydvl/#pydvl--the-python-data-valuation-library-api","title":"The Python Data Valuation Library API","text":"<p>This is the API documentation for the Python Data Valuation Library (PyDVL). Use the table of contents to access the documentation for each module.</p> <p>The two main modules you will want to look at are value and influence.</p>"},{"location":"api/pydvl/influence/","title":"Influence","text":"<p>This package contains algorithms for the computation of the influence function.</p> <p>Warning: Much of the code in this package is experimental or untested and is subject to modification. In particular, the package structure and basic API will probably change.</p>"},{"location":"api/pydvl/influence/general/","title":"General","text":"<p>This module contains influence calculation functions for general models, as introduced in (Koh and Liang, 2017)<sup>1</sup>.</p>"},{"location":"api/pydvl/influence/general/#pydvl.influence.general--references","title":"References","text":"<ol> <li> <p>Koh, P.W., Liang, P., 2017. Understanding Black-box Predictions via Influence Functions. In: Proceedings of the 34th International Conference on Machine Learning, pp. 1885\u20131894. PMLR.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/influence/general/#pydvl.influence.general.InfluenceType","title":"<code>InfluenceType</code>","text":"<p>             Bases: <code>str</code>, <code>Enum</code></p> <p>Enum representation for the types of influence.</p> ATTRIBUTE DESCRIPTION <code>Up</code> <p>Up-weighting a training point, see section 2.1 of (Koh and Liang, 2017)<sup>1</sup></p> <p> </p> <code>Perturbation</code> <p>Perturb a training point, see section 2.2 of (Koh and Liang, 2017)<sup>1</sup></p> <p> </p>"},{"location":"api/pydvl/influence/general/#pydvl.influence.general.compute_influence_factors","title":"<code>compute_influence_factors(model, training_data, test_data, inversion_method, *, hessian_perturbation=0.0, progress=False, **kwargs)</code>","text":"<p>Calculates influence factors of a model for training and test data.</p> <p>Given a test point \\(z_{test} = (x_{test}, y_{test})\\), a loss \\(L(z_{test}, \\theta)\\) (\\(\\theta\\) being the parameters of the model) and the Hessian of the model \\(H_{\\theta}\\), influence factors are defined as:</p> \\[ s_{test} = H_{\\theta}^{-1} \\operatorname{grad}_{\\theta} L(z_{test}, \\theta). \\] <p>They are used for efficient influence calculation. This method first (implicitly) calculates the Hessian and then (explicitly) finds the influence factors for the model using the given inversion method. The parameter <code>hessian_perturbation</code> is used to regularize the inversion of the Hessian. For more info, refer to (Koh and Liang, 2017)<sup>1</sup>, paragraph 3.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A model wrapped in the TwiceDifferentiable interface.</p> <p> TYPE: <code>TwiceDifferentiable</code> </p> <code>training_data</code> <p>DataLoader containing the training data.</p> <p> TYPE: <code>DataLoaderType</code> </p> <code>test_data</code> <p>DataLoader containing the test data.</p> <p> TYPE: <code>DataLoaderType</code> </p> <code>inversion_method</code> <p>Name of method for computing inverse hessian vector products.</p> <p> TYPE: <code>InversionMethod</code> </p> <code>hessian_perturbation</code> <p>Regularization of the hessian.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>progress</code> <p>If True, display progress bars.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>array</code> <p>An array of size (N, D) containing the influence factors for each dimension (D) and test sample (N).</p> <p> TYPE: <code>InverseHvpResult</code> </p> Source code in <code>src/pydvl/influence/general.py</code> <pre><code>def compute_influence_factors(\nmodel: TwiceDifferentiable,\ntraining_data: DataLoaderType,\ntest_data: DataLoaderType,\ninversion_method: InversionMethod,\n*,\nhessian_perturbation: float = 0.0,\nprogress: bool = False,\n**kwargs: Any,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Calculates influence factors of a model for training and test data.\n    Given a test point \\(z_{test} = (x_{test}, y_{test})\\), a loss \\(L(z_{test}, \\theta)\\)\n    (\\(\\theta\\) being the parameters of the model) and the Hessian of the model \\(H_{\\theta}\\),\n    influence factors are defined as:\n    \\[\n    s_{test} = H_{\\theta}^{-1} \\operatorname{grad}_{\\theta} L(z_{test}, \\theta).\n    \\]\n    They are used for efficient influence calculation. This method first (implicitly) calculates\n    the Hessian and then (explicitly) finds the influence factors for the model using the given\n    inversion method. The parameter `hessian_perturbation` is used to regularize the inversion of\n    the Hessian. For more info, refer to (Koh and Liang, 2017)&lt;sup&gt;&lt;a href=\"#koh_liang_2017\"&gt;1&lt;/a&gt;&lt;/sup&gt;, paragraph 3.\n    Args:\n        model: A model wrapped in the TwiceDifferentiable interface.\n        training_data: DataLoader containing the training data.\n        test_data: DataLoader containing the test data.\n        inversion_method: Name of method for computing inverse hessian vector products.\n        hessian_perturbation: Regularization of the hessian.\n        progress: If True, display progress bars.\n    Returns:\n        array: An array of size (N, D) containing the influence factors for each dimension (D) and test sample (N).\n    \"\"\"\ntensor_util: Type[TensorUtilities] = TensorUtilities.from_twice_differentiable(\nmodel\n)\nstack = tensor_util.stack\nunsqueeze = tensor_util.unsqueeze\ncat_gen = tensor_util.cat_gen\ncat = tensor_util.cat\ndef test_grads() -&gt; Generator[TensorType, None, None]:\nfor x_test, y_test in maybe_progress(\ntest_data, progress, desc=\"Batch Test Gradients\"\n):\nyield stack(\n[\nmodel.grad(inpt, target)\nfor inpt, target in zip(unsqueeze(x_test, 1), y_test)\n]\n)  # type:ignore\ntry:\n# if provided input_data implements __len__, pre-allocate the result tensor to reduce memory consumption\nresulting_shape = (len(test_data), model.num_params)  # type:ignore\nrhs = cat_gen(\ntest_grads(), resulting_shape, model  # type:ignore\n)  # type:ignore\nexcept Exception as e:\nlogger.warning(\nf\"Failed to pre-allocate result tensor: {e}\\n\"\nf\"Evaluate all resulting tensor and concatenate\"\n)\nrhs = cat(list(test_grads()))\nreturn solve_hvp(\ninversion_method,\nmodel,\ntraining_data,\nrhs,\nhessian_perturbation=hessian_perturbation,\n**kwargs,\n)\n</code></pre>"},{"location":"api/pydvl/influence/general/#pydvl.influence.general.compute_influences_up","title":"<code>compute_influences_up(model, input_data, influence_factors, *, progress=False)</code>","text":"<p>Given the model, the training points, and the influence factors, this function calculates the influences using the up-weighting method.</p> <p>The procedure involves two main steps: 1. Calculating the gradients of the model with respect to each training sample    (\\(\\operatorname{grad}_{\\theta} L\\), where \\(L\\) is the loss of a single point and \\(\\theta\\) are the    parameters of the model). 2. Multiplying each gradient with the influence factors.</p> <p>For a detailed description of the methodology, see section 2.1 of (Koh and Liang, 2017)<sup>1</sup>.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A model that implements the TwiceDifferentiable interface.</p> <p> TYPE: <code>TwiceDifferentiable</code> </p> <code>input_data</code> <p>DataLoader containing the samples for which the influence will be calculated.</p> <p> TYPE: <code>DataLoaderType</code> </p> <code>influence_factors</code> <p>Array containing pre-computed influence factors.</p> <p> TYPE: <code>TensorType</code> </p> <code>progress</code> <p>If set to True, progress bars will be displayed during computation.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>TensorType</code> <p>An array of shape [NxM], where N is the number of influence factors, and M is the number of input samples.</p> Source code in <code>src/pydvl/influence/general.py</code> <pre><code>def compute_influences_up(\nmodel: TwiceDifferentiable,\ninput_data: DataLoaderType,\ninfluence_factors: TensorType,\n*,\nprogress: bool = False,\n) -&gt; TensorType:\nr\"\"\"\n    Given the model, the training points, and the influence factors, this function calculates the\n    influences using the up-weighting method.\n    The procedure involves two main steps:\n    1. Calculating the gradients of the model with respect to each training sample\n       (\\(\\operatorname{grad}_{\\theta} L\\), where \\(L\\) is the loss of a single point and \\(\\theta\\) are the\n       parameters of the model).\n    2. Multiplying each gradient with the influence factors.\n    For a detailed description of the methodology, see section 2.1 of (Koh and Liang, 2017)&lt;sup&gt;&lt;a href=\"#koh_liang_2017\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    Args:\n        model: A model that implements the TwiceDifferentiable interface.\n        input_data: DataLoader containing the samples for which the influence will be calculated.\n        influence_factors: Array containing pre-computed influence factors.\n        progress: If set to True, progress bars will be displayed during computation.\n    Returns:\n        An array of shape [NxM], where N is the number of influence factors, and M is the number of input samples.\n    \"\"\"\ntensor_util: Type[TensorUtilities] = TensorUtilities.from_twice_differentiable(\nmodel\n)\nstack = tensor_util.stack\nunsqueeze = tensor_util.unsqueeze\ncat_gen = tensor_util.cat_gen\ncat = tensor_util.cat\neinsum = tensor_util.einsum\ndef train_grads() -&gt; Generator[TensorType, None, None]:\nfor x, y in maybe_progress(\ninput_data, progress, desc=\"Batch Split Input Gradients\"\n):\nyield stack(\n[model.grad(inpt, target) for inpt, target in zip(unsqueeze(x, 1), y)]\n)  # type:ignore\ntry:\n# if provided input_data implements __len__, pre-allocate the result tensor to reduce memory consumption\nresulting_shape = (len(input_data), model.num_params)  # type:ignore\ntrain_grad_tensor = cat_gen(\ntrain_grads(), resulting_shape, model  # type:ignore\n)  # type:ignore\nexcept Exception as e:\nlogger.warning(\nf\"Failed to pre-allocate result tensor: {e}\\n\"\nf\"Evaluate all resulting tensor and concatenate\"\n)\ntrain_grad_tensor = cat([x for x in train_grads()])  # type:ignore\nreturn einsum(\"ta,va-&gt;tv\", influence_factors, train_grad_tensor)  # type:ignore\n</code></pre>"},{"location":"api/pydvl/influence/general/#pydvl.influence.general.compute_influences_pert","title":"<code>compute_influences_pert(model, input_data, influence_factors, *, progress=False)</code>","text":"<p>Calculates the influence values based on the influence factors and training samples using the perturbation method.</p> <p>The process involves two main steps: 1. Calculating the gradient of the model with respect to each training sample    (\\(\\operatorname{grad}_{\\theta} L\\), where \\(L\\) is the loss of the model for a single data point and \\(\\theta\\)    are the parameters of the model). 2. Using the method TwiceDifferentiable.mvp    to efficiently compute the product of the    influence factors and \\(\\operatorname{grad}_x \\operatorname{grad}_{\\theta} L\\).</p> <p>For a detailed methodology, see section 2.2 of (Koh and Liang, 2017)<sup>1</sup>.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A model that implements the TwiceDifferentiable interface.</p> <p> TYPE: <code>TwiceDifferentiable</code> </p> <code>input_data</code> <p>DataLoader containing the samples for which the influence will be calculated.</p> <p> TYPE: <code>DataLoaderType</code> </p> <code>influence_factors</code> <p>Array containing pre-computed influence factors.</p> <p> TYPE: <code>TensorType</code> </p> <code>progress</code> <p>If set to True, progress bars will be displayed during computation.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>TensorType</code> <p>A 3D array with shape [NxMxP], where N is the number of influence factors, M is the number of input samples, and P is the number of features.</p> Source code in <code>src/pydvl/influence/general.py</code> <pre><code>def compute_influences_pert(\nmodel: TwiceDifferentiable,\ninput_data: DataLoaderType,\ninfluence_factors: TensorType,\n*,\nprogress: bool = False,\n) -&gt; TensorType:\nr\"\"\"\n    Calculates the influence values based on the influence factors and training samples using the perturbation method.\n    The process involves two main steps:\n    1. Calculating the gradient of the model with respect to each training sample\n       (\\(\\operatorname{grad}_{\\theta} L\\), where \\(L\\) is the loss of the model for a single data point and \\(\\theta\\)\n       are the parameters of the model).\n    2. Using the method [TwiceDifferentiable.mvp][pydvl.influence.twice_differentiable.TwiceDifferentiable.mvp]\n       to efficiently compute the product of the\n       influence factors and \\(\\operatorname{grad}_x \\operatorname{grad}_{\\theta} L\\).\n    For a detailed methodology, see section 2.2 of (Koh and Liang, 2017)&lt;sup&gt;&lt;a href=\"#koh_liang_2017\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    Args:\n        model: A model that implements the TwiceDifferentiable interface.\n        input_data: DataLoader containing the samples for which the influence will be calculated.\n        influence_factors: Array containing pre-computed influence factors.\n        progress: If set to True, progress bars will be displayed during computation.\n    Returns:\n        A 3D array with shape [NxMxP], where N is the number of influence factors,\n            M is the number of input samples, and P is the number of features.\n    \"\"\"\ntensor_util: Type[TensorUtilities] = TensorUtilities.from_twice_differentiable(\nmodel\n)\nstack = tensor_util.stack\ntu_slice = tensor_util.slice\nreshape = tensor_util.reshape\nget_element = tensor_util.get_element\nshape = tensor_util.shape\nall_pert_influences = []\nfor x, y in maybe_progress(\ninput_data,\nprogress,\ndesc=\"Batch Influence Perturbation\",\n):\nfor i in range(len(x)):\ntensor_x = tu_slice(x, i, i + 1)\ngrad_xy = model.grad(tensor_x, get_element(y, i), create_graph=True)\nperturbation_influences = model.mvp(\ngrad_xy,\ninfluence_factors,\nbackprop_on=tensor_x,\n)\nall_pert_influences.append(\nreshape(perturbation_influences, (-1, *shape(get_element(x, i))))\n)\nreturn stack(all_pert_influences, axis=1)  # type:ignore\n</code></pre>"},{"location":"api/pydvl/influence/general/#pydvl.influence.general.compute_influences","title":"<code>compute_influences(differentiable_model, training_data, *, test_data=None, input_data=None, inversion_method=InversionMethod.Direct, influence_type=InfluenceType.Up, hessian_regularization=0.0, progress=False, **kwargs)</code>","text":"<p>Calculates the influence of each input data point on the specified test points.</p> <p>This method operates in two primary stages: 1. Computes the influence factors for all test points concerning the model and its training data. 2. Uses these factors to derive the influences over the complete set of input data.</p> <p>The influence calculation relies on the twice-differentiable nature of the provided model.</p> PARAMETER  DESCRIPTION <code>differentiable_model</code> <p>A model bundled with its corresponding loss in the <code>TwiceDifferentiable</code> wrapper.</p> <p> TYPE: <code>TwiceDifferentiable</code> </p> <code>training_data</code> <p>DataLoader instance supplying the training data. This data is pivotal in computing the            Hessian matrix for the model's loss.</p> <p> TYPE: <code>DataLoaderType</code> </p> <code>test_data</code> <p>DataLoader instance with the test samples. Defaults to <code>training_data</code> if None.</p> <p> TYPE: <code>Optional[DataLoaderType]</code> DEFAULT: <code>None</code> </p> <code>input_data</code> <p>DataLoader instance holding samples whose influences need to be computed. Defaults to         <code>training_data</code> if None.</p> <p> TYPE: <code>Optional[DataLoaderType]</code> DEFAULT: <code>None</code> </p> <code>inversion_method</code> <p>An enumeration value determining the approach for inverting matrices or computing inverse operations, see [.inversion.InversionMethod]</p> <p> TYPE: <code>InversionMethod</code> DEFAULT: <code>Direct</code> </p> <code>progress</code> <p>A boolean indicating whether progress bars should be displayed during computation.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>influence_type</code> <p>Determines the methodology for computing influences. Valid choices include 'up' (for up-weighting) and 'perturbation'. For an in-depth understanding, see (Koh and Liang, 2017)<sup>1</sup>.</p> <p> TYPE: <code>InfluenceType</code> DEFAULT: <code>Up</code> </p> <code>hessian_regularization</code> <p>A lambda value used in Hessian regularization. The regularized Hessian, \\( H_{reg} \\), is computed as \\( H + \\lambda \\times I \\), where \\( I \\) is the identity matrix and \\( H \\) is the simple, unmodified Hessian. This regularization is typically utilized for more sophisticated models to ensure that the Hessian remains positive definite.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> RETURNS DESCRIPTION <code>TensorType</code> <p>The shape of this array varies based on the <code>influence_type</code>. If 'up', the shape is [NxM], where N denotes the number of test points and M denotes the number of training points. Conversely, if the influence_type is 'perturbation', the shape is [NxMxP], with P representing the number of input features.</p> Source code in <code>src/pydvl/influence/general.py</code> <pre><code>def compute_influences(\ndifferentiable_model: TwiceDifferentiable,\ntraining_data: DataLoaderType,\n*,\ntest_data: Optional[DataLoaderType] = None,\ninput_data: Optional[DataLoaderType] = None,\ninversion_method: InversionMethod = InversionMethod.Direct,\ninfluence_type: InfluenceType = InfluenceType.Up,\nhessian_regularization: float = 0.0,\nprogress: bool = False,\n**kwargs: Any,\n) -&gt; TensorType:  # type: ignore # ToDO fix typing\nr\"\"\"\n    Calculates the influence of each input data point on the specified test points.\n    This method operates in two primary stages:\n    1. Computes the influence factors for all test points concerning the model and its training data.\n    2. Uses these factors to derive the influences over the complete set of input data.\n    The influence calculation relies on the twice-differentiable nature of the provided model.\n    Args:\n        differentiable_model: A model bundled with its corresponding loss in the `TwiceDifferentiable` wrapper.\n        training_data: DataLoader instance supplying the training data. This data is pivotal in computing the\n                       Hessian matrix for the model's loss.\n        test_data: DataLoader instance with the test samples. Defaults to `training_data` if None.\n        input_data: DataLoader instance holding samples whose influences need to be computed. Defaults to\n                    `training_data` if None.\n        inversion_method: An enumeration value determining the approach for inverting matrices\n            or computing inverse operations, see [.inversion.InversionMethod]\n        progress: A boolean indicating whether progress bars should be displayed during computation.\n        influence_type: Determines the methodology for computing influences.\n            Valid choices include 'up' (for up-weighting) and 'perturbation'.\n            For an in-depth understanding, see (Koh and Liang, 2017)&lt;sup&gt;&lt;a href=\"#koh_liang_2017\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n        hessian_regularization: A lambda value used in Hessian regularization. The regularized Hessian, \\( H_{reg} \\),\n            is computed as \\( H + \\lambda \\times I \\), where \\( I \\) is the identity matrix and \\( H \\)\n            is the simple, unmodified Hessian. This regularization is typically utilized for more\n            sophisticated models to ensure that the Hessian remains positive definite.\n    Returns:\n        The shape of this array varies based on the `influence_type`. If 'up', the shape is [NxM], where\n            N denotes the number of test points and M denotes the number of training points. Conversely, if the\n            influence_type is 'perturbation', the shape is [NxMxP], with P representing the number of input features.\n    \"\"\"\nif input_data is None:\ninput_data = deepcopy(training_data)\nif test_data is None:\ntest_data = deepcopy(training_data)\ninfluence_factors, _ = compute_influence_factors(\ndifferentiable_model,\ntraining_data,\ntest_data,\ninversion_method,\nhessian_perturbation=hessian_regularization,\nprogress=progress,\n**kwargs,\n)\nreturn influence_type_registry[influence_type](\ndifferentiable_model,\ninput_data,\ninfluence_factors,\nprogress=progress,\n)\n</code></pre>"},{"location":"api/pydvl/influence/inversion/","title":"Inversion","text":"<p>Contains methods to invert the hessian vector product.</p>"},{"location":"api/pydvl/influence/inversion/#pydvl.influence.inversion.InversionMethod","title":"<code>InversionMethod</code>","text":"<p>             Bases: <code>str</code>, <code>Enum</code></p> <p>Different inversion methods types.</p>"},{"location":"api/pydvl/influence/inversion/#pydvl.influence.inversion.InversionRegistry","title":"<code>InversionRegistry</code>","text":"<p>A registry to hold inversion methods for different models.</p>"},{"location":"api/pydvl/influence/inversion/#pydvl.influence.inversion.InversionRegistry.register","title":"<code>register(model_type, inversion_method, overwrite=False)</code>  <code>classmethod</code>","text":"<p>Register a function for a specific model type and inversion method.</p> <p>The function to be registered must conform to the following signature: <code>(model: TwiceDifferentiable, training_data: DataLoaderType, b: TensorType, hessian_perturbation: float = 0.0, ...)</code>.</p> PARAMETER  DESCRIPTION <code>model_type</code> <p>The type of the model the function should be registered for.</p> <p> TYPE: <code>Type[TwiceDifferentiable]</code> </p> <code>inversion_method</code> <p>The inversion method the function should be registered for.</p> <p> TYPE: <code>InversionMethod</code> </p> <code>overwrite</code> <p>If <code>True</code>, allows overwriting of an existing registered function for the same model type and inversion method. If <code>False</code>, logs a warning when attempting to register a function for an already registered model type and inversion method.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RAISES DESCRIPTION <code>TypeError</code> <p>If the provided model_type or inversion_method are of the wrong type.</p> <code>ValueError</code> <p>If the function to be registered does not match the required signature.</p> RETURNS DESCRIPTION <p>A decorator for registering a function.</p> Source code in <code>src/pydvl/influence/inversion.py</code> <pre><code>@classmethod\ndef register(\ncls,\nmodel_type: Type[TwiceDifferentiable],\ninversion_method: InversionMethod,\noverwrite: bool = False,\n):\n\"\"\"\n    Register a function for a specific model type and inversion method.\n    The function to be registered must conform to the following signature:\n    `(model: TwiceDifferentiable, training_data: DataLoaderType, b: TensorType,\n    hessian_perturbation: float = 0.0, ...)`.\n    Args:\n        model_type: The type of the model the function should be registered for.\n        inversion_method: The inversion method the function should be\n            registered for.\n        overwrite: If ``True``, allows overwriting of an existing registered\n            function for the same model type and inversion method. If ``False``,\n            logs a warning when attempting to register a function for an already\n            registered model type and inversion method.\n    Raises:\n        TypeError: If the provided model_type or inversion_method are of the wrong type.\n        ValueError: If the function to be registered does not match the required signature.\n    Returns:\n        A decorator for registering a function.\n    \"\"\"\nif not isinstance(model_type, type):\nraise TypeError(\nf\"'model_type' is of type {type(model_type)} but should be a Type[TwiceDifferentiable]\"\n)\nif not isinstance(inversion_method, InversionMethod):\nraise TypeError(\nf\"'inversion_method' must be an 'InversionMethod' \"\nf\"but has type {type(inversion_method)} instead.\"\n)\nkey = (model_type, inversion_method)\ndef decorator(func):\nif not overwrite and key in cls.registry:\nwarnings.warn(\nf\"There is already a function registered for model type {model_type} \"\nf\"and inversion method {inversion_method}. \"\nf\"To overwrite the existing function {cls.registry.get(key)} with {func}, set overwrite to True.\"\n)\nsig = inspect.signature(func)\nparams = list(sig.parameters.values())\nexpected_args = [\n(\"model\", model_type),\n(\"training_data\", DataLoaderType.__bound__),\n(\"b\", model_type.tensor_type()),\n(\"hessian_perturbation\", float),\n]\nfor (name, typ), param in zip(expected_args, params):\nif not (\nisinstance(param.annotation, typ)\nor issubclass(param.annotation, typ)\n):\nraise ValueError(\nf'Parameter \"{name}\" must be of type \"{typ.__name__}\"'\n)\n@functools.wraps(func)\ndef wrapper(*args, **kwargs):\nreturn func(*args, **kwargs)\ncls.registry[key] = wrapper\nreturn wrapper\nreturn decorator\n</code></pre>"},{"location":"api/pydvl/influence/inversion/#pydvl.influence.inversion.InversionRegistry.call","title":"<code>call(inversion_method, model, training_data, b, hessian_perturbation, **kwargs)</code>  <code>classmethod</code>","text":"<p>Call a registered function with the provided parameters.</p> PARAMETER  DESCRIPTION <code>inversion_method</code> <p>The inversion method to use.</p> <p> TYPE: <code>InversionMethod</code> </p> <code>model</code> <p>A model wrapped in the TwiceDifferentiable interface.</p> <p> TYPE: <code>TwiceDifferentiable</code> </p> <code>training_data</code> <p>The training data to use.</p> <p> TYPE: <code>DataLoaderType</code> </p> <code>b</code> <p>Array as the right hand side of the equation \\(Ax = b\\).</p> <p> TYPE: <code>TensorType</code> </p> <code>hessian_perturbation</code> <p>Regularization of the hessian.</p> <p> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the inversion method.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>InverseHvpResult</code> <p>An instance of InverseHvpResult, that contains an array, which solves the inverse problem, i.e. it returns \\(x\\) such that \\(Ax = b\\), and a dictionary containing information about the inversion process.</p> Source code in <code>src/pydvl/influence/inversion.py</code> <pre><code>@classmethod\ndef call(\ncls,\ninversion_method: InversionMethod,\nmodel: TwiceDifferentiable,\ntraining_data: DataLoaderType,\nb: TensorType,\nhessian_perturbation,\n**kwargs,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Call a registered function with the provided parameters.\n    Args:\n        inversion_method: The inversion method to use.\n        model: A model wrapped in the TwiceDifferentiable interface.\n        training_data: The training data to use.\n        b: Array as the right hand side of the equation \\(Ax = b\\).\n        hessian_perturbation: Regularization of the hessian.\n        kwargs: Additional keyword arguments to pass to the inversion method.\n    Returns:\n        An instance of [InverseHvpResult][pydvl.influence.twice_differentiable.InverseHvpResult],\n            that contains an array, which solves the inverse problem,\n            i.e. it returns \\(x\\) such that \\(Ax = b\\), and a dictionary containing information\n            about the inversion process.\n    \"\"\"\nreturn cls.get(type(model), inversion_method)(\nmodel, training_data, b, hessian_perturbation, **kwargs\n)\n</code></pre>"},{"location":"api/pydvl/influence/inversion/#pydvl.influence.inversion.solve_hvp","title":"<code>solve_hvp(inversion_method, model, training_data, b, *, hessian_perturbation=0.0, **kwargs)</code>","text":"<p>Finds \\( x \\) such that \\( Ax = b \\), where \\( A \\) is the hessian of the model, and \\( b \\) a vector. Depending on the inversion method, the hessian is either calculated directly and then inverted, or implicitly and then inverted through matrix vector product. The method also allows to add a small regularization term (hessian_perturbation) to facilitate inversion of non fully trained models.</p> PARAMETER  DESCRIPTION <code>inversion_method</code> <p> TYPE: <code>InversionMethod</code> </p> <code>model</code> <p>A model wrapped in the TwiceDifferentiable interface.</p> <p> TYPE: <code>TwiceDifferentiable</code> </p> <code>training_data</code> <p> TYPE: <code>DataLoaderType</code> </p> <code>b</code> <p>Array as the right hand side of the equation \\( Ax = b \\)</p> <p> TYPE: <code>TensorType</code> </p> <code>hessian_perturbation</code> <p>regularization of the hessian.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>kwargs</code> <p>kwargs to pass to the inversion method.</p> <p> TYPE: <code>Any</code> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>InverseHvpResult</code> <p>Instance of InverseHvpResult, with an array that solves the inverse problem, i.e., it returns \\( x \\) such that \\( Ax = b \\) and a dictionary containing information about the inversion process.</p> Source code in <code>src/pydvl/influence/inversion.py</code> <pre><code>def solve_hvp(\ninversion_method: InversionMethod,\nmodel: TwiceDifferentiable,\ntraining_data: DataLoaderType,\nb: TensorType,\n*,\nhessian_perturbation: float = 0.0,\n**kwargs: Any,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Finds \\( x \\) such that \\( Ax = b \\), where \\( A \\) is the hessian of the model,\n    and \\( b \\) a vector. Depending on the inversion method, the hessian is either\n    calculated directly and then inverted, or implicitly and then inverted through\n    matrix vector product. The method also allows to add a small regularization term\n    (hessian_perturbation) to facilitate inversion of non fully trained models.\n    Args:\n        inversion_method:\n        model: A model wrapped in the TwiceDifferentiable interface.\n        training_data:\n        b: Array as the right hand side of the equation \\( Ax = b \\)\n        hessian_perturbation: regularization of the hessian.\n        kwargs: kwargs to pass to the inversion method.\n    Returns:\n        Instance of [InverseHvpResult][pydvl.influence.twice_differentiable.InverseHvpResult], with\n            an array that solves the inverse problem, i.e., it returns \\( x \\) such that \\( Ax = b \\)\n            and a dictionary containing information about the inversion process.\n    \"\"\"\nreturn InversionRegistry.call(\ninversion_method,\nmodel,\ntraining_data,\nb,\nhessian_perturbation=hessian_perturbation,\n**kwargs,\n)\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/","title":"Twice differentiable","text":""},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorType","title":"<code>TensorType = TypeVar('TensorType', bound=Sequence)</code>  <code>module-attribute</code>","text":"<p>Type variable for tensors, i.e. sequences of numbers</p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.ModelType","title":"<code>ModelType = TypeVar('ModelType', bound='TwiceDifferentiable')</code>  <code>module-attribute</code>","text":"<p>Type variable for twice differentiable models</p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.DataLoaderType","title":"<code>DataLoaderType = TypeVar('DataLoaderType', bound=Iterable)</code>  <code>module-attribute</code>","text":"<p>Type variable for data loaders</p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.InverseHvpResult","title":"<code>InverseHvpResult</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Generic[TensorType]</code></p> <p>Container class for results of solving a problem \\(Ax=b\\)</p> PARAMETER  DESCRIPTION <code>x</code> <p>solution of a problem \\(Ax=b\\)</p> <p> TYPE: <code>TensorType</code> </p> <code>info</code> <p>additional information, to couple with the solution itself</p> <p> TYPE: <code>Dict[str, Any]</code> </p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TwiceDifferentiable","title":"<code>TwiceDifferentiable</code>","text":"<p>             Bases: <code>ABC</code>, <code>Generic[TensorType]</code></p> <p>Abstract base class for wrappers of differentiable models and losses. Meant to be subclassed for each supported framework. Provides methods to compute gradients and second derivative of the loss wrt. the model parameters</p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TwiceDifferentiable.num_params","title":"<code>num_params: int</code>  <code>abstractmethod</code> <code>property</code>","text":"<p>Returns the number of parameters of the model</p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TwiceDifferentiable.parameters","title":"<code>parameters: List[TensorType]</code>  <code>abstractmethod</code> <code>property</code>","text":"<p>Returns all the model parameters that require differentiation</p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TwiceDifferentiable.grad","title":"<code>grad(x, y, create_graph=False)</code>","text":"<p>Calculates gradient of model parameters with respect to the model parameters.</p> PARAMETER  DESCRIPTION <code>x</code> <p>A matrix representing the features \\(x_i\\).</p> <p> TYPE: <code>TensorType</code> </p> <code>y</code> <p>A matrix representing the target values \\(y_i\\).</p> <p> TYPE: <code>TensorType</code> </p> <code>create_graph</code> <p>Used for further differentiation on input parameters.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>TensorType</code> <p>An array with the gradients of the model.</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>def grad(\nself, x: TensorType, y: TensorType, create_graph: bool = False\n) -&gt; TensorType:\nr\"\"\"\n    Calculates gradient of model parameters with respect to the model parameters.\n    Args:\n        x: A matrix representing the features \\(x_i\\).\n        y: A matrix representing the target values \\(y_i\\).\n        create_graph: Used for further differentiation on input parameters.\n    Returns:\n        An array with the gradients of the model.\n    \"\"\"\npass\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TwiceDifferentiable.hessian","title":"<code>hessian(x, y)</code>","text":"<p>Calculates the full Hessian of \\(L(f(x),y)\\) with respect to the model parameters given data \\(x\\) and \\(y\\).</p> PARAMETER  DESCRIPTION <code>x</code> <p>An array representing the features \\(x_i\\).</p> <p> TYPE: <code>TensorType</code> </p> <code>y</code> <p>An array representing the target values \\(y_i\\).</p> <p> TYPE: <code>TensorType</code> </p> RETURNS DESCRIPTION <code>TensorType</code> <p>A tensor representing the Hessian of the model, i.e. the second derivative with respect to the model parameters.</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>def hessian(self, x: TensorType, y: TensorType) -&gt; TensorType:\nr\"\"\"\n    Calculates the full Hessian of \\(L(f(x),y)\\) with respect to the model parameters given data \\(x\\) and \\(y\\).\n    Args:\n        x: An array representing the features \\(x_i\\).\n        y: An array representing the target values \\(y_i\\).\n    Returns:\n        A tensor representing the Hessian of the model, i.e. the second derivative\n            with respect to the model parameters.\n    \"\"\"\npass\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TwiceDifferentiable.mvp","title":"<code>mvp(grad_xy, v, backprop_on, *, progress=False)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Calculates the second order derivative of the model along directions \\(v\\). The second order derivative can be selected through the <code>backprop_on</code> argument.</p> PARAMETER  DESCRIPTION <code>grad_xy</code> <p>An array [P] holding the gradients of the model parameters with respect to input \\(x\\) and labels \\(y\\). \\(P\\) is the number of parameters of the model. Typically obtained through <code>self.grad</code>.</p> <p> TYPE: <code>TensorType</code> </p> <code>v</code> <p>An array ([DxP] or even one-dimensional [D]) which multiplies the matrix. \\(D\\) is the number of directions.</p> <p> TYPE: <code>TensorType</code> </p> <code>progress</code> <p>If <code>True</code>, progress is displayed.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>backprop_on</code> <p>Tensor used in the second backpropagation. The first one is along \\(x\\) and \\(y\\) as defined via <code>grad_xy</code>.</p> <p> TYPE: <code>TensorType</code> </p> RETURNS DESCRIPTION <code>TensorType</code> <p>A matrix representing the implicit matrix-vector product of the model along the given directions. Output shape is [DxM], where \\(M\\) is the number of elements of <code>backprop_on</code>.</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef mvp(\ngrad_xy: TensorType,\nv: TensorType,\nbackprop_on: TensorType,\n*,\nprogress: bool = False,\n) -&gt; TensorType:\nr\"\"\"\n    Calculates the second order derivative of the model along directions \\(v\\).\n    The second order derivative can be selected through the `backprop_on` argument.\n    Args:\n        grad_xy: An array [P] holding the gradients of the model parameters with respect to input \\(x\\) and\n            labels \\(y\\). \\(P\\) is the number of parameters of the model. Typically obtained through `self.grad`.\n        v: An array ([DxP] or even one-dimensional [D]) which multiplies the matrix.\n            \\(D\\) is the number of directions.\n        progress: If `True`, progress is displayed.\n        backprop_on: Tensor used in the second backpropagation. The first one is along \\(x\\) and \\(y\\)\n            as defined via `grad_xy`.\n    Returns:\n        A matrix representing the implicit matrix-vector product of the model along the given directions.\n            Output shape is [DxM], where \\(M\\) is the number of elements of `backprop_on`.\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities","title":"<code>TensorUtilities</code>","text":"<p>             Bases: <code>Generic[TensorType, ModelType]</code>, <code>ABC</code></p>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.__init_subclass__","title":"<code>__init_subclass__(**kwargs)</code>","text":"<p>Automatically registers non-abstract subclasses in the registry.</p> <p>This method checks if <code>twice_differentiable_type</code> is defined in the subclass and if it is of the correct type. If either attribute is missing or incorrect, a <code>TypeError</code> is raised.</p> PARAMETER  DESCRIPTION <code>kwargs</code> <p>Additional keyword arguments.</p> <p> DEFAULT: <code>{}</code> </p> RAISES DESCRIPTION <code>TypeError</code> <p>If the subclass does not define <code>twice_differentiable_type</code>, or if it is not of the correct type.</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>def __init_subclass__(cls, **kwargs):\n\"\"\"\n    Automatically registers non-abstract subclasses in the registry.\n    This method checks if `twice_differentiable_type` is defined in the subclass and if it is of the correct type.\n    If either attribute is missing or incorrect, a `TypeError` is raised.\n    Args:\n        kwargs: Additional keyword arguments.\n    Raises:\n        TypeError: If the subclass does not define `twice_differentiable_type`, or if it is not of the correct type.\n    \"\"\"\nif not hasattr(cls, \"twice_differentiable_type\") or not isinstance(\ncls.twice_differentiable_type, type\n):\nraise TypeError(\nf\"'twice_differentiable_type' must be a Type[TwiceDifferentiable]\"\n)\ncls.registry[cls.twice_differentiable_type] = cls\nsuper().__init_subclass__(**kwargs)\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.einsum","title":"<code>einsum(equation, *operands)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Sums the product of the elements of the input <code>operands</code> along dimensions specified using a notation based on the Einstein summation convention.</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef einsum(equation, *operands) -&gt; TensorType:\n\"\"\"Sums the product of the elements of the input `operands` along dimensions specified using a notation\n    based on the Einstein summation convention.\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.cat","title":"<code>cat(a, **kwargs)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Concatenates a sequence of tensors into a single torch tensor</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef cat(a: Sequence[TensorType], **kwargs) -&gt; TensorType:\n\"\"\"Concatenates a sequence of tensors into a single torch tensor\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.stack","title":"<code>stack(a, **kwargs)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Stacks a sequence of tensors into a single torch tensor</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef stack(a: Sequence[TensorType], **kwargs) -&gt; TensorType:\n\"\"\"Stacks a sequence of tensors into a single torch tensor\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.unsqueeze","title":"<code>unsqueeze(x, dim)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Add a singleton dimension at a specified position in a tensor</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef unsqueeze(x: TensorType, dim: int) -&gt; TensorType:\n\"\"\"Add a singleton dimension at a specified position in a tensor\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.get_element","title":"<code>get_element(x, idx)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Get the tensor element x[i] from the first non-singular dimension</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef get_element(x: TensorType, idx: int) -&gt; TensorType:\n\"\"\"Get the tensor element x[i] from the first non-singular dimension\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.slice","title":"<code>slice(x, start, stop, axis=0)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Slice a tensor in the provided axis</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef slice(x: TensorType, start: int, stop: int, axis: int = 0) -&gt; TensorType:\n\"\"\"Slice a tensor in the provided axis\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.shape","title":"<code>shape(x)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Slice a tensor in the provided axis</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef shape(x: TensorType) -&gt; Tuple[int, ...]:\n\"\"\"Slice a tensor in the provided axis\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.reshape","title":"<code>reshape(x, shape)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Reshape a tensor to the provided shape</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef reshape(x: TensorType, shape: Tuple[int, ...]) -&gt; TensorType:\n\"\"\"Reshape a tensor to the provided shape\"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.cat_gen","title":"<code>cat_gen(a, resulting_shape, model)</code>  <code>abstractmethod</code> <code>staticmethod</code>","text":"<p>Concatenate tensors from a generator. Resulting tensor is of shape resulting_shape and compatible to model</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@staticmethod\n@abstractmethod\ndef cat_gen(\na: Generator[TensorType, None, None],\nresulting_shape: Tuple[int, ...],\nmodel: ModelType,\n) -&gt; TensorType:\n\"\"\"Concatenate tensors from a generator. Resulting tensor is of shape resulting_shape\n    and compatible to model\n    \"\"\"\n</code></pre>"},{"location":"api/pydvl/influence/twice_differentiable/#pydvl.influence.twice_differentiable.TensorUtilities.from_twice_differentiable","title":"<code>from_twice_differentiable(twice_diff)</code>  <code>classmethod</code>","text":"<p>Factory method to create an instance of a subclass TensorUtilities from an instance of a subclass of TwiceDifferentiable.</p> PARAMETER  DESCRIPTION <code>twice_diff</code> <p>An instance of a subclass of TwiceDifferentiable for which a corresponding TensorUtilities object is required.</p> <p> TYPE: <code>TwiceDifferentiable</code> </p> RETURNS DESCRIPTION <code>Type[TensorUtilities]</code> <p>An subclass of TensorUtilities registered to the provided subclass instance of TwiceDifferentiable object.</p> RAISES DESCRIPTION <code>KeyError</code> <p>If there's no registered TensorUtilities for the provided TwiceDifferentiable type.</p> Source code in <code>src/pydvl/influence/twice_differentiable.py</code> <pre><code>@classmethod\ndef from_twice_differentiable(\ncls,\ntwice_diff: TwiceDifferentiable,\n) -&gt; Type[\"TensorUtilities\"]:\n\"\"\"\n    Factory method to create an instance of a subclass\n    [TensorUtilities][pydvl.influence.twice_differentiable.TensorUtilities] from an instance of a subclass of\n    [TwiceDifferentiable][pydvl.influence.twice_differentiable.TwiceDifferentiable].\n    Args:\n        twice_diff: An instance of a subclass of\n            [TwiceDifferentiable][pydvl.influence.twice_differentiable.TwiceDifferentiable]\n            for which a corresponding [TensorUtilities][pydvl.influence.twice_differentiable.TensorUtilities]\n            object is required.\n    Returns:\n        An subclass of [TensorUtilities][pydvl.influence.twice_differentiable.TensorUtilities]\n            registered to the provided subclass instance of\n            [TwiceDifferentiable][pydvl.influence.twice_differentiable.TwiceDifferentiable] object.\n    Raises:\n        KeyError: If there's no registered [TensorUtilities][pydvl.influence.twice_differentiable.TensorUtilities]\n            for the provided [TwiceDifferentiable][pydvl.influence.twice_differentiable.TwiceDifferentiable] type.\n    \"\"\"\ntu = cls.registry.get(type(twice_diff), None)\nif tu is None:\nraise KeyError(\nf\"No registered TensorUtilities for the type {type(twice_diff).__name__}\"\n)\nreturn tu\n</code></pre>"},{"location":"api/pydvl/influence/torch/","title":"Torch","text":""},{"location":"api/pydvl/influence/torch/functional/","title":"Functional","text":""},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.hvp","title":"<code>hvp(func, params, vec, reverse_only=True)</code>","text":"<p>Computes the Hessian-vector product (HVP) for a given function at given parameters, i.e.</p> \\[\\nabla_{\\theta} \\nabla_{\\theta} f (\\theta)\\cdot v\\] <p>This function can operate in two modes, either reverse-mode autodiff only or both forward- and reverse-mode autodiff.</p> PARAMETER  DESCRIPTION <code>func</code> <p>The scalar-valued function for which the HVP is computed.</p> <p> TYPE: <code>Callable[[TorchTensorContainerType], Tensor]</code> </p> <code>params</code> <p>The parameters at which the HVP is computed.</p> <p> TYPE: <code>TorchTensorContainerType</code> </p> <code>vec</code> <p>The vector with which the Hessian is multiplied.</p> <p> TYPE: <code>TorchTensorContainerType</code> </p> <code>reverse_only</code> <p>Whether to use only reverse-mode autodiff (True, default) or both forward- and reverse-mode autodiff (False).</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>TorchTensorContainerType</code> <p>The HVP of the function at the given parameters with the given vector.</p> <p>Example: <pre><code>&gt;&gt;&gt; def f(z): return torch.sum(z**2)\n&gt;&gt;&gt; u = torch.ones(10, requires_grad=True)\n&gt;&gt;&gt; v = torch.ones(10)\n&gt;&gt;&gt; hvp_vec = hvp(f, u, v)\n&gt;&gt;&gt; assert torch.allclose(hvp_vec, torch.full((10, ), 2.0))\n</code></pre></p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def hvp(\nfunc: Callable[[TorchTensorContainerType], torch.Tensor],\nparams: TorchTensorContainerType,\nvec: TorchTensorContainerType,\nreverse_only: bool = True,\n) -&gt; TorchTensorContainerType:\nr\"\"\"\n    Computes the Hessian-vector product (HVP) for a given function at given parameters, i.e.\n    \\[\\nabla_{\\theta} \\nabla_{\\theta} f (\\theta)\\cdot v\\]\n    This function can operate in two modes, either reverse-mode autodiff only or both\n    forward- and reverse-mode autodiff.\n    Args:\n        func: The scalar-valued function for which the HVP is computed.\n        params: The parameters at which the HVP is computed.\n        vec: The vector with which the Hessian is multiplied.\n        reverse_only: Whether to use only reverse-mode autodiff\n            (True, default) or both forward- and reverse-mode autodiff (False).\n    Returns:\n       The HVP of the function at the given parameters with the given vector.\n    Example:\n    ```python\n    &gt;&gt;&gt; def f(z): return torch.sum(z**2)\n    &gt;&gt;&gt; u = torch.ones(10, requires_grad=True)\n    &gt;&gt;&gt; v = torch.ones(10)\n    &gt;&gt;&gt; hvp_vec = hvp(f, u, v)\n    &gt;&gt;&gt; assert torch.allclose(hvp_vec, torch.full((10, ), 2.0))\n    ```\n    \"\"\"\noutput: TorchTensorContainerType\nif reverse_only:\n_, vjp_fn = vjp(grad(func), params)\noutput = vjp_fn(vec)[0]\nelse:\noutput = jvp(grad(func), (params,), (vec,))[1]\nreturn output\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.batch_hvp_gen","title":"<code>batch_hvp_gen(model, loss, data_loader, reverse_only=True)</code>","text":"<p>Generates a sequence of batch Hessian-vector product (HVP) computations for the provided model, loss function, and data loader. If \\(f_i\\) is the model's loss on the \\(i\\)-th batch and \\(\\theta\\) the model parameters, this is the sequence of the callable matrix vector products for the matrices</p> \\[\\nabla_{\\theta}\\nabla_{\\theta}f_i(\\theta), \\quad i=1,\\dots, \\text{num_batches} \\] <p>i.e. iterating over the data_loader, yielding partial function calls for calculating HVPs.</p> PARAMETER  DESCRIPTION <code>model</code> <p>The PyTorch model for which the HVP is calculated.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>The loss function used to calculate the gradient and HVP.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>data_loader</code> <p>PyTorch DataLoader object containing the dataset for which the HVP is calculated.</p> <p> TYPE: <code>DataLoader</code> </p> <code>reverse_only</code> <p>Whether to use only reverse-mode autodiff (True, default) or both forward- and reverse-mode autodiff (False).</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> YIELDS DESCRIPTION <code>Callable[[Tensor], Tensor]</code> <p>Partial functions <code>H_{batch}(vec)=hvp(model, loss, inputs, targets, vec)</code> that when called, will compute the Hessian-vector product H(vec) for the given model and loss in a batch-wise manner, where (inputs, targets) coming from one batch.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def batch_hvp_gen(\nmodel: torch.nn.Module,\nloss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\ndata_loader: DataLoader,\nreverse_only: bool = True,\n) -&gt; Generator[Callable[[torch.Tensor], torch.Tensor], None, None]:\nr\"\"\"\n    Generates a sequence of batch Hessian-vector product (HVP) computations for the provided model, loss function,\n    and data loader. If \\(f_i\\) is the model's loss on the \\(i\\)-th batch and \\(\\theta\\) the model parameters,\n    this is the sequence of the callable matrix vector products for the matrices\n    \\[\\nabla_{\\theta}\\nabla_{\\theta}f_i(\\theta), \\quad i=1,\\dots, \\text{num_batches} \\]\n    i.e. iterating over the data_loader, yielding partial function calls for calculating HVPs.\n    Args:\n        model: The PyTorch model for which the HVP is calculated.\n        loss: The loss function used to calculate the gradient and HVP.\n        data_loader: PyTorch DataLoader object containing the dataset for which the HVP is calculated.\n        reverse_only: Whether to use only reverse-mode autodiff\n            (True, default) or both forward- and reverse-mode autodiff (False).\n    Yields:\n        Partial functions `H_{batch}(vec)=hvp(model, loss, inputs, targets, vec)` that when called,\n            will compute the Hessian-vector product H(vec) for the given model and loss in a batch-wise manner, where\n            (inputs, targets) coming from one batch.\n    \"\"\"\nfor inputs, targets in iter(data_loader):\nbatch_loss = batch_loss_function(model, loss, inputs, targets)\nmodel_params = dict(model.named_parameters())\ndef batch_hvp(vec: torch.Tensor):\nreturn flatten_tensors_to_vector(\nhvp(\nbatch_loss,\nmodel_params,\nalign_structure(model_params, vec),\nreverse_only=reverse_only,\n).values()\n)\nyield batch_hvp\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.empirical_loss_function","title":"<code>empirical_loss_function(model, loss, data_loader)</code>","text":"<p>Creates a function to compute the empirical loss of a given model on a given dataset. If we denote the model parameters with \\( \\theta \\), the resulting function approximates:</p> \\[f(\\theta) = \\frac{1}{N}\\sum_{i=1}^N \\operatorname{loss}(y_i, \\operatorname{model}(\\theta, x_i))\\] <p>Args: - model: The model for which the loss should be computed. - loss: The loss function to be used. - data_loader: The data loader for iterating over the dataset.</p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor]], Tensor]</code> <p>A function that computes the empirical loss of the model on the dataset for given model parameters.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def empirical_loss_function(\nmodel: torch.nn.Module,\nloss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\ndata_loader: DataLoader,\n) -&gt; Callable[[Dict[str, torch.Tensor]], torch.Tensor]:\nr\"\"\"\n    Creates a function to compute the empirical loss of a given model on a given dataset.\n    If we denote the model parameters with \\( \\theta \\), the resulting function approximates:\n    \\[f(\\theta) = \\frac{1}{N}\\sum_{i=1}^N \\operatorname{loss}(y_i, \\operatorname{model}(\\theta, x_i))\\]\n    Args:\n    - model: The model for which the loss should be computed.\n    - loss: The loss function to be used.\n    - data_loader: The data loader for iterating over the dataset.\n    Returns:\n        A function that computes the empirical loss of the model on the dataset for given model parameters.\n    \"\"\"\ndef empirical_loss(params: Dict[str, torch.Tensor]):\ntotal_loss = to_model_device(torch.zeros((), requires_grad=True), model)\ntotal_samples = to_model_device(torch.zeros(()), model)\nfor x, y in iter(data_loader):\noutput = functional_call(\nmodel, params, (to_model_device(x, model),), strict=True\n)\nloss_value = loss(output, to_model_device(y, model))\ntotal_loss = total_loss + loss_value * x.size(0)\ntotal_samples += x.size(0)\nreturn total_loss / total_samples\nreturn empirical_loss\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.batch_loss_function","title":"<code>batch_loss_function(model, loss, x, y)</code>","text":"<p>Creates a function to compute the loss of a given model on a given batch of data, i.e. for the \\(i\\)-th batch \\(B_i\\)</p> \\[\\frac{1}{|B_i|}\\sum_{x,y \\in B_i} \\operatorname{loss}(y, \\operatorname{model}(\\theta, x))\\] PARAMETER  DESCRIPTION <code>model</code> <p>The model for which the loss should be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>The loss function to be used.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>x</code> <p>The input data for the batch.</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>The true labels for the batch.</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Callable[[Dict[str, Tensor]], Tensor]</code> <p>A function that computes the loss of the model on the batch for given model parameters.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def batch_loss_function(\nmodel: torch.nn.Module,\nloss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\nx: torch.Tensor,\ny: torch.Tensor,\n) -&gt; Callable[[Dict[str, torch.Tensor]], torch.Tensor]:\nr\"\"\"\n    Creates a function to compute the loss of a given model on a given batch of data, i.e. for the $i$-th batch $B_i$\n    \\[\\frac{1}{|B_i|}\\sum_{x,y \\in B_i} \\operatorname{loss}(y, \\operatorname{model}(\\theta, x))\\]\n    Args:\n        model: The model for which the loss should be computed.\n        loss: The loss function to be used.\n        x: The input data for the batch.\n        y: The true labels for the batch.\n    Returns:\n        A function that computes the loss of the model on the batch for given model parameters.\n    \"\"\"\ndef batch_loss(params: Dict[str, torch.Tensor]):\noutputs = functional_call(\nmodel, params, (to_model_device(x, model),), strict=True\n)\nreturn loss(outputs, y)\nreturn batch_loss\n</code></pre>"},{"location":"api/pydvl/influence/torch/functional/#pydvl.influence.torch.functional.get_hvp_function","title":"<code>get_hvp_function(model, loss, data_loader, use_hessian_avg=True, reverse_only=True, track_gradients=False)</code>","text":"<p>Returns a function that calculates the approximate Hessian-vector product for a given vector. If you want to compute the exact hessian, i.e., pulling all data into memory and compute a full gradient computation, use the function <code>hvp</code>.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch module representing the model whose loss function's Hessian is to be computed.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A callable that takes the model's output and target as input and returns the scalar loss.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> <code>data_loader</code> <p>A DataLoader instance that provides batches of data for calculating the Hessian-vector product. Each batch from the DataLoader is assumed to return a tuple where the first element is the model's input and the second element is the target output.</p> <p> TYPE: <code>DataLoader</code> </p> <code>use_hessian_avg</code> <p>If True, the returned function uses batch-wise Hessian computation via batch_loss_function and averages the results. If False, the function uses backpropagation on the full empirical_loss_function, which is more accurate than averaging the batch hessians, but probably has a way higher memory usage.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>reverse_only</code> <p>Whether to use only reverse-mode autodiff (True, default) or both forward- and reverse-mode autodiff (False).</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>track_gradients</code> <p>Whether to track gradients for the resulting tensor of the hessian vector products are (False, default).</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Callable[[Tensor], Tensor]</code> <p>A function that takes a single argument, a vector, and returns the product of the Hessian of the <code>loss</code> function with respect to the <code>model</code>'s parameters and the input vector.</p> Source code in <code>src/pydvl/influence/torch/functional.py</code> <pre><code>def get_hvp_function(\nmodel: torch.nn.Module,\nloss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\ndata_loader: DataLoader,\nuse_hessian_avg: bool = True,\nreverse_only: bool = True,\ntrack_gradients: bool = False,\n) -&gt; Callable[[torch.Tensor], torch.Tensor]:\n\"\"\"\n    Returns a function that calculates the approximate Hessian-vector product for a given vector. If you want to\n    compute the exact hessian, i.e., pulling all data into memory and compute a full gradient computation, use\n    the function `hvp`.\n    Args:\n        model: A PyTorch module representing the model whose loss function's Hessian is to be computed.\n        loss: A callable that takes the model's output and target as input and returns the scalar loss.\n        data_loader: A DataLoader instance that provides batches of data for calculating the Hessian-vector product.\n            Each batch from the DataLoader is assumed to return a tuple where the first element\n            is the model's input and the second element is the target output.\n        use_hessian_avg: If True, the returned function uses batch-wise Hessian computation via\n            [batch_loss_function][pydvl.influence.torch.functional.batch_loss_function] and averages the results.\n            If False, the function uses backpropagation on the full\n            [empirical_loss_function][pydvl.influence.torch.functional.empirical_loss_function],\n            which is more accurate than averaging the batch hessians, but probably has a way higher memory usage.\n        reverse_only: Whether to use only reverse-mode autodiff (True, default) or\n            both forward- and reverse-mode autodiff (False).\n        track_gradients: Whether to track gradients for the resulting tensor of the hessian vector\n            products are (False, default).\n    Returns:\n        A function that takes a single argument, a vector, and returns the product of the Hessian of the `loss`\n            function with respect to the `model`'s parameters and the input vector.\n    \"\"\"\nparams = {\nk: p if track_gradients else p.detach() for k, p in model.named_parameters()\n}\ndef hvp_function(vec: torch.Tensor) -&gt; torch.Tensor:\nv = align_structure(params, vec)\nempirical_loss = empirical_loss_function(model, loss, data_loader)\nreturn flatten_tensors_to_vector(\nhvp(empirical_loss, params, v, reverse_only=reverse_only).values()\n)\ndef avg_hvp_function(vec: torch.Tensor) -&gt; torch.Tensor:\nv = align_structure(params, vec)\nbatch_hessians_vector_products: Iterable[torch.Tensor] = map(\nlambda x: x(v), batch_hvp_gen(model, loss, data_loader, reverse_only)\n)\nnum_batches = len(data_loader)\navg_hessian = to_model_device(torch.zeros_like(vec), model)\nfor batch_hvp in batch_hessians_vector_products:\navg_hessian += batch_hvp\nreturn avg_hessian / float(num_batches)\nreturn avg_hvp_function if use_hessian_avg else hvp_function\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/","title":"Torch differentiable","text":"<p>Contains methods for differentiating  a pyTorch model. Most of the methods focus on ways to calculate matrix vector products. Moreover, it contains several methods to invert the Hessian vector product. These are used to calculate the influence of a training point on the model.</p>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable--references","title":"References","text":"<ol> <li> <p>Koh, P.W., Liang, P., 2017. Understanding Black-box Predictions via Influence Functions. In: Proceedings of the 34th International Conference on Machine Learning, pp. 1885\u20131894. PMLR.\u00a0\u21a9</p> </li> <li> <p>Agarwal, N., Bullins, B., Hazan, E., 2017. Second-Order Stochastic Optimization for Machine Learning in Linear Time. In: Journal of Machine Learning Research, Vol. 18, pp. 1\u201340. JMLR.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTwiceDifferentiable","title":"<code>TorchTwiceDifferentiable(model, loss)</code>","text":"<p>             Bases: <code>TwiceDifferentiable[Tensor]</code></p> <p>Wraps a torch.nn.Module and a loss function and provides methods to compute gradients and second derivative of the loss wrt. the model parameters</p> PARAMETER  DESCRIPTION <code>model</code> <p>A (differentiable) function.</p> <p> TYPE: <code>Module</code> </p> <code>loss</code> <p>A differentiable scalar loss \\( L(\\hat{y}, y) \\), mapping a prediction and a target to a real value.</p> <p> TYPE: <code>Callable[[Tensor, Tensor], Tensor]</code> </p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>def __init__(\nself,\nmodel: nn.Module,\nloss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],\n):\nif model.training:\nlogger.warning(\n\"Passed model not in evaluation mode. This can create several issues in influence \"\n\"computation, e.g. due to batch normalization. Please call model.eval() before \"\n\"computing influences.\"\n)\nself.loss = loss\nself.model = model\nfirst_param = next(model.parameters())\nself.device = first_param.device\nself.dtype = first_param.dtype\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTwiceDifferentiable.parameters","title":"<code>parameters: List[torch.Tensor]</code>  <code>property</code>","text":"RETURNS DESCRIPTION <code>List[Tensor]</code> <p>All model parameters that require differentiating.</p>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTwiceDifferentiable.num_params","title":"<code>num_params: int</code>  <code>property</code>","text":"<p>Get the number of parameters of model f.</p> RETURNS DESCRIPTION <code>int</code> <p>Number of parameters.</p> <p> TYPE: <code>int</code> </p>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTwiceDifferentiable.grad","title":"<code>grad(x, y, create_graph=False)</code>","text":"<p>Calculates gradient of model parameters with respect to the model parameters.</p> PARAMETER  DESCRIPTION <code>x</code> <p>A matrix [NxD] representing the features \\( x_i \\).</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>A matrix [NxK] representing the target values \\( y_i \\).</p> <p> TYPE: <code>Tensor</code> </p> <code>create_graph</code> <p>If True, the resulting gradient tensor can be used for further differentiation.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>An array [P] with the gradients of the model.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>def grad(\nself, x: torch.Tensor, y: torch.Tensor, create_graph: bool = False\n) -&gt; torch.Tensor:\nr\"\"\"\n    Calculates gradient of model parameters with respect to the model parameters.\n    Args:\n        x: A matrix [NxD] representing the features \\( x_i \\).\n        y: A matrix [NxK] representing the target values \\( y_i \\).\n        create_graph (bool): If True, the resulting gradient tensor can be used for further differentiation.\n    Returns:\n        An array [P] with the gradients of the model.\n    \"\"\"\nx = x.to(self.device)\ny = y.to(self.device)\nif create_graph and not x.requires_grad:\nx = x.requires_grad_(True)\nloss_value = self.loss(torch.squeeze(self.model(x)), torch.squeeze(y))\ngrad_f = torch.autograd.grad(\nloss_value, self.parameters, create_graph=create_graph\n)\nreturn flatten_tensors_to_vector(grad_f)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTwiceDifferentiable.hessian","title":"<code>hessian(x, y)</code>","text":"<p>Calculates the explicit hessian of model parameters given data \\(x\\) and \\(y\\).</p> PARAMETER  DESCRIPTION <code>x</code> <p>A matrix [NxD] representing the features \\(x_i\\).</p> <p> TYPE: <code>Tensor</code> </p> <code>y</code> <p>A matrix [NxK] representing the target values \\(y_i\\).</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A tensor representing the hessian of the loss with respect to the model parameters.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>def hessian(self, x: torch.Tensor, y: torch.Tensor) -&gt; torch.Tensor:\nr\"\"\"\n    Calculates the explicit hessian of model parameters given data \\(x\\) and \\(y\\).\n    Args:\n        x: A matrix [NxD] representing the features \\(x_i\\).\n        y: A matrix [NxK] representing the target values \\(y_i\\).\n    Returns:\n        A tensor representing the hessian of the loss with respect to the model parameters.\n    \"\"\"\ndef model_func(param):\noutputs = torch.func.functional_call(\nself.model,\nalign_structure(\n{k: p for k, p in self.model.named_parameters() if p.requires_grad},\nparam,\n),\n(x.to(self.device),),\nstrict=True,\n)\nreturn self.loss(outputs, y.to(self.device))\nparams = flatten_tensors_to_vector(\np.detach() for p in self.model.parameters() if p.requires_grad\n)\nreturn torch.func.hessian(model_func)(params)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTwiceDifferentiable.mvp","title":"<code>mvp(grad_xy, v, backprop_on, *, progress=False)</code>  <code>staticmethod</code>","text":"<p>Calculates the second-order derivative of the model along directions v. This second-order derivative can be selected through the <code>backprop_on</code> argument.</p> PARAMETER  DESCRIPTION <code>grad_xy</code> <p>An array [P] holding the gradients of the model parameters with respect to input \\(x\\) and labels \\(y\\), where P is the number of parameters of the model. It is typically obtained through <code>self.grad</code>.</p> <p> TYPE: <code>Tensor</code> </p> <code>v</code> <p>An array ([DxP] or even one-dimensional [D]) which multiplies the matrix, where D is the number of directions.</p> <p> TYPE: <code>Tensor</code> </p> <code>progress</code> <p>If True, progress will be printed.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>backprop_on</code> <p>Tensor used in the second backpropagation (the first one is defined via grad_xy).</p> <p> TYPE: <code>Tensor</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A matrix representing the implicit matrix-vector product of the model along the given directions. The output shape is [DxM], with M being the number of elements of <code>backprop_on</code>.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@staticmethod\ndef mvp(\ngrad_xy: torch.Tensor,\nv: torch.Tensor,\nbackprop_on: torch.Tensor,\n*,\nprogress: bool = False,\n) -&gt; torch.Tensor:\nr\"\"\"\n    Calculates the second-order derivative of the model along directions v.\n    This second-order derivative can be selected through the `backprop_on` argument.\n    Args:\n        grad_xy: An array [P] holding the gradients of the model parameters with respect to input\n            \\(x\\) and labels \\(y\\), where P is the number of parameters of the model.\n            It is typically obtained through `self.grad`.\n        v: An array ([DxP] or even one-dimensional [D]) which multiplies the matrix,\n            where D is the number of directions.\n        progress: If True, progress will be printed.\n        backprop_on: Tensor used in the second backpropagation\n            (the first one is defined via grad_xy).\n    Returns:\n        A matrix representing the implicit matrix-vector product of the model along the given directions.\n            The output shape is [DxM], with M being the number of elements of `backprop_on`.\n    \"\"\"\ndevice = grad_xy.device\nv = as_tensor(v, warn=False).to(device)\nif v.ndim == 1:\nv = v.unsqueeze(0)\nz = (grad_xy * Variable(v)).sum(dim=1)\nmvp = []\nfor i in maybe_progress(range(len(z)), progress, desc=\"MVP\"):\nmvp.append(\nflatten_tensors_to_vector(\nautograd.grad(z[i], backprop_on, retain_graph=True)\n)\n)\nreturn torch.stack([grad.contiguous().view(-1) for grad in mvp]).detach()\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.LowRankProductRepresentation","title":"<code>LowRankProductRepresentation</code>  <code>dataclass</code>","text":"<p>Representation of a low rank product of the form \\(H = V D V^T\\), where D is a diagonal matrix and V is orthogonal.</p> PARAMETER  DESCRIPTION <code>eigen_vals</code> <p>Diagonal of D.</p> <p> TYPE: <code>Tensor</code> </p> <code>projections</code> <p>The matrix V.</p> <p> TYPE: <code>Tensor</code> </p>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.LowRankProductRepresentation.to","title":"<code>to(device)</code>","text":"<p>Move the representing tensors to a device</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>def to(self, device: torch.device):\n\"\"\"\n    Move the representing tensors to a device\n    \"\"\"\nreturn LowRankProductRepresentation(\nself.eigen_vals.to(device), self.projections.to(device)\n)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTensorUtilities","title":"<code>TorchTensorUtilities</code>","text":"<p>             Bases: <code>TensorUtilities[Tensor, TorchTwiceDifferentiable]</code></p>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTensorUtilities.einsum","title":"<code>einsum(equation, *operands)</code>  <code>staticmethod</code>","text":"<p>Sums the product of the elements of the input :attr:<code>operands</code> along dimensions specified using a notation based on the Einstein summation convention.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@staticmethod\ndef einsum(equation: str, *operands) -&gt; torch.Tensor:\n\"\"\"Sums the product of the elements of the input :attr:`operands` along dimensions specified using a notation\n    based on the Einstein summation convention.\n    \"\"\"\nreturn torch.einsum(equation, *operands)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTensorUtilities.cat","title":"<code>cat(a, **kwargs)</code>  <code>staticmethod</code>","text":"<p>Concatenates a sequence of tensors into a single torch tensor</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@staticmethod\ndef cat(a: Sequence[torch.Tensor], **kwargs) -&gt; torch.Tensor:\n\"\"\"Concatenates a sequence of tensors into a single torch tensor\"\"\"\nreturn torch.cat(a, **kwargs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTensorUtilities.stack","title":"<code>stack(a, **kwargs)</code>  <code>staticmethod</code>","text":"<p>Stacks a sequence of tensors into a single torch tensor</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@staticmethod\ndef stack(a: Sequence[torch.Tensor], **kwargs) -&gt; torch.Tensor:\n\"\"\"Stacks a sequence of tensors into a single torch tensor\"\"\"\nreturn torch.stack(a, **kwargs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.TorchTensorUtilities.unsqueeze","title":"<code>unsqueeze(x, dim)</code>  <code>staticmethod</code>","text":"<p>Add a singleton dimension at a specified position in a tensor.</p> PARAMETER  DESCRIPTION <code>x</code> <p>A PyTorch tensor.</p> <p> TYPE: <code>Tensor</code> </p> <code>dim</code> <p>The position at which to add the singleton dimension. Zero-based indexing.</p> <p> TYPE: <code>int</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A new tensor with an additional singleton dimension.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@staticmethod\ndef unsqueeze(x: torch.Tensor, dim: int) -&gt; torch.Tensor:\n\"\"\"\n    Add a singleton dimension at a specified position in a tensor.\n    Args:\n        x: A PyTorch tensor.\n        dim: The position at which to add the singleton dimension. Zero-based indexing.\n    Returns:\n        A new tensor with an additional singleton dimension.\n    \"\"\"\nreturn x.unsqueeze(dim)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.lanzcos_low_rank_hessian_approx","title":"<code>lanzcos_low_rank_hessian_approx(hessian_vp, matrix_shape, hessian_perturbation=0.0, rank_estimate=10, krylov_dimension=None, tol=1e-06, max_iter=None, device=None, eigen_computation_on_gpu=False, torch_dtype=None)</code>","text":"<p>Calculates a low-rank approximation of the Hessian matrix of a scalar-valued function using the implicitly restarted Lanczos algorithm, i.e.:</p> \\[ H_{\\text{approx}} = V D V^T\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) <code>rank_estimate</code> eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors.</p> PARAMETER  DESCRIPTION <code>hessian_vp</code> <p>A function that takes a vector and returns the product of the Hessian of the loss function.</p> <p> TYPE: <code>Callable[[Tensor], Tensor]</code> </p> <code>matrix_shape</code> <p>The shape of the matrix, represented by the hessian vector product.</p> <p> TYPE: <code>Tuple[int, int]</code> </p> <code>hessian_perturbation</code> <p>Regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>rank_estimate</code> <p>The number of eigenvalues and corresponding eigenvectors to compute. Represents the desired rank of the Hessian approximation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>krylov_dimension</code> <p>The number of Krylov vectors to use for the Lanczos method. If not provided, it defaults to \\( \\min(\\text{model.num_parameters}, \\max(2 \\times \\text{rank_estimate} + 1, 20)) \\).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>tol</code> <p>The stopping criteria for the Lanczos algorithm, which stops when the difference in the approximated eigenvalue is less than <code>tol</code>. Defaults to 1e-6.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-06</code> </p> <code>max_iter</code> <p>The maximum number of iterations for the Lanczos method. If not provided, it defaults to \\( 10 \\cdot \\text{model.num_parameters}\\).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>device</code> <p>The device to use for executing the hessian vector product.</p> <p> TYPE: <code>Optional[device]</code> DEFAULT: <code>None</code> </p> <code>eigen_computation_on_gpu</code> <p>If True, tries to execute the eigen pair approximation on the provided device via cupy implementation. Ensure that either your model is small enough, or you use a small rank_estimate to fit your device's memory. If False, the eigen pair approximation is executed on the CPU with scipy's wrapper to ARPACK.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>torch_dtype</code> <p>If not provided, the current torch default dtype is used for conversion to torch.</p> <p> TYPE: <code>dtype</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>LowRankProductRepresentation</code> <p>A LowRankProductRepresentation instance that contains the top (up until rank_estimate) eigenvalues and corresponding eigenvectors of the Hessian.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>def lanzcos_low_rank_hessian_approx(\nhessian_vp: Callable[[torch.Tensor], torch.Tensor],\nmatrix_shape: Tuple[int, int],\nhessian_perturbation: float = 0.0,\nrank_estimate: int = 10,\nkrylov_dimension: Optional[int] = None,\ntol: float = 1e-6,\nmax_iter: Optional[int] = None,\ndevice: Optional[torch.device] = None,\neigen_computation_on_gpu: bool = False,\ntorch_dtype: torch.dtype = None,\n) -&gt; LowRankProductRepresentation:\nr\"\"\"\n    Calculates a low-rank approximation of the Hessian matrix of a scalar-valued\n    function using the implicitly restarted Lanczos algorithm, i.e.:\n    \\[ H_{\\text{approx}} = V D V^T\\]\n    where \\(D\\) is a diagonal matrix with the top (in absolute value) `rank_estimate` eigenvalues of the Hessian\n    and \\(V\\) contains the corresponding eigenvectors.\n    Args:\n        hessian_vp: A function that takes a vector and returns the product of\n            the Hessian of the loss function.\n        matrix_shape: The shape of the matrix, represented by the hessian vector\n            product.\n        hessian_perturbation: Regularization parameter added to the\n            Hessian-vector product for numerical stability.\n        rank_estimate: The number of eigenvalues and corresponding eigenvectors\n            to compute. Represents the desired rank of the Hessian approximation.\n        krylov_dimension: The number of Krylov vectors to use for the Lanczos\n            method. If not provided, it defaults to\n            \\( \\min(\\text{model.num_parameters}, \\max(2 \\times \\text{rank_estimate} + 1, 20)) \\).\n        tol: The stopping criteria for the Lanczos algorithm, which stops when\n            the difference in the approximated eigenvalue is less than `tol`.\n            Defaults to 1e-6.\n        max_iter: The maximum number of iterations for the Lanczos method. If\n            not provided, it defaults to \\( 10 \\cdot \\text{model.num_parameters}\\).\n        device: The device to use for executing the hessian vector product.\n        eigen_computation_on_gpu: If True, tries to execute the eigen pair\n            approximation on the provided device via [cupy](https://cupy.dev/)\n            implementation. Ensure that either your model is small enough, or you\n            use a small rank_estimate to fit your device's memory. If False, the\n            eigen pair approximation is executed on the CPU with scipy's wrapper to\n            ARPACK.\n        torch_dtype: If not provided, the current torch default dtype is used for\n            conversion to torch.\n    Returns:\n        A [LowRankProductRepresentation][pydvl.influence.torch.torch_differentiable.LowRankProductRepresentation]\n            instance that contains the top (up until rank_estimate) eigenvalues\n            and corresponding eigenvectors of the Hessian.\n    \"\"\"\ntorch_dtype = torch.get_default_dtype() if torch_dtype is None else torch_dtype\nif eigen_computation_on_gpu:\ntry:\nimport cupy as cp\nfrom cupyx.scipy.sparse.linalg import LinearOperator, eigsh\nfrom torch.utils.dlpack import from_dlpack, to_dlpack\nexcept ImportError as e:\nraise ImportError(\nf\"Try to install missing dependencies or set eigen_computation_on_gpu to False: {e}\"\n)\nif device is None:\nraise ValueError(\n\"Without setting an explicit device, cupy is not supported\"\n)\ndef to_torch_conversion_function(x):\nreturn from_dlpack(x.toDlpack()).to(torch_dtype)\ndef mv(x):\nx = to_torch_conversion_function(x)\ny = hessian_vp(x) + hessian_perturbation * x\nreturn cp.from_dlpack(to_dlpack(y))\nelse:\nfrom scipy.sparse.linalg import LinearOperator, eigsh\ndef mv(x):\nx_torch = torch.as_tensor(x, device=device, dtype=torch_dtype)\ny: NDArray = (\n(hessian_vp(x_torch) + hessian_perturbation * x_torch)\n.detach()\n.cpu()\n.numpy()\n)\nreturn y\nto_torch_conversion_function = partial(torch.as_tensor, dtype=torch_dtype)\ntry:\neigen_vals, eigen_vecs = eigsh(\nLinearOperator(matrix_shape, matvec=mv),\nk=rank_estimate,\nmaxiter=max_iter,\ntol=tol,\nncv=krylov_dimension,\nreturn_eigenvectors=True,\n)\nexcept ArpackNoConvergence as e:\nlogger.warning(\nf\"ARPACK did not converge for parameters {max_iter=}, {tol=}, {krylov_dimension=}, \"\nf\"{rank_estimate=}. \\n Returning the best approximation found so far. Use those with care or \"\nf\"modify parameters.\\n Original error: {e}\"\n)\neigen_vals, eigen_vecs = e.eigenvalues, e.eigenvectors\neigen_vals = to_torch_conversion_function(eigen_vals)\neigen_vecs = to_torch_conversion_function(eigen_vecs)\nreturn LowRankProductRepresentation(eigen_vals, eigen_vecs)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.model_hessian_low_rank","title":"<code>model_hessian_low_rank(model, training_data, hessian_perturbation=0.0, rank_estimate=10, krylov_dimension=None, tol=1e-06, max_iter=None, eigen_computation_on_gpu=False)</code>","text":"<p>Calculates a low-rank approximation of the Hessian matrix of the model's loss function using the implicitly restarted Lanczos algorithm, i.e.</p> \\[ H_{\\text{approx}} = V D V^T\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) <code>rank_estimate</code> eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model instance that is twice differentiable, wrapped into <code>TorchTwiceDifferential</code>. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>TorchTwiceDifferentiable</code> </p> <code>training_data</code> <p>A DataLoader instance that provides the model's training data. Used in calculating the Hessian-vector products.</p> <p> TYPE: <code>DataLoader</code> </p> <code>hessian_perturbation</code> <p>Optional regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>rank_estimate</code> <p>The number of eigenvalues and corresponding eigenvectors to compute. Represents the desired rank of the Hessian approximation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>krylov_dimension</code> <p>The number of Krylov vectors to use for the Lanczos method. If not provided, it defaults to min(model.num_parameters, max(2*rank_estimate + 1, 20)).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>tol</code> <p>The stopping criteria for the Lanczos algorithm, which stops when the difference in the approximated eigenvalue is less than <code>tol</code>. Defaults to 1e-6.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-06</code> </p> <code>max_iter</code> <p>The maximum number of iterations for the Lanczos method. If not provided, it defaults to 10*model.num_parameters.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>eigen_computation_on_gpu</code> <p>If True, tries to execute the eigen pair approximation on the provided device via cupy implementation. Make sure, that either your model is small enough or you use a small rank_estimate to fit your device's memory. If False, the eigen pair approximation is executed on the CPU by scipy wrapper to ARPACK.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>LowRankProductRepresentation</code> <p>A LowRankProductRepresentation instance that contains the top (up until rank_estimate) eigenvalues and corresponding eigenvectors of the Hessian.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>def model_hessian_low_rank(\nmodel: TorchTwiceDifferentiable,\ntraining_data: DataLoader,\nhessian_perturbation: float = 0.0,\nrank_estimate: int = 10,\nkrylov_dimension: Optional[int] = None,\ntol: float = 1e-6,\nmax_iter: Optional[int] = None,\neigen_computation_on_gpu: bool = False,\n) -&gt; LowRankProductRepresentation:\nr\"\"\"\n    Calculates a low-rank approximation of the Hessian matrix of the model's loss function using the implicitly\n    restarted Lanczos algorithm, i.e.\n    \\[ H_{\\text{approx}} = V D V^T\\]\n    where \\(D\\) is a diagonal matrix with the top (in absolute value) `rank_estimate` eigenvalues of the Hessian\n    and \\(V\\) contains the corresponding eigenvectors.\n    Args:\n        model: A PyTorch model instance that is twice differentiable, wrapped into `TorchTwiceDifferential`.\n            The Hessian will be calculated with respect to this model's parameters.\n        training_data: A DataLoader instance that provides the model's training data.\n            Used in calculating the Hessian-vector products.\n        hessian_perturbation: Optional regularization parameter added to the Hessian-vector product\n            for numerical stability.\n        rank_estimate: The number of eigenvalues and corresponding eigenvectors to compute.\n            Represents the desired rank of the Hessian approximation.\n        krylov_dimension: The number of Krylov vectors to use for the Lanczos method.\n            If not provided, it defaults to min(model.num_parameters, max(2*rank_estimate + 1, 20)).\n        tol: The stopping criteria for the Lanczos algorithm, which stops when the difference\n            in the approximated eigenvalue is less than `tol`. Defaults to 1e-6.\n        max_iter: The maximum number of iterations for the Lanczos method. If not provided, it defaults to\n            10*model.num_parameters.\n        eigen_computation_on_gpu: If True, tries to execute the eigen pair approximation on the provided\n            device via cupy implementation.\n            Make sure, that either your model is small enough or you use a\n            small rank_estimate to fit your device's memory.\n            If False, the eigen pair approximation is executed on the CPU by scipy wrapper to\n            ARPACK.\n    Returns:\n        A [LowRankProductRepresentation][pydvl.influence.torch.torch_differentiable.LowRankProductRepresentation]\n            instance that contains the top (up until rank_estimate) eigenvalues\n            and corresponding eigenvectors of the Hessian.\n    \"\"\"\nraw_hvp = get_hvp_function(\nmodel.model, model.loss, training_data, use_hessian_avg=True\n)\nreturn lanzcos_low_rank_hessian_approx(\nhessian_vp=raw_hvp,\nmatrix_shape=(model.num_params, model.num_params),\nhessian_perturbation=hessian_perturbation,\nrank_estimate=rank_estimate,\nkrylov_dimension=krylov_dimension,\ntol=tol,\nmax_iter=max_iter,\ndevice=model.device if hasattr(model, \"device\") else None,\neigen_computation_on_gpu=eigen_computation_on_gpu,\n)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.solve_linear","title":"<code>solve_linear(model, training_data, b, hessian_perturbation=0.0)</code>","text":"<p>Given a model and training data, it finds x such that \\(Hx = b\\), with \\(H\\) being the model hessian.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A model wrapped in the TwiceDifferentiable interface.</p> <p> TYPE: <code>TorchTwiceDifferentiable</code> </p> <code>training_data</code> <p>A DataLoader containing the training data.</p> <p> TYPE: <code>DataLoader</code> </p> <code>b</code> <p>A vector or matrix, the right hand side of the equation \\(Hx = b\\).</p> <p> TYPE: <code>Tensor</code> </p> <code>hessian_perturbation</code> <p>Regularization of the hessian.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> RETURNS DESCRIPTION <code>InverseHvpResult</code> <p>Instance of InverseHvpResult, having an array that solves the inverse problem, i.e. it returns \\(x\\) such that \\(Hx = b\\), and a dictionary containing information about the solution.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@InversionRegistry.register(TorchTwiceDifferentiable, InversionMethod.Direct)\ndef solve_linear(\nmodel: TorchTwiceDifferentiable,\ntraining_data: DataLoader,\nb: torch.Tensor,\nhessian_perturbation: float = 0.0,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Given a model and training data, it finds x such that \\(Hx = b\\), with \\(H\\) being the model hessian.\n    Args:\n        model: A model wrapped in the TwiceDifferentiable interface.\n        training_data: A DataLoader containing the training data.\n        b: A vector or matrix, the right hand side of the equation \\(Hx = b\\).\n        hessian_perturbation: Regularization of the hessian.\n    Returns:\n        Instance of [InverseHvpResult][pydvl.influence.twice_differentiable.InverseHvpResult],\n            having an array that solves the inverse problem, i.e. it returns \\(x\\) such that \\(Hx = b\\),\n            and a dictionary containing information about the solution.\n    \"\"\"\nall_x, all_y = [], []\nfor x, y in training_data:\nall_x.append(x)\nall_y.append(y)\nhessian = model.hessian(torch.cat(all_x), torch.cat(all_y))\nmatrix = hessian + hessian_perturbation * torch.eye(\nmodel.num_params, device=model.device\n)\ninfo = {\"hessian\": hessian}\nreturn InverseHvpResult(x=torch.linalg.solve(matrix, b.T).T, info=info)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.solve_batch_cg","title":"<code>solve_batch_cg(model, training_data, b, hessian_perturbation=0.0, *, x0=None, rtol=1e-07, atol=1e-07, maxiter=None, progress=False)</code>","text":"<p>Given a model and training data, it uses conjugate gradient to calculate the inverse of the Hessian Vector Product. More precisely, it finds x such that \\(Hx = b\\), with \\(H\\) being the model hessian. For more info, see Wikipedia.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A model wrapped in the TwiceDifferentiable interface.</p> <p> TYPE: <code>TorchTwiceDifferentiable</code> </p> <code>training_data</code> <p>A DataLoader containing the training data.</p> <p> TYPE: <code>DataLoader</code> </p> <code>b</code> <p>A vector or matrix, the right hand side of the equation \\(Hx = b\\).</p> <p> TYPE: <code>Tensor</code> </p> <code>hessian_perturbation</code> <p>Regularization of the hessian.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>x0</code> <p>Initial guess for hvp. If None, defaults to b.</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>rtol</code> <p>Maximum relative tolerance of result.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-07</code> </p> <code>atol</code> <p>Absolute tolerance of result.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-07</code> </p> <code>maxiter</code> <p>Maximum number of iterations. If None, defaults to 10*len(b).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, display progress bars.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>InverseHvpResult</code> <p>Instance of InverseHvpResult, having a matrix of shape [NxP] with each line being a solution of \\(Ax=b\\), and a dictionary containing information about the convergence of CG, one entry for each line of the matrix.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@InversionRegistry.register(TorchTwiceDifferentiable, InversionMethod.Cg)\ndef solve_batch_cg(\nmodel: TorchTwiceDifferentiable,\ntraining_data: DataLoader,\nb: torch.Tensor,\nhessian_perturbation: float = 0.0,\n*,\nx0: Optional[torch.Tensor] = None,\nrtol: float = 1e-7,\natol: float = 1e-7,\nmaxiter: Optional[int] = None,\nprogress: bool = False,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Given a model and training data, it uses conjugate gradient to calculate the\n    inverse of the Hessian Vector Product. More precisely, it finds x such that \\(Hx =\n    b\\), with \\(H\\) being the model hessian. For more info, see\n    [Wikipedia](https://en.wikipedia.org/wiki/Conjugate_gradient_method).\n    Args:\n        model: A model wrapped in the TwiceDifferentiable interface.\n        training_data: A DataLoader containing the training data.\n        b: A vector or matrix, the right hand side of the equation \\(Hx = b\\).\n        hessian_perturbation: Regularization of the hessian.\n        x0: Initial guess for hvp. If None, defaults to b.\n        rtol: Maximum relative tolerance of result.\n        atol: Absolute tolerance of result.\n        maxiter: Maximum number of iterations. If None, defaults to 10*len(b).\n        progress: If True, display progress bars.\n    Returns:\n        Instance of [InverseHvpResult][pydvl.influence.twice_differentiable.InverseHvpResult],\n            having a matrix of shape [NxP] with each line being a solution of \\(Ax=b\\),\n            and a dictionary containing information about the convergence of CG,\n            one entry for each line of the matrix.\n    \"\"\"\ntotal_grad_xy = 0\ntotal_points = 0\nfor x, y in maybe_progress(training_data, progress, desc=\"Batch Train Gradients\"):\ngrad_xy = model.grad(x, y, create_graph=True)\ntotal_grad_xy += grad_xy * len(x)\ntotal_points += len(x)\nbackprop_on = model.parameters\nreg_hvp = lambda v: model.mvp(\ntotal_grad_xy / total_points, v, backprop_on\n) + hessian_perturbation * v.type(torch.float64)\nbatch_cg = torch.zeros_like(b)\ninfo = {}\nfor idx, bi in enumerate(maybe_progress(b, progress, desc=\"Conjugate gradient\")):\nbatch_result, batch_info = solve_cg(\nreg_hvp, bi, x0=x0, rtol=rtol, atol=atol, maxiter=maxiter\n)\nbatch_cg[idx] = batch_result\ninfo[f\"batch_{idx}\"] = batch_info\nreturn InverseHvpResult(x=batch_cg, info=info)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.solve_cg","title":"<code>solve_cg(hvp, b, *, x0=None, rtol=1e-07, atol=1e-07, maxiter=None)</code>","text":"<p>Conjugate gradient solver for the Hessian vector product.</p> PARAMETER  DESCRIPTION <code>hvp</code> <p>A callable Hvp, operating with tensors of size N.</p> <p> TYPE: <code>Callable[[Tensor], Tensor]</code> </p> <code>b</code> <p>A vector or matrix, the right hand side of the equation \\(Hx = b\\).</p> <p> TYPE: <code>Tensor</code> </p> <code>x0</code> <p>Initial guess for hvp.</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>rtol</code> <p>Maximum relative tolerance of result.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-07</code> </p> <code>atol</code> <p>Absolute tolerance of result.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-07</code> </p> <code>maxiter</code> <p>Maximum number of iterations. If None, defaults to 10*len(b).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>InverseHvpResult</code> <p>Instance of InverseHvpResult, with a vector x, solution of \\(Ax=b\\), and a dictionary containing information about the convergence of CG.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>def solve_cg(\nhvp: Callable[[torch.Tensor], torch.Tensor],\nb: torch.Tensor,\n*,\nx0: Optional[torch.Tensor] = None,\nrtol: float = 1e-7,\natol: float = 1e-7,\nmaxiter: Optional[int] = None,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Conjugate gradient solver for the Hessian vector product.\n    Args:\n        hvp: A callable Hvp, operating with tensors of size N.\n        b: A vector or matrix, the right hand side of the equation \\(Hx = b\\).\n        x0: Initial guess for hvp.\n        rtol: Maximum relative tolerance of result.\n        atol: Absolute tolerance of result.\n        maxiter: Maximum number of iterations. If None, defaults to 10*len(b).\n    Returns:\n        Instance of [InverseHvpResult][pydvl.influence.twice_differentiable.InverseHvpResult],\n            with a vector x, solution of \\(Ax=b\\), and a dictionary containing\n            information about the convergence of CG.\n    \"\"\"\nif x0 is None:\nx0 = torch.clone(b)\nif maxiter is None:\nmaxiter = len(b) * 10\ny_norm = torch.sum(torch.matmul(b, b)).item()\nstopping_val = max([rtol**2 * y_norm, atol**2])\nx = x0\np = r = (b - hvp(x)).squeeze().type(torch.float64)\ngamma = torch.sum(torch.matmul(r, r)).item()\noptimal = False\nfor k in range(maxiter):\nif gamma &lt; stopping_val:\noptimal = True\nbreak\nAp = hvp(p).squeeze()\nalpha = gamma / torch.sum(torch.matmul(p, Ap)).item()\nx += alpha * p\nr -= alpha * Ap\ngamma_ = torch.sum(torch.matmul(r, r)).item()\nbeta = gamma_ / gamma\ngamma = gamma_\np = r + beta * p\ninfo = {\"niter\": k, \"optimal\": optimal, \"gamma\": gamma}\nreturn InverseHvpResult(x=x, info=info)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.solve_lissa","title":"<code>solve_lissa(model, training_data, b, hessian_perturbation=0.0, *, maxiter=1000, dampen=0.0, scale=10.0, h0=None, rtol=0.0001, progress=False)</code>","text":"<p>Uses LISSA, Linear time Stochastic Second-Order Algorithm, to iteratively approximate the inverse Hessian. More precisely, it finds x s.t. \\(Hx = b\\), with \\(H\\) being the model's second derivative wrt. the parameters. This is done with the update</p> \\[H^{-1}_{j+1} b = b + (I - d) \\ H - \\frac{H^{-1}_j b}{s},\\] <p>where \\(I\\) is the identity matrix, \\(d\\) is a dampening term and \\(s\\) a scaling factor that are applied to help convergence. For details, see (Koh and Liang, 2017)<sup>1</sup> and the original paper (Agarwal et. al.)<sup>2</sup>.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A model wrapped in the TwiceDifferentiable interface.</p> <p> TYPE: <code>TorchTwiceDifferentiable</code> </p> <code>training_data</code> <p>A DataLoader containing the training data.</p> <p> TYPE: <code>DataLoader</code> </p> <code>b</code> <p>A vector or matrix, the right hand side of the equation \\(Hx = b\\).</p> <p> TYPE: <code>Tensor</code> </p> <code>hessian_perturbation</code> <p>Regularization of the hessian.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>maxiter</code> <p>Maximum number of iterations.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1000</code> </p> <code>dampen</code> <p>Dampening factor, defaults to 0 for no dampening.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>scale</code> <p>Scaling factor, defaults to 10.</p> <p> TYPE: <code>float</code> DEFAULT: <code>10.0</code> </p> <code>h0</code> <p>Initial guess for hvp.</p> <p> TYPE: <code>Optional[Tensor]</code> DEFAULT: <code>None</code> </p> <code>rtol</code> <p>tolerance to use for early stopping</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0001</code> </p> <code>progress</code> <p>If True, display progress bars.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>InverseHvpResult</code> <p>Instance of InverseHvpResult, with a matrix of shape [NxP] with each line being a solution of \\(Ax=b\\), and a dictionary containing information about the accuracy of the solution.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@InversionRegistry.register(TorchTwiceDifferentiable, InversionMethod.Lissa)\ndef solve_lissa(\nmodel: TorchTwiceDifferentiable,\ntraining_data: DataLoader,\nb: torch.Tensor,\nhessian_perturbation: float = 0.0,\n*,\nmaxiter: int = 1000,\ndampen: float = 0.0,\nscale: float = 10.0,\nh0: Optional[torch.Tensor] = None,\nrtol: float = 1e-4,\nprogress: bool = False,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Uses LISSA, Linear time Stochastic Second-Order Algorithm, to iteratively\n    approximate the inverse Hessian. More precisely, it finds x s.t. \\(Hx = b\\),\n    with \\(H\\) being the model's second derivative wrt. the parameters.\n    This is done with the update\n    \\[H^{-1}_{j+1} b = b + (I - d) \\ H - \\frac{H^{-1}_j b}{s},\\]\n    where \\(I\\) is the identity matrix, \\(d\\) is a dampening term and \\(s\\) a scaling\n    factor that are applied to help convergence. For details, see\n    (Koh and Liang, 2017)&lt;sup&gt;&lt;a href=\"#koh_liang_2017\"&gt;1&lt;/a&gt;&lt;/sup&gt; and the original paper\n    (Agarwal et. al.)&lt;sup&gt;&lt;a href=\"#agarwal_secondorder_2017\"&gt;2&lt;/a&gt;&lt;/sup&gt;.\n    Args:\n        model: A model wrapped in the TwiceDifferentiable interface.\n        training_data: A DataLoader containing the training data.\n        b: A vector or matrix, the right hand side of the equation \\(Hx = b\\).\n        hessian_perturbation: Regularization of the hessian.\n        maxiter: Maximum number of iterations.\n        dampen: Dampening factor, defaults to 0 for no dampening.\n        scale: Scaling factor, defaults to 10.\n        h0: Initial guess for hvp.\n        rtol: tolerance to use for early stopping\n        progress: If True, display progress bars.\n    Returns:\n        Instance of [InverseHvpResult][pydvl.influence.twice_differentiable.InverseHvpResult], with a matrix of shape [NxP] with each line being a solution of \\(Ax=b\\),\n            and a dictionary containing information about the accuracy of the solution.\n    \"\"\"\nif h0 is None:\nh_estimate = torch.clone(b)\nelse:\nh_estimate = h0\nshuffled_training_data = DataLoader(\ntraining_data.dataset, training_data.batch_size, shuffle=True\n)\ndef lissa_step(\nh: torch.Tensor, reg_hvp: Callable[[torch.Tensor], torch.Tensor]\n) -&gt; torch.Tensor:\n\"\"\"Given an estimate of the hessian inverse and the regularised hessian\n        vector product, it computes the next estimate.\n        Args:\n            h: An estimate of the hessian inverse.\n            reg_hvp: Regularised hessian vector product.\n        Returns:\n            The next estimate of the hessian inverse.\n        \"\"\"\nreturn b + (1 - dampen) * h - reg_hvp(h) / scale\nfor _ in maybe_progress(range(maxiter), progress, desc=\"Lissa\"):\nx, y = next(iter(shuffled_training_data))\ngrad_xy = model.grad(x, y, create_graph=True)\nreg_hvp = (\nlambda v: model.mvp(grad_xy, v, model.parameters) + hessian_perturbation * v\n)\nresidual = lissa_step(h_estimate, reg_hvp) - h_estimate\nh_estimate += residual\nif torch.isnan(h_estimate).any():\nraise RuntimeError(\"NaNs in h_estimate. Increase scale or dampening.\")\nmax_residual = torch.max(torch.abs(residual / h_estimate))\nif max_residual &lt; rtol:\nbreak\nmean_residual = torch.mean(torch.abs(residual / h_estimate))\nlogger.info(\nf\"Terminated Lissa with {max_residual*100:.2f} % max residual.\"\nf\" Mean residual: {mean_residual*100:.5f} %\"\n)\ninfo = {\n\"max_perc_residual\": max_residual * 100,\n\"mean_perc_residual\": mean_residual * 100,\n}\nreturn InverseHvpResult(x=h_estimate / scale, info=info)\n</code></pre>"},{"location":"api/pydvl/influence/torch/torch_differentiable/#pydvl.influence.torch.torch_differentiable.solve_arnoldi","title":"<code>solve_arnoldi(model, training_data, b, hessian_perturbation=0.0, *, rank_estimate=10, krylov_dimension=None, low_rank_representation=None, tol=1e-06, max_iter=None, eigen_computation_on_gpu=False)</code>","text":"<p>Solves the linear system Hx = b, where H is the Hessian of the model's loss function and b is the given right-hand side vector. It employs the implicitly restarted Arnoldi method for computing a partial eigen decomposition, which is used fo the inversion i.e.</p> \\[x = V D^{-1} V^T b\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) <code>rank_estimate</code> eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors.</p> PARAMETER  DESCRIPTION <code>model</code> <p>A PyTorch model instance that is twice differentiable, wrapped into TorchTwiceDifferential. The Hessian will be calculated with respect to this model's parameters.</p> <p> TYPE: <code>TorchTwiceDifferentiable</code> </p> <code>training_data</code> <p>A DataLoader instance that provides the model's training data. Used in calculating the Hessian-vector products.</p> <p> TYPE: <code>DataLoader</code> </p> <code>b</code> <p>The right-hand side vector in the system Hx = b.</p> <p> TYPE: <code>Tensor</code> </p> <code>hessian_perturbation</code> <p>Optional regularization parameter added to the Hessian-vector product for numerical stability.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>rank_estimate</code> <p>The number of eigenvalues and corresponding eigenvectors to compute. Represents the desired rank of the Hessian approximation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>krylov_dimension</code> <p>The number of Krylov vectors to use for the Lanczos method. Defaults to min(model's number of parameters, max(2 times rank_estimate + 1, 20)).</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>low_rank_representation</code> <p>An instance of LowRankProductRepresentation containing a previously computed low-rank representation of the Hessian. If provided, all other parameters are ignored; otherwise, a new low-rank representation is computed using provided parameters.</p> <p> TYPE: <code>Optional[LowRankProductRepresentation]</code> DEFAULT: <code>None</code> </p> <code>tol</code> <p>The stopping criteria for the Lanczos algorithm. Ignored if <code>low_rank_representation</code> is provided.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1e-06</code> </p> <code>max_iter</code> <p>The maximum number of iterations for the Lanczos method. Ignored if <code>low_rank_representation</code> is provided.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>eigen_computation_on_gpu</code> <p>If True, tries to execute the eigen pair approximation on the model's device via a cupy implementation. Ensure the model size or rank_estimate is appropriate for device memory. If False, the eigen pair approximation is executed on the CPU by the scipy wrapper to ARPACK.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>InverseHvpResult</code> <p>Instance of InverseHvpResult, having the solution vector x that satisfies the system \\(Ax = b\\), where \\(A\\) is a low-rank approximation of the Hessian \\(H\\) of the model's loss function, and an instance of LowRankProductRepresentation, which represents the approximation of H.</p> Source code in <code>src/pydvl/influence/torch/torch_differentiable.py</code> <pre><code>@InversionRegistry.register(TorchTwiceDifferentiable, InversionMethod.Arnoldi)\ndef solve_arnoldi(\nmodel: TorchTwiceDifferentiable,\ntraining_data: DataLoader,\nb: torch.Tensor,\nhessian_perturbation: float = 0.0,\n*,\nrank_estimate: int = 10,\nkrylov_dimension: Optional[int] = None,\nlow_rank_representation: Optional[LowRankProductRepresentation] = None,\ntol: float = 1e-6,\nmax_iter: Optional[int] = None,\neigen_computation_on_gpu: bool = False,\n) -&gt; InverseHvpResult:\nr\"\"\"\n    Solves the linear system Hx = b, where H is the Hessian of the model's loss function and b is the given\n    right-hand side vector.\n    It employs the [implicitly restarted Arnoldi method](https://en.wikipedia.org/wiki/Arnoldi_iteration) for\n    computing a partial eigen decomposition, which is used fo the inversion i.e.\n    \\[x = V D^{-1} V^T b\\]\n    where \\(D\\) is a diagonal matrix with the top (in absolute value) `rank_estimate` eigenvalues of the Hessian\n    and \\(V\\) contains the corresponding eigenvectors.\n    Args:\n        model: A PyTorch model instance that is twice differentiable, wrapped into\n            [TorchTwiceDifferential][pydvl.influence.torch.torch_differentiable.TorchTwiceDifferentiable].\n            The Hessian will be calculated with respect to this model's parameters.\n        training_data: A DataLoader instance that provides the model's training data.\n            Used in calculating the Hessian-vector products.\n        b: The right-hand side vector in the system Hx = b.\n        hessian_perturbation: Optional regularization parameter added to the Hessian-vector\n            product for numerical stability.\n        rank_estimate: The number of eigenvalues and corresponding eigenvectors to compute.\n            Represents the desired rank of the Hessian approximation.\n        krylov_dimension: The number of Krylov vectors to use for the Lanczos method.\n            Defaults to min(model's number of parameters, max(2 times rank_estimate + 1, 20)).\n        low_rank_representation: An instance of\n            [LowRankProductRepresentation][pydvl.influence.torch.torch_differentiable.LowRankProductRepresentation]\n            containing a previously computed low-rank representation of the Hessian. If provided, all other parameters\n            are ignored; otherwise, a new low-rank representation is computed\n            using provided parameters.\n        tol: The stopping criteria for the Lanczos algorithm.\n            Ignored if `low_rank_representation` is provided.\n        max_iter: The maximum number of iterations for the Lanczos method.\n            Ignored if `low_rank_representation` is provided.\n        eigen_computation_on_gpu: If True, tries to execute the eigen pair approximation on the model's device\n            via a cupy implementation. Ensure the model size or rank_estimate is appropriate for device memory.\n            If False, the eigen pair approximation is executed on the CPU by the scipy wrapper to ARPACK.\n    Returns:\n        Instance of [InverseHvpResult][pydvl.influence.torch.torch_differentiable.InverseHvpResult],\n            having the solution vector x that satisfies the system \\(Ax = b\\),\n            where \\(A\\) is a low-rank approximation of the Hessian \\(H\\) of the model's loss function, and an instance\n            of [LowRankProductRepresentation][pydvl.influence.torch.torch_differentiable.LowRankProductRepresentation],\n            which represents the approximation of H.\n    \"\"\"\nb_device = b.device if hasattr(b, \"device\") else torch.device(\"cpu\")\nif low_rank_representation is None:\nif b_device.type == \"cuda\" and not eigen_computation_on_gpu:\nraise ValueError(\n\"Using 'eigen_computation_on_gpu=False' while 'b' is on a 'cuda' device is not supported. \"\n\"To address this, consider the following options:\\n\"\n\" - Set eigen_computation_on_gpu=True if your model and data are small enough \"\n\"and if 'cupy' is available in your environment.\\n\"\n\" - Move 'b' to the CPU with b.to('cpu').\\n\"\n\" - Precompute a low rank representation and move it to the 'b' device using:\\n\"\n\"     low_rank_representation = model_hessian_low_rank(model, training_data, ..., \"\n\"eigen_computation_on_gpu=False).to(b.device)\"\n)\nlow_rank_representation = model_hessian_low_rank(\nmodel,\ntraining_data,\nhessian_perturbation=hessian_perturbation,\nrank_estimate=rank_estimate,\nkrylov_dimension=krylov_dimension,\ntol=tol,\nmax_iter=max_iter,\neigen_computation_on_gpu=eigen_computation_on_gpu,\n)\nelse:\nif b_device.type != low_rank_representation.device.type:\nraise RuntimeError(\nf\"The devices for 'b' and 'low_rank_representation' do not match.\\n\"\nf\" - 'b' is on device: {b_device}\\n\"\nf\" - 'low_rank_representation' is on device: {low_rank_representation.device}\\n\"\nf\"\\nTo resolve this, consider moving 'low_rank_representation' to '{b_device}' by using:\\n\"\nf\"low_rank_representation = low_rank_representation.to(b.device)\"\n)\nlogger.info(\"Using provided low rank representation, ignoring other parameters\")\nresult = low_rank_representation.projections @ (\ntorch.diag_embed(1.0 / low_rank_representation.eigen_vals)\n@ (low_rank_representation.projections.t() @ b.t())\n)\nreturn InverseHvpResult(\nx=result.t(),\ninfo={\n\"eigenvalues\": low_rank_representation.eigen_vals,\n\"eigenvectors\": low_rank_representation.projections,\n},\n)\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/","title":"Util","text":""},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.TorchTensorContainerType","title":"<code>TorchTensorContainerType = TypeVar('TorchTensorContainerType', torch.Tensor, Tuple[torch.Tensor, ...], Dict[str, torch.Tensor])</code>  <code>module-attribute</code>","text":"<p>Type variable for a PyTorch tensor or a container thereof.</p>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.to_model_device","title":"<code>to_model_device(x, model)</code>","text":"<p>Returns the tensor <code>x</code> moved to the device of the <code>model</code>, if device of model is set</p> PARAMETER  DESCRIPTION <code>x</code> <p>The tensor to be moved to the device of the model.</p> <p> TYPE: <code>Tensor</code> </p> <code>model</code> <p>The model whose device will be used to move the tensor.</p> <p> TYPE: <code>Module</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>The tensor <code>x</code> moved to the device of the <code>model</code>, if device of model is set.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def to_model_device(x: torch.Tensor, model: torch.nn.Module) -&gt; torch.Tensor:\n\"\"\"\n    Returns the tensor `x` moved to the device of the `model`, if device of model is set\n    Args:\n        x: The tensor to be moved to the device of the model.\n        model: The model whose device will be used to move the tensor.\n    Returns:\n        The tensor `x` moved to the device of the `model`, if device of model is set.\n    \"\"\"\nif hasattr(model, \"device\"):\nreturn x.to(model.device)\nreturn x\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.flatten_tensors_to_vector","title":"<code>flatten_tensors_to_vector(tensors)</code>","text":"<p>Flatten multiple tensors into a single 1D tensor (vector).</p> <p>This function takes an iterable of tensors and reshapes each of them into a 1D tensor. These reshaped tensors are then concatenated together into a single 1D tensor in the order they were given.</p> PARAMETER  DESCRIPTION <code>tensors</code> <p>An iterable of tensors to be reshaped and concatenated.</p> <p> TYPE: <code>Iterable[Tensor]</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A 1D tensor that is the concatenation of all the reshaped input tensors.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def flatten_tensors_to_vector(tensors: Iterable[torch.Tensor]) -&gt; torch.Tensor:\n\"\"\"\n    Flatten multiple tensors into a single 1D tensor (vector).\n    This function takes an iterable of tensors and reshapes each of them into a 1D tensor.\n    These reshaped tensors are then concatenated together into a single 1D tensor in the order they were given.\n    Args:\n        tensors: An iterable of tensors to be reshaped and concatenated.\n    Returns:\n        A 1D tensor that is the concatenation of all the reshaped input tensors.\n    \"\"\"\nreturn torch.cat([t.contiguous().view(-1) for t in tensors])\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.reshape_vector_to_tensors","title":"<code>reshape_vector_to_tensors(input_vector, target_shapes)</code>","text":"<p>Reshape a 1D tensor into multiple tensors with specified shapes.</p> <p>This function takes a 1D tensor (input_vector) and reshapes it into a series of tensors with shapes given by 'target_shapes'. The reshaped tensors are returned as a tuple in the same order as their corresponding shapes.</p> <p>Note: The total number of elements in 'input_vector' must be equal to the sum of the products of the shapes in 'target_shapes'.</p> PARAMETER  DESCRIPTION <code>input_vector</code> <p>The 1D tensor to be reshaped. Must be 1D.</p> <p> TYPE: <code>Tensor</code> </p> <code>target_shapes</code> <p>An iterable of tuples. Each tuple defines the shape of a tensor to be reshaped from the 'input_vector'.</p> <p> TYPE: <code>Iterable[Tuple[int, ...]]</code> </p> RETURNS DESCRIPTION <code>Tuple[Tensor, ...]</code> <p>A tuple of reshaped tensors.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If 'input_vector' is not a 1D tensor or if the total number of elements in 'input_vector' does not match the sum of the products of the shapes in 'target_shapes'.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def reshape_vector_to_tensors(\ninput_vector: torch.Tensor, target_shapes: Iterable[Tuple[int, ...]]\n) -&gt; Tuple[torch.Tensor, ...]:\n\"\"\"\n    Reshape a 1D tensor into multiple tensors with specified shapes.\n    This function takes a 1D tensor (input_vector) and reshapes it into a series of tensors with shapes given by 'target_shapes'.\n    The reshaped tensors are returned as a tuple in the same order as their corresponding shapes.\n    Note: The total number of elements in 'input_vector' must be equal to the sum of the products of the shapes in 'target_shapes'.\n    Args:\n        input_vector: The 1D tensor to be reshaped. Must be 1D.\n        target_shapes: An iterable of tuples. Each tuple defines the shape of a tensor to be reshaped from the 'input_vector'.\n    Returns:\n        A tuple of reshaped tensors.\n    Raises:\n        ValueError: If 'input_vector' is not a 1D tensor or if the total number of elements in 'input_vector' does not match the sum of the products of the shapes in 'target_shapes'.\n    \"\"\"\nif input_vector.dim() != 1:\nraise ValueError(\"Input vector must be a 1D tensor\")\ntotal_elements = sum(math.prod(shape) for shape in target_shapes)\nif total_elements != input_vector.shape[0]:\nraise ValueError(\nf\"The total elements in shapes {total_elements} does not match the vector length {input_vector.shape[0]}\"\n)\ntensors = []\nstart = 0\nfor shape in target_shapes:\nsize = math.prod(shape)  # compute the total size of the tensor with this shape\ntensors.append(\ninput_vector[start : start + size].view(shape)\n)  # slice the vector and reshape it\nstart += size\nreturn tuple(tensors)\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.align_structure","title":"<code>align_structure(source, target)</code>","text":"<p>This function transforms <code>target</code> to have the same structure as <code>source</code>, i.e., it should be a dictionary with the same keys as <code>source</code> and each corresponding value in <code>target</code> should have the same shape as the value in <code>source</code>.</p> PARAMETER  DESCRIPTION <code>source</code> <p>The reference dictionary containing PyTorch tensors.</p> <p> TYPE: <code>Dict[str, Tensor]</code> </p> <code>target</code> <p>The input to be harmonized. It can be a dictionary, tuple, or tensor.</p> <p> TYPE: <code>TorchTensorContainerType</code> </p> RETURNS DESCRIPTION <code>Dict[str, Tensor]</code> <p>The harmonized version of <code>target</code>.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If <code>target</code> cannot be harmonized to match <code>source</code>.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def align_structure(\nsource: Dict[str, torch.Tensor],\ntarget: TorchTensorContainerType,\n) -&gt; Dict[str, torch.Tensor]:\n\"\"\"\n    This function transforms `target` to have the same structure as `source`, i.e.,\n    it should be a dictionary with the same keys as `source` and each corresponding\n    value in `target` should have the same shape as the value in `source`.\n    Args:\n        source: The reference dictionary containing PyTorch tensors.\n        target: The input to be harmonized. It can be a dictionary, tuple, or tensor.\n    Returns:\n        The harmonized version of `target`.\n    Raises:\n        ValueError: If `target` cannot be harmonized to match `source`.\n    \"\"\"\ntangent_dict: Dict[str, torch.Tensor]\nif isinstance(target, dict):\nif list(target.keys()) != list(source.keys()):\nraise ValueError(\"The keys in 'target' do not match the keys in 'source'.\")\nif [v.shape for v in target.values()] != [v.shape for v in source.values()]:\nraise ValueError(\n\"The shapes of the values in 'target' do not match the shapes of the values in 'source'.\"\n)\ntangent_dict = target\nelif isinstance(target, tuple) or isinstance(target, list):\nif [v.shape for v in target] != [v.shape for v in source.values()]:\nraise ValueError(\n\"'target' is a tuple/list but its elements' shapes do not match the shapes \"\n\"of the values in 'source'.\"\n)\ntangent_dict = dict(zip(source.keys(), target))\nelif isinstance(target, torch.Tensor):\ntry:\ntangent_dict = dict(\nzip(\nsource.keys(),\nreshape_vector_to_tensors(\ntarget, [p.shape for p in source.values()]\n),\n)\n)\nexcept Exception as e:\nraise ValueError(\nf\"'target' is a tensor but cannot be reshaped to match 'source'. Original error: {e}\"\n)\nelse:\nraise ValueError(f\"'target' is of type {type(target)} which is not supported.\")\nreturn tangent_dict\n</code></pre>"},{"location":"api/pydvl/influence/torch/util/#pydvl.influence.torch.util.as_tensor","title":"<code>as_tensor(a, warn=True, **kwargs)</code>","text":"<p>Converts an array into a torch tensor.</p> PARAMETER  DESCRIPTION <code>a</code> <p>Array to convert to tensor.</p> <p> TYPE: <code>Any</code> </p> <code>warn</code> <p>If True, warns that <code>a</code> will be converted.</p> <p> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>Tensor</code> <p>A torch tensor converted from the input array.</p> Source code in <code>src/pydvl/influence/torch/util.py</code> <pre><code>def as_tensor(a: Any, warn=True, **kwargs) -&gt; torch.Tensor:\n\"\"\"\n    Converts an array into a torch tensor.\n    Args:\n        a: Array to convert to tensor.\n        warn: If True, warns that `a` will be converted.\n    Returns:\n        A torch tensor converted from the input array.\n    \"\"\"\nif warn and not isinstance(a, torch.Tensor):\nlogger.warning(\"Converting tensor to type torch.Tensor.\")\nreturn torch.as_tensor(a, **kwargs)\n</code></pre>"},{"location":"api/pydvl/parallel/","title":"Parallel","text":"<p>This module provides a common interface to parallelization backends. The list of supported backends is here. Backends can be selected with the <code>backend</code> argument of an instance of ParallelConfig, as seen in the examples below.</p> <p>We use executors to submit tasks in parallel. The basic high-level pattern is</p> <pre><code>from pydvl.parallel import init_executor, ParallelConfig\nconfig = ParallelConfig(backend=\"ray\")\nwith init_executor(max_workers=1, config=config) as executor:\nfuture = executor.submit(lambda x: x + 1, 1)\nresult = future.result()\nassert result == 2\n</code></pre> <p>Running a map-reduce job is also easy:</p> <pre><code>from pydvl.parallel import init_executor, ParallelConfig\nconfig = ParallelConfig(backend=\"joblib\")\nwith init_executor(config=config) as executor:\nresults = list(executor.map(lambda x: x + 1, range(5)))\nassert results == [1, 2, 3, 4, 5]\n</code></pre> <p>There is an alternative map-reduce implementation MapReduceJob which internally uses joblib's higher level API with <code>Parallel()</code></p>"},{"location":"api/pydvl/parallel/backend/","title":"Backend","text":""},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.CancellationPolicy","title":"<code>CancellationPolicy</code>","text":"<p>             Bases: <code>Flag</code></p> <p>Policy to use when cancelling futures after exiting an Executor.</p> <p>Note</p> <p>Not all backends support all policies.</p> ATTRIBUTE DESCRIPTION <code>NONE</code> <p>Do not cancel any futures.</p> <p> </p> <code>PENDING</code> <p>Cancel all pending futures, but not running ones.</p> <p> </p> <code>RUNNING</code> <p>Cancel all running futures, but not pending ones.</p> <p> </p> <code>ALL</code> <p>Cancel all pending and running futures.</p> <p> </p>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.BaseParallelBackend","title":"<code>BaseParallelBackend</code>","text":"<p>Abstract base class for all parallel backends.</p>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.BaseParallelBackend.executor","title":"<code>executor(max_workers=None, config=ParallelConfig(), cancel_futures=CancellationPolicy.PENDING)</code>  <code>abstractmethod</code> <code>classmethod</code>","text":"<p>Returns an executor for the parallel backend.</p> Source code in <code>src/pydvl/parallel/backend.py</code> <pre><code>@classmethod\n@abstractmethod\ndef executor(\ncls,\nmax_workers: int | None = None,\nconfig: ParallelConfig = ParallelConfig(),\ncancel_futures: CancellationPolicy = CancellationPolicy.PENDING,\n) -&gt; Executor:\n\"\"\"Returns an executor for the parallel backend.\"\"\"\n...\n</code></pre>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.init_parallel_backend","title":"<code>init_parallel_backend(config)</code>","text":"<p>Initializes the parallel backend and returns an instance of it.</p> <p>The following example creates a parallel backend instance with the default configuration, which is a local joblib backend.</p> Example <pre><code>config = ParallelConfig()\nparallel_backend = init_parallel_backend(config)\n</code></pre> <p>To create a parallel backend instance with a different backend, e.g. ray, you can pass the backend name as a string to the constructor of ParallelConfig.</p> Example <pre><code>config = ParallelConfig(backend=\"ray\")\nparallel_backend = init_parallel_backend(config)\n</code></pre> PARAMETER  DESCRIPTION <code>config</code> <p>instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> </p> Source code in <code>src/pydvl/parallel/backend.py</code> <pre><code>def init_parallel_backend(config: ParallelConfig) -&gt; BaseParallelBackend:\n\"\"\"Initializes the parallel backend and returns an instance of it.\n    The following example creates a parallel backend instance with the default\n    configuration, which is a local joblib backend.\n    ??? Example\n        ``` python\n        config = ParallelConfig()\n        parallel_backend = init_parallel_backend(config)\n        ```\n    To create a parallel backend instance with a different backend, e.g. ray,\n    you can pass the backend name as a string to the constructor of\n    [ParallelConfig][pydvl.utils.config.ParallelConfig].\n    ??? Example\n        ```python\n        config = ParallelConfig(backend=\"ray\")\n        parallel_backend = init_parallel_backend(config)\n        ```\n    Args:\n        config: instance of [ParallelConfig][pydvl.utils.config.ParallelConfig]\n            with cluster address, number of cpus, etc.\n    \"\"\"\ntry:\nparallel_backend_cls = BaseParallelBackend.BACKENDS[config.backend]\nexcept KeyError:\nraise NotImplementedError(f\"Unexpected parallel backend {config.backend}\")\nreturn parallel_backend_cls.create(config)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.available_cpus","title":"<code>available_cpus()</code>","text":"<p>Platform-independent count of available cores.</p> <p>FIXME: do we really need this or is <code>os.cpu_count</code> enough? Is this portable?</p> RETURNS DESCRIPTION <code>int</code> <p>Number of cores, or 1 if it is not possible to determine.</p> Source code in <code>src/pydvl/parallel/backend.py</code> <pre><code>def available_cpus() -&gt; int:\n\"\"\"Platform-independent count of available cores.\n    FIXME: do we really need this or is `os.cpu_count` enough? Is this portable?\n    Returns:\n        Number of cores, or 1 if it is not possible to determine.\n    \"\"\"\nfrom platform import system\nif system() != \"Linux\":\nreturn os.cpu_count() or 1\nreturn len(os.sched_getaffinity(0))  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/backend/#pydvl.parallel.backend.effective_n_jobs","title":"<code>effective_n_jobs(n_jobs, config=ParallelConfig())</code>","text":"<p>Returns the effective number of jobs.</p> <p>This number may vary depending on the parallel backend and the resources available.</p> PARAMETER  DESCRIPTION <code>n_jobs</code> <p>the number of jobs requested. If -1, the number of available CPUs is returned.</p> <p> TYPE: <code>int</code> </p> <code>config</code> <p>instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> RETURNS DESCRIPTION <code>int</code> <p>The effective number of jobs, guaranteed to be &gt;= 1.</p> RAISES DESCRIPTION <code>RuntimeError</code> <p>if the effective number of jobs returned by the backend is &lt; 1.</p> Source code in <code>src/pydvl/parallel/backend.py</code> <pre><code>def effective_n_jobs(n_jobs: int, config: ParallelConfig = ParallelConfig()) -&gt; int:\n\"\"\"Returns the effective number of jobs.\n    This number may vary depending on the parallel backend and the resources\n    available.\n    Args:\n        n_jobs: the number of jobs requested. If -1, the number of available\n            CPUs is returned.\n        config: instance of [ParallelConfig][pydvl.utils.config.ParallelConfig] with\n            cluster address, number of cpus, etc.\n    Returns:\n        The effective number of jobs, guaranteed to be &gt;= 1.\n    Raises:\n        RuntimeError: if the effective number of jobs returned by the backend\n            is &lt; 1.\n    \"\"\"\nparallel_backend = init_parallel_backend(config)\nif (eff_n_jobs := parallel_backend.effective_n_jobs(n_jobs)) &lt; 1:\nraise RuntimeError(\nf\"Invalid number of jobs {eff_n_jobs} obtained from parallel backend {config.backend}\"\n)\nreturn eff_n_jobs\n</code></pre>"},{"location":"api/pydvl/parallel/config/","title":"Config","text":""},{"location":"api/pydvl/parallel/config/#pydvl.parallel.config.ParallelConfig","title":"<code>ParallelConfig</code>  <code>dataclass</code>","text":"<p>Configuration for parallel computation backend.</p> PARAMETER  DESCRIPTION <code>backend</code> <p>Type of backend to use. Defaults to 'joblib'</p> <p> TYPE: <code>Literal['joblib', 'ray']</code> DEFAULT: <code>'joblib'</code> </p> <code>address</code> <p>Address of existing remote or local cluster to use.</p> <p> TYPE: <code>Optional[Union[str, Tuple[str, int]]]</code> DEFAULT: <code>None</code> </p> <code>n_cpus_local</code> <p>Number of CPUs to use when creating a local ray cluster. This has no effect when using an existing ray cluster.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>logging_level</code> <p>Logging level for the parallel backend's worker.</p> <p> TYPE: <code>int</code> DEFAULT: <code>WARNING</code> </p> <code>wait_timeout</code> <p>Timeout in seconds for waiting on futures.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p>"},{"location":"api/pydvl/parallel/map_reduce/","title":"Map reduce","text":"<p>This module contains a wrapper around joblib's <code>Parallel()</code> class that makes it easy to run map-reduce jobs.</p> <p>Deprecation</p> <p>This interface might be deprecated or changed in a future release before 1.0</p>"},{"location":"api/pydvl/parallel/map_reduce/#pydvl.parallel.map_reduce.MapReduceJob","title":"<code>MapReduceJob(inputs, map_func, reduce_func=identity, map_kwargs=None, reduce_kwargs=None, config=ParallelConfig(), *, n_jobs=-1, timeout=None)</code>","text":"<p>             Bases: <code>Generic[T, R]</code></p> <p>Takes an embarrassingly parallel fun and runs it in <code>n_jobs</code> parallel jobs, splitting the data evenly into a number of chunks equal to the number of jobs.</p> <p>Typing information for objects of this class requires the type of the inputs that are split for <code>map_func</code> and the type of its output.</p> PARAMETER  DESCRIPTION <code>inputs</code> <p>The input that will be split and passed to <code>map_func</code>. if it's not a sequence object. It will be repeat <code>n_jobs</code> number of times.</p> <p> TYPE: <code>Union[Collection[T], T]</code> </p> <code>map_func</code> <p>Function that will be applied to the input chunks in each job.</p> <p> TYPE: <code>MapFunction[R]</code> </p> <code>reduce_func</code> <p>Function that will be applied to the results of <code>map_func</code> to reduce them.</p> <p> TYPE: <code>ReduceFunction[R]</code> DEFAULT: <code>identity</code> </p> <code>map_kwargs</code> <p>Keyword arguments that will be passed to <code>map_func</code> in each job. Alternatively, one can use functools.partial.</p> <p> TYPE: <code>Optional[Dict]</code> DEFAULT: <code>None</code> </p> <code>reduce_kwargs</code> <p>Keyword arguments that will be passed to <code>reduce_func</code> in each job. Alternatively, one can use functools.partial.</p> <p> TYPE: <code>Optional[Dict]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>Instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to run. Does not accept 0</p> <p> TYPE: <code>int</code> DEFAULT: <code>-1</code> </p> Example <p>A simple usage example with 2 jobs:</p> <pre><code>&gt;&gt;&gt; from pydvl.parallel import MapReduceJob\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; map_reduce_job: MapReduceJob[np.ndarray, np.ndarray] = MapReduceJob(\n...     np.arange(5),\n...     map_func=np.sum,\n...     reduce_func=np.sum,\n...     n_jobs=2,\n... )\n&gt;&gt;&gt; map_reduce_job()\n10\n</code></pre> <p>When passed a single object as input, it will be repeated for each job: <pre><code>&gt;&gt;&gt; from pydvl.parallel import MapReduceJob\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; map_reduce_job: MapReduceJob[int, np.ndarray] = MapReduceJob(\n...     5,\n...     map_func=lambda x: np.array([x]),\n...     reduce_func=np.sum,\n...     n_jobs=2,\n... )\n&gt;&gt;&gt; map_reduce_job()\n10\n</code></pre></p> Source code in <code>src/pydvl/parallel/map_reduce.py</code> <pre><code>def __init__(\nself,\ninputs: Union[Collection[T], T],\nmap_func: MapFunction[R],\nreduce_func: ReduceFunction[R] = identity,\nmap_kwargs: Optional[Dict] = None,\nreduce_kwargs: Optional[Dict] = None,\nconfig: ParallelConfig = ParallelConfig(),\n*,\nn_jobs: int = -1,\ntimeout: Optional[float] = None,\n):\nself.config = config\nparallel_backend = init_parallel_backend(self.config)\nself.parallel_backend = parallel_backend\nself.timeout = timeout\n# This uses the setter defined below\nself.n_jobs = n_jobs\nself.inputs_ = inputs\nself.map_kwargs = map_kwargs if map_kwargs is not None else dict()\nself.reduce_kwargs = reduce_kwargs if reduce_kwargs is not None else dict()\nself._map_func = reduce(maybe_add_argument, [\"job_id\", \"seed\"], map_func)\nself._reduce_func = reduce_func\n</code></pre>"},{"location":"api/pydvl/parallel/map_reduce/#pydvl.parallel.map_reduce.MapReduceJob.n_jobs","title":"<code>n_jobs: int</code>  <code>property</code> <code>writable</code>","text":"<p>Effective number of jobs according to the used ParallelBackend instance.</p>"},{"location":"api/pydvl/parallel/map_reduce/#pydvl.parallel.map_reduce.MapReduceJob.__call__","title":"<code>__call__(seed=None)</code>","text":"<p>Runs the map-reduce job.</p> PARAMETER  DESCRIPTION <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Union[Seed, SeedSequence]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>R</code> <p>The result of the reduce function.</p> Source code in <code>src/pydvl/parallel/map_reduce.py</code> <pre><code>def __call__(\nself,\nseed: Optional[Union[Seed, SeedSequence]] = None,\n) -&gt; R:\n\"\"\"\n    Runs the map-reduce job.\n    Args:\n        seed: Either an instance of a numpy random number generator or a seed for\n            it.\n    Returns:\n         The result of the reduce function.\n    \"\"\"\nif self.config.backend == \"joblib\":\nbackend = \"loky\"\nelse:\nbackend = self.config.backend\n# In joblib the levels are reversed.\n# 0 means no logging and 50 means log everything to stdout\nverbose = 50 - self.config.logging_level\nseed_seq = ensure_seed_sequence(seed)\nwith Parallel(backend=backend, n_jobs=self.n_jobs, verbose=verbose) as parallel:\nchunks = self._chunkify(self.inputs_, n_chunks=self.n_jobs)\nmap_results: List[R] = parallel(\ndelayed(self._map_func)(\nnext_chunk, job_id=j, seed=seed, **self.map_kwargs\n)\nfor j, (next_chunk, seed) in enumerate(\nzip(chunks, seed_seq.spawn(len(chunks)))\n)\n)\nreduce_results: R = self._reduce_func(map_results, **self.reduce_kwargs)\nreturn reduce_results\n</code></pre>"},{"location":"api/pydvl/parallel/backends/","title":"Backends","text":""},{"location":"api/pydvl/parallel/backends/joblib/","title":"Joblib","text":""},{"location":"api/pydvl/parallel/backends/joblib/#pydvl.parallel.backends.joblib.JoblibParallelBackend","title":"<code>JoblibParallelBackend(config)</code>","text":"<p>             Bases: <code>BaseParallelBackend</code></p> <p>Class used to wrap joblib to make it transparent to algorithms.</p> <p>It shouldn't be initialized directly. You should instead call init_parallel_backend().</p> PARAMETER  DESCRIPTION <code>config</code> <p>instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> </p> Source code in <code>src/pydvl/parallel/backends/joblib.py</code> <pre><code>def __init__(self, config: ParallelConfig):\nself.config = {\n\"logging_level\": config.logging_level,\n\"n_jobs\": config.n_cpus_local,\n}\n</code></pre>"},{"location":"api/pydvl/parallel/backends/joblib/#pydvl.parallel.backends.joblib.JoblibParallelBackend.wrap","title":"<code>wrap(fun, **kwargs)</code>","text":"<p>Wraps a function as a joblib delayed.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>the function to wrap</p> <p> TYPE: <code>Callable</code> </p> RETURNS DESCRIPTION <code>Callable</code> <p>The delayed function.</p> Source code in <code>src/pydvl/parallel/backends/joblib.py</code> <pre><code>def wrap(self, fun: Callable, **kwargs) -&gt; Callable:\n\"\"\"Wraps a function as a joblib delayed.\n    Args:\n        fun: the function to wrap\n    Returns:\n        The delayed function.\n    \"\"\"\nreturn delayed(fun)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/backends/ray/","title":"Ray","text":""},{"location":"api/pydvl/parallel/backends/ray/#pydvl.parallel.backends.ray.RayParallelBackend","title":"<code>RayParallelBackend(config)</code>","text":"<p>             Bases: <code>BaseParallelBackend</code></p> <p>Class used to wrap ray to make it transparent to algorithms.</p> <p>It shouldn't be initialized directly. You should instead call init_parallel_backend().</p> PARAMETER  DESCRIPTION <code>config</code> <p>instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> </p> Source code in <code>src/pydvl/parallel/backends/ray.py</code> <pre><code>def __init__(self, config: ParallelConfig):\nself.config = {\"address\": config.address, \"logging_level\": config.logging_level}\nif self.config[\"address\"] is None:\nself.config[\"num_cpus\"] = config.n_cpus_local\nif not ray.is_initialized():\nray.init(**self.config)\n# Register ray joblib backend\nregister_ray()\n</code></pre>"},{"location":"api/pydvl/parallel/backends/ray/#pydvl.parallel.backends.ray.RayParallelBackend.wrap","title":"<code>wrap(fun, **kwargs)</code>","text":"<p>Wraps a function as a ray remote.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>the function to wrap</p> <p> TYPE: <code>Callable</code> </p> <code>kwargs</code> <p>keyword arguments to pass to @ray.remote</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Callable</code> <p>The <code>.remote</code> method of the ray <code>RemoteFunction</code>.</p> Source code in <code>src/pydvl/parallel/backends/ray.py</code> <pre><code>def wrap(self, fun: Callable, **kwargs) -&gt; Callable:\n\"\"\"Wraps a function as a ray remote.\n    Args:\n        fun: the function to wrap\n        kwargs: keyword arguments to pass to @ray.remote\n    Returns:\n        The `.remote` method of the ray `RemoteFunction`.\n    \"\"\"\nif len(kwargs) &gt; 0:\nreturn ray.remote(**kwargs)(fun).remote  # type: ignore\nreturn ray.remote(fun).remote  # type: ignore\n</code></pre>"},{"location":"api/pydvl/parallel/futures/","title":"Futures","text":""},{"location":"api/pydvl/parallel/futures/#pydvl.parallel.futures.init_executor","title":"<code>init_executor(max_workers=None, config=ParallelConfig(), **kwargs)</code>","text":"<p>Initializes a futures executor for the given parallel configuration.</p> PARAMETER  DESCRIPTION <code>max_workers</code> <p>Maximum number of concurrent tasks.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>kwargs</code> <p>Other optional parameter that will be passed to the executor.</p> <p> DEFAULT: <code>{}</code> </p> Examples <p><pre><code>from pydvl.parallel import init_executor, ParallelConfig\nconfig = ParallelConfig(backend=\"ray\")\nwith init_executor(max_workers=1, config=config) as executor:\nfuture = executor.submit(lambda x: x + 1, 1)\nresult = future.result()\nassert result == 2\n</code></pre> <pre><code>from pydvl.parallel.futures import init_executor\nwith init_executor() as executor:\nresults = list(executor.map(lambda x: x + 1, range(5)))\nassert results == [1, 2, 3, 4, 5]\n</code></pre></p> Source code in <code>src/pydvl/parallel/futures/__init__.py</code> <pre><code>@contextmanager\ndef init_executor(\nmax_workers: Optional[int] = None,\nconfig: ParallelConfig = ParallelConfig(),\n**kwargs,\n) -&gt; Generator[Executor, None, None]:\n\"\"\"Initializes a futures executor for the given parallel configuration.\n    Args:\n        max_workers: Maximum number of concurrent tasks.\n        config: instance of [ParallelConfig][pydvl.utils.config.ParallelConfig]\n            with cluster address, number of cpus, etc.\n        kwargs: Other optional parameter that will be passed to the executor.\n    ??? Examples\n        ``` python\n        from pydvl.parallel import init_executor, ParallelConfig\n        config = ParallelConfig(backend=\"ray\")\n        with init_executor(max_workers=1, config=config) as executor:\n            future = executor.submit(lambda x: x + 1, 1)\n            result = future.result()\n        assert result == 2\n        ```\n        ``` python\n        from pydvl.parallel.futures import init_executor\n        with init_executor() as executor:\n            results = list(executor.map(lambda x: x + 1, range(5)))\n        assert results == [1, 2, 3, 4, 5]\n        ```\n    \"\"\"\ntry:\ncls = BaseParallelBackend.BACKENDS[config.backend]\nwith cls.executor(max_workers=max_workers, config=config, **kwargs) as e:\nyield e\nexcept KeyError:\nraise NotImplementedError(f\"Unexpected parallel backend {config.backend}\")\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/","title":"Ray","text":""},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor","title":"<code>RayExecutor(max_workers=None, *, config=ParallelConfig(), cancel_futures=CancellationPolicy.ALL)</code>","text":"<p>             Bases: <code>Executor</code></p> <p>Asynchronous executor using Ray that implements the concurrent.futures API.</p> <p>It shouldn't be initialized directly. You should instead call init_executor().</p> PARAMETER  DESCRIPTION <code>max_workers</code> <p>Maximum number of concurrent tasks. Each task can request itself any number of vCPUs. You must ensure the product of this value and the n_cpus_per_job parameter passed to submit() does not exceed available cluster resources. If set to <code>None</code>, it will default to the total number of vCPUs in the ray cluster.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>instance of ParallelConfig with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>cancel_futures</code> <p>Select which futures will be cancelled when exiting this context manager. <code>Pending</code> is the default, which will cancel all pending futures, but not running ones, as done by concurrent.futures.ProcessPoolExecutor. Additionally, <code>All</code> cancels all pending and running futures, and <code>None</code> doesn't cancel any. See CancellationPolicy</p> <p> TYPE: <code>CancellationPolicy</code> DEFAULT: <code>ALL</code> </p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>@deprecated(\ntarget=True,\ndeprecated_in=\"0.7.0\",\nremove_in=\"0.8.0\",\nargs_mapping={\"cancel_futures_on_exit\": \"cancel_futures\"},\n)\ndef __init__(\nself,\nmax_workers: Optional[int] = None,\n*,\nconfig: ParallelConfig = ParallelConfig(),\ncancel_futures: CancellationPolicy = CancellationPolicy.ALL,\n):\nif config.backend != \"ray\":\nraise ValueError(\nf\"Parallel backend must be set to 'ray' and not '{config.backend}'\"\n)\nif max_workers is not None:\nif max_workers &lt;= 0:\nraise ValueError(\"max_workers must be greater than 0\")\nmax_workers = max_workers\nif isinstance(cancel_futures, CancellationPolicy):\nself._cancel_futures = cancel_futures\nelse:\nself._cancel_futures = (\nCancellationPolicy.PENDING\nif cancel_futures\nelse CancellationPolicy.NONE\n)\nself.config = {\"address\": config.address, \"logging_level\": config.logging_level}\nif config.address is None:\nself.config[\"num_cpus\"] = config.n_cpus_local\nif not ray.is_initialized():\nray.init(**self.config)\nself._max_workers = max_workers\nif self._max_workers is None:\nself._max_workers = int(ray._private.state.cluster_resources()[\"CPU\"])\nself._shutdown = False\nself._shutdown_lock = threading.Lock()\nself._queue_lock = threading.Lock()\nself._work_queue: \"queue.Queue[Optional[_WorkItem]]\" = queue.Queue(\nmaxsize=self._max_workers\n)\nself._pending_queue: \"queue.SimpleQueue[Optional[_WorkItem]]\" = (\nqueue.SimpleQueue()\n)\n# Work Item Manager Thread\nself._work_item_manager_thread: Optional[_WorkItemManagerThread] = None\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor.submit","title":"<code>submit(fn, *args, **kwargs)</code>","text":"<p>Submits a callable to be executed with the given arguments.</p> <p>Schedules the callable to be executed as fn(*args, **kwargs) and returns a Future instance representing the execution of the callable.</p> PARAMETER  DESCRIPTION <code>fn</code> <p>Callable.</p> <p> TYPE: <code>Callable[..., T]</code> </p> <code>args</code> <p>Positional arguments that will be passed to <code>fn</code>.</p> <p> DEFAULT: <code>()</code> </p> <code>kwargs</code> <p>Keyword arguments that will be passed to <code>fn</code>. It can also optionally contain options for the ray remote function as a dictionary as the keyword argument <code>remote_function_options</code>.</p> <p> DEFAULT: <code>{}</code> </p> <p>Returns:     A Future representing the given call.</p> RAISES DESCRIPTION <code>RuntimeError</code> <p>If a task is submitted after the executor has been shut down.</p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>def submit(self, fn: Callable[..., T], *args, **kwargs) -&gt; \"Future[T]\":\nr\"\"\"Submits a callable to be executed with the given arguments.\n    Schedules the callable to be executed as fn(\\*args, \\**kwargs)\n    and returns a Future instance representing the execution of the callable.\n    Args:\n        fn: Callable.\n        args: Positional arguments that will be passed to `fn`.\n        kwargs: Keyword arguments that will be passed to `fn`.\n            It can also optionally contain options for the ray remote function\n            as a dictionary as the keyword argument `remote_function_options`.\n    Returns:\n        A Future representing the given call.\n    Raises:\n        RuntimeError: If a task is submitted after the executor has been shut down.\n    \"\"\"\nwith self._shutdown_lock:\nlogger.debug(\"executor acquired shutdown lock\")\nif self._shutdown:\nraise RuntimeError(\"cannot schedule new futures after shutdown\")\nlogging.debug(\"Creating future and putting work item in work queue\")\nfuture: \"Future[T]\" = Future()\nremote_function_options = kwargs.pop(\"remote_function_options\", None)\nw = _WorkItem(\nfuture,\nfn,\nargs,\nkwargs,\nremote_function_options=remote_function_options,\n)\nself._put_work_item_in_queue(w)\n# We delay starting the thread until the first call to submit\nself._start_work_item_manager_thread()\nreturn future\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor.shutdown","title":"<code>shutdown(wait=True, *, cancel_futures=None)</code>","text":"<p>Clean up the resources associated with the Executor.</p> <p>This method tries to mimic the behaviour of Executor.shutdown while allowing one more value for <code>cancel_futures</code> which instructs it to use the CancellationPolicy defined upon construction.</p> PARAMETER  DESCRIPTION <code>wait</code> <p>Whether to wait for pending futures to finish.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>cancel_futures</code> <p>Overrides the executor's default policy for cancelling futures on exit. If <code>True</code>, all pending futures are cancelled, and if <code>False</code>, no futures are cancelled. If <code>None</code> (default), the executor's policy set at initialization is used.</p> <p> TYPE: <code>Optional[bool]</code> DEFAULT: <code>None</code> </p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>def shutdown(\nself, wait: bool = True, *, cancel_futures: Optional[bool] = None\n) -&gt; None:\n\"\"\"Clean up the resources associated with the Executor.\n    This method tries to mimic the behaviour of\n    [Executor.shutdown][concurrent.futures.Executor.shutdown]\n    while allowing one more value for ``cancel_futures`` which instructs it\n    to use the [CancellationPolicy][pydvl.parallel.backend.CancellationPolicy]\n    defined upon construction.\n    Args:\n        wait: Whether to wait for pending futures to finish.\n        cancel_futures: Overrides the executor's default policy for\n            cancelling futures on exit. If ``True``, all pending futures are\n            cancelled, and if ``False``, no futures are cancelled. If ``None``\n            (default), the executor's policy set at initialization is used.\n    \"\"\"\nlogger.debug(\"executor shutting down\")\nwith self._shutdown_lock:\nlogger.debug(\"executor acquired shutdown lock\")\nself._shutdown = True\nself._cancel_futures = {\nNone: self._cancel_futures,\nTrue: CancellationPolicy.PENDING,\nFalse: CancellationPolicy.NONE,\n}[cancel_futures]\nif wait:\nlogger.debug(\"executor waiting for futures to finish\")\nif self._work_item_manager_thread is not None:\n# Putting None in the queue to signal\n# to work item manager thread that we are shutting down\nself._put_work_item_in_queue(None)\nlogger.debug(\n\"executor waiting for work item manager thread to terminate\"\n)\nself._work_item_manager_thread.join()\n# To reduce the risk of opening too many files, remove references to\n# objects that use file descriptors.\nself._work_item_manager_thread = None\ndel self._work_queue\ndel self._pending_queue\n</code></pre>"},{"location":"api/pydvl/parallel/futures/ray/#pydvl.parallel.futures.ray.RayExecutor.__exit__","title":"<code>__exit__(exc_type, exc_val, exc_tb)</code>","text":"<p>Exit the runtime context related to the RayExecutor object.</p> Source code in <code>src/pydvl/parallel/futures/ray.py</code> <pre><code>def __exit__(self, exc_type, exc_val, exc_tb):\n\"\"\"Exit the runtime context related to the RayExecutor object.\"\"\"\nself.shutdown()\nreturn False\n</code></pre>"},{"location":"api/pydvl/reporting/","title":"Reporting","text":""},{"location":"api/pydvl/reporting/plots/","title":"Plots","text":""},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.shaded_mean_std","title":"<code>shaded_mean_std(data, abscissa=None, num_std=1.0, mean_color='dodgerblue', shade_color='lightblue', title=None, xlabel=None, ylabel=None, ax=None, **kwargs)</code>","text":"<p>The usual mean \\(\\pm\\) std deviation plot to aggregate runs of experiments.</p> PARAMETER  DESCRIPTION <code>data</code> <p>axis 0 is to be aggregated on (e.g. runs) and axis 1 is the data for each run.</p> <p> TYPE: <code>ndarray</code> </p> <code>abscissa</code> <p>values for the x-axis. Leave empty to use increasing integers.</p> <p> TYPE: <code>Optional[Sequence[Any]]</code> DEFAULT: <code>None</code> </p> <code>num_std</code> <p>number of standard deviations to shade around the mean.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>mean_color</code> <p>color for the mean</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'dodgerblue'</code> </p> <code>shade_color</code> <p>color for the shaded region</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>'lightblue'</code> </p> <code>title</code> <p>Title text. To use mathematics, use LaTeX notation.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>xlabel</code> <p>Text for the horizontal axis.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ylabel</code> <p>Text for the vertical axis</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ax</code> <p>If passed, axes object into which to insert the figure. Otherwise, a new figure is created and returned</p> <p> TYPE: <code>Optional[Axes]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>these are forwarded to the ax.plot() call for the mean.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Axes</code> <p>The axes used (or created)</p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def shaded_mean_std(\ndata: np.ndarray,\nabscissa: Optional[Sequence[Any]] = None,\nnum_std: float = 1.0,\nmean_color: Optional[str] = \"dodgerblue\",\nshade_color: Optional[str] = \"lightblue\",\ntitle: Optional[str] = None,\nxlabel: Optional[str] = None,\nylabel: Optional[str] = None,\nax: Optional[Axes] = None,\n**kwargs,\n) -&gt; Axes:\n\"\"\"The usual mean \\(\\pm\\) std deviation plot to aggregate runs of experiments.\n    Args:\n        data: axis 0 is to be aggregated on (e.g. runs) and axis 1 is the\n            data for each run.\n        abscissa: values for the x-axis. Leave empty to use increasing integers.\n        num_std: number of standard deviations to shade around the mean.\n        mean_color: color for the mean\n        shade_color: color for the shaded region\n        title: Title text. To use mathematics, use LaTeX notation.\n        xlabel: Text for the horizontal axis.\n        ylabel: Text for the vertical axis\n        ax: If passed, axes object into which to insert the figure. Otherwise,\n            a new figure is created and returned\n        kwargs: these are forwarded to the ax.plot() call for the mean.\n    Returns:\n        The axes used (or created)\n    \"\"\"\nassert len(data.shape) == 2\nmean = data.mean(axis=0)\nstd = num_std * data.std(axis=0)\nif ax is None:\nfig, ax = plt.subplots()\nif abscissa is None:\nabscissa = list(range(data.shape[1]))\nax.fill_between(abscissa, mean - std, mean + std, alpha=0.3, color=shade_color)\nax.plot(abscissa, mean, color=mean_color, **kwargs)\nax.set_title(title)\nax.set_xlabel(xlabel)\nax.set_ylabel(ylabel)\nreturn ax\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.spearman_correlation","title":"<code>spearman_correlation(vv, num_values, pvalue)</code>","text":"<p>Simple matrix plots with spearman correlation for each pair in vv.</p> PARAMETER  DESCRIPTION <code>vv</code> <p>list of OrderedDicts with index: value. Spearman correlation is computed for the keys.</p> <p> TYPE: <code>List[OrderedDict]</code> </p> <code>num_values</code> <p>Use only these many values from the data (from the start of the OrderedDicts)</p> <p> TYPE: <code>int</code> </p> <code>pvalue</code> <p>correlation coefficients for which the p-value is below the threshold <code>pvalue/len(vv)</code> will be discarded.</p> <p> TYPE: <code>float</code> </p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def spearman_correlation(vv: List[OrderedDict], num_values: int, pvalue: float):\n\"\"\"Simple matrix plots with spearman correlation for each pair in vv.\n    Args:\n        vv: list of OrderedDicts with index: value. Spearman correlation\n            is computed for the keys.\n        num_values: Use only these many values from the data (from the start\n            of the OrderedDicts)\n        pvalue: correlation coefficients for which the p-value is below the\n            threshold `pvalue/len(vv)` will be discarded.\n    \"\"\"\nr: np.ndarray = np.ndarray((len(vv), len(vv)))\np: np.ndarray = np.ndarray((len(vv), len(vv)))\nfor i, a in enumerate(vv):\nfor j, b in enumerate(vv):\nfrom scipy.stats._stats_py import SpearmanrResult\nspearman: SpearmanrResult = sp.stats.spearmanr(\nlist(a.keys())[:num_values], list(b.keys())[:num_values]\n)\nr[i][j] = (\nspearman.correlation if spearman.pvalue &lt; pvalue / len(vv) else np.nan\n)  # Bonferroni correction\np[i][j] = spearman.pvalue\nfig, axs = plt.subplots(1, 2, figsize=(16, 7))\nplot1 = axs[0].matshow(r, vmin=-1, vmax=1)\naxs[0].set_title(f\"Spearman correlation (top {num_values} values)\")\naxs[0].set_xlabel(\"Runs\")\naxs[0].set_ylabel(\"Runs\")\nfig.colorbar(plot1, ax=axs[0])\nplot2 = axs[1].matshow(p, vmin=0, vmax=1)\naxs[1].set_title(\"p-value\")\naxs[1].set_xlabel(\"Runs\")\naxs[1].set_ylabel(\"Runs\")\nfig.colorbar(plot2, ax=axs[1])\nreturn fig\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.plot_shapley","title":"<code>plot_shapley(df, *, level=0.05, ax=None, title=None, xlabel=None, ylabel=None)</code>","text":"<p>Plots the shapley values, as returned from compute_shapley_values, with error bars corresponding to an \\(\\alpha\\)-level confidence interval.</p> PARAMETER  DESCRIPTION <code>df</code> <p>dataframe with the shapley values</p> <p> TYPE: <code>DataFrame</code> </p> <code>level</code> <p>confidence level for the error bars</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.05</code> </p> <code>ax</code> <p>axes to plot on or None if a new subplots should be created</p> <p> TYPE: <code>Optional[Axes]</code> DEFAULT: <code>None</code> </p> <code>title</code> <p>string, title of the plot</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>xlabel</code> <p>string, x label of the plot</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>ylabel</code> <p>string, y label of the plot</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Axes</code> <p>The axes created or used</p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def plot_shapley(\ndf: pd.DataFrame,\n*,\nlevel: float = 0.05,\nax: Optional[plt.Axes] = None,\ntitle: Optional[str] = None,\nxlabel: Optional[str] = None,\nylabel: Optional[str] = None,\n) -&gt; plt.Axes:\nr\"\"\"Plots the shapley values, as returned from\n    [compute_shapley_values][pydvl.value.shapley.common.compute_shapley_values], with error bars\n    corresponding to an $\\alpha$-level confidence interval.\n    Args:\n        df: dataframe with the shapley values\n        level: confidence level for the error bars\n        ax: axes to plot on or None if a new subplots should be created\n        title: string, title of the plot\n        xlabel: string, x label of the plot\n        ylabel: string, y label of the plot\n    Returns:\n        The axes created or used\n    \"\"\"\nif ax is None:\n_, ax = plt.subplots()\nyerr = norm.ppf(1 - level / 2) * df[\"data_value_stderr\"]\nax.errorbar(x=df.index, y=df[\"data_value\"], yerr=yerr, fmt=\"o\", capsize=6)\nax.set_xlabel(xlabel)\nax.set_ylabel(ylabel)\nax.set_title(title)\nplt.xticks(rotation=60)\nreturn ax\n</code></pre>"},{"location":"api/pydvl/reporting/plots/#pydvl.reporting.plots.plot_influence_distribution_by_label","title":"<code>plot_influence_distribution_by_label(influences, labels, title_extra='')</code>","text":"<p>Plots the histogram of the influence that all samples in the training set have over a single sample index, separated by labels.</p> PARAMETER  DESCRIPTION <code>influences</code> <p>array of influences (training samples x test samples)</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>labels</code> <p>labels for the training set.</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>title_extra</code> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> Source code in <code>src/pydvl/reporting/plots.py</code> <pre><code>def plot_influence_distribution_by_label(\ninfluences: NDArray[np.float_], labels: NDArray[np.float_], title_extra: str = \"\"\n):\n\"\"\"Plots the histogram of the influence that all samples in the training set\n    have over a single sample index, separated by labels.\n    Args:\n       influences: array of influences (training samples x test samples)\n       labels: labels for the training set.\n       title_extra:\n    \"\"\"\n_, ax = plt.subplots()\nunique_labels = np.unique(labels)\nfor label in unique_labels:\nax.hist(influences[labels == label], label=label, alpha=0.7)\nax.set_xlabel(\"Influence values\")\nax.set_ylabel(\"Number of samples\")\nax.set_title(f\"Distribution of influences \" + title_extra)\nax.legend()\nplt.show()\n</code></pre>"},{"location":"api/pydvl/reporting/scores/","title":"Scores","text":""},{"location":"api/pydvl/reporting/scores/#pydvl.reporting.scores.compute_removal_score","title":"<code>compute_removal_score(u, values, percentages, *, remove_best=False, progress=False)</code>","text":"<p>Fits model and computes score on the test set after incrementally removing a percentage of data points from the training set, based on their values.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>values</code> <p>Data values of data instances in the training set.</p> <p> TYPE: <code>ValuationResult</code> </p> <code>percentages</code> <p>Sequence of removal percentages.</p> <p> TYPE: <code>Union[NDArray[float_], Iterable[float]]</code> </p> <code>remove_best</code> <p>If True, removes data points in order of decreasing valuation.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>progress</code> <p>If True, display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>Dict[float, float]</code> <p>Dictionary that maps the percentages to their respective scores.</p> Source code in <code>src/pydvl/reporting/scores.py</code> <pre><code>def compute_removal_score(\nu: Utility,\nvalues: ValuationResult,\npercentages: Union[NDArray[np.float_], Iterable[float]],\n*,\nremove_best: bool = False,\nprogress: bool = False,\n) -&gt; Dict[float, float]:\nr\"\"\"Fits model and computes score on the test set after incrementally removing\n    a percentage of data points from the training set, based on their values.\n    Args:\n        u: Utility object with model, data, and scoring function.\n        values: Data values of data instances in the training set.\n        percentages: Sequence of removal percentages.\n        remove_best: If True, removes data points in order of decreasing valuation.\n        progress: If True, display a progress bar.\n    Returns:\n        Dictionary that maps the percentages to their respective scores.\n    \"\"\"\n# Sanity checks\nif np.any([x &gt;= 1.0 or x &lt; 0.0 for x in percentages]):\nraise ValueError(\"All percentages should be in the range [0.0, 1.0)\")\nif len(values) != len(u.data.indices):\nraise ValueError(\nf\"The number of values, {len(values) }, should be equal to the number of data indices, {len(u.data.indices)}\"\n)\nscores = {}\n# We sort in descending order if we want to remove the best values\nvalues.sort(reverse=remove_best)\nfor pct in maybe_progress(percentages, display=progress, desc=\"Removal Scores\"):\nn_removal = int(pct * len(u.data))\nindices = values.indices[n_removal:]\nscore = u(indices)\nscores[pct] = score\nreturn scores\n</code></pre>"},{"location":"api/pydvl/utils/","title":"Utils","text":""},{"location":"api/pydvl/utils/caching/","title":"Caching","text":"<p>Distributed caching of functions.</p> <p>pyDVL uses memcached to cache utility values, through pymemcache. This allows sharing evaluations across processes and nodes in a cluster. You can run memcached as a service, locally or remotely, see Setting up the cache</p> <p>Warning</p> <p>Function evaluations are cached with a key based on the function's signature and code. This can lead to undesired cache hits, see Cache reuse.</p> <p>Remember not to reuse utility objects for different datasets.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--configuration","title":"Configuration","text":"<p>Memoization is disabled by default but can be enabled easily, see Setting up the cache. When enabled, it will be added to any callable used to construct a Utility (done with the decorator @memcached). Depending on the nature of the utility you might want to enable the computation of a running average of function values, see Usage with stochastic functions. You can see all configuration options under MemcachedConfig.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--default-configuration","title":"Default configuration","text":"<pre><code>default_config = dict(\nserver=('localhost', 11211),\nconnect_timeout=1.0,\ntimeout=0.1,\n# IMPORTANT! Disable small packet consolidation:\nno_delay=True,\nserde=serde.PickleSerde(pickle_version=PICKLE_VERSION)\n)\n</code></pre>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--usage-with-stochastic-functions","title":"Usage with stochastic functions","text":"<p>In addition to standard memoization, the decorator memcached() can compute running average and standard error of repeated evaluations for the same input. This can be useful for stochastic functions with high variance (e.g. model training for small sample sizes), but drastically reduces the speed benefits of memoization.</p> <p>This behaviour can be activated with the argument <code>allow_repeated_evaluations</code> to memcached().</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--cache-reuse","title":"Cache reuse","text":"<p>When working directly with memcached(),  it is essential to only cache pure functions. If they have any kind of state, either internal or external (e.g. a closure over some data that may change), then the cache will fail to notice this and the same value will be returned.</p> <p>When a function is wrapped with memcached() for memoization, its signature (input and output names) and code are used as a key for the cache. Alternatively you can pass a custom value to be used as key with</p> <pre><code>cached_fun = memcached(**asdict(cache_options))(fun, signature=custom_signature)\n</code></pre> <p>If you are running experiments with the same Utility but different datasets, this will lead to evaluations of the utility on new data returning old values because utilities only use sample indices as arguments (so there is no way to tell the difference between '1' for dataset A and '1' for dataset 2 from the point of view of the cache). One solution is to empty the cache between runs, but the preferred one is to use a different Utility object for each dataset.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching--unexpected-cache-misses","title":"Unexpected cache misses","text":"<p>Because all arguments to a function are used as part of the key for the cache, sometimes one must exclude some of them. For example, If a function is going to run across multiple processes and some reporting arguments are added (like a <code>job_id</code> for logging purposes), these will be part of the signature and make the functions distinct to the eyes of the cache. This can be avoided with the use of ignore_args in the configuration.</p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching.CacheStats","title":"<code>CacheStats</code>  <code>dataclass</code>","text":"<p>Statistics gathered by cached functions.</p> ATTRIBUTE DESCRIPTION <code>sets</code> <p>number of times a value was set in the cache</p> <p> TYPE: <code>int</code> </p> <code>misses</code> <p>number of times a value was not found in the cache</p> <p> TYPE: <code>int</code> </p> <code>hits</code> <p>number of times a value was found in the cache</p> <p> TYPE: <code>int</code> </p> <code>timeouts</code> <p>number of times a timeout occurred</p> <p> TYPE: <code>int</code> </p> <code>errors</code> <p>number of times an error occurred</p> <p> TYPE: <code>int</code> </p> <code>reconnects</code> <p>number of times the client reconnected to the server</p> <p> TYPE: <code>int</code> </p>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching.serialize","title":"<code>serialize(x)</code>","text":"<p>Serialize an object to bytes. Args:     x: object to serialize.</p> RETURNS DESCRIPTION <code>bytes</code> <p>serialized object.</p> Source code in <code>src/pydvl/utils/caching.py</code> <pre><code>def serialize(x: Any) -&gt; bytes:\n\"\"\"Serialize an object to bytes.\n    Args:\n        x: object to serialize.\n    Returns:\n        serialized object.\n    \"\"\"\npickled_output = BytesIO()\npickler = Pickler(pickled_output, PICKLE_VERSION)\npickler.dump(x)\nreturn pickled_output.getvalue()\n</code></pre>"},{"location":"api/pydvl/utils/caching/#pydvl.utils.caching.memcached","title":"<code>memcached(client_config=None, time_threshold=0.3, allow_repeated_evaluations=False, rtol_stderr=0.1, min_repetitions=3, ignore_args=None)</code>","text":"<p>Transparent, distributed memoization of function calls.</p> <p>Given a function and its signature, memcached uses a distributed cache that, for each set of inputs, keeps track of the average returned value, with variance and number of times it was calculated.</p> <p>If the function is deterministic, i.e. same input corresponds to the same exact output, set <code>allow_repeated_evaluations</code> to <code>False</code>. If instead the function is stochastic (like the training of a model depending on random initializations), memcached() allows to set a minimum number of evaluations to compute a running average, and a tolerance after which the function will not be called anymore. In other words, the function will be recomputed until the value has stabilized with a standard error smaller than <code>rtol_stderr * running average</code>.</p> <p>Warning</p> <p>Do not cache functions with state! See Cache reuse</p> Example <pre><code>cached_fun = memcached(**asdict(cache_options))(heavy_computation)\n</code></pre> PARAMETER  DESCRIPTION <code>client_config</code> <p>configuration for pymemcache's Client. Will be merged on top of the default configuration (see below).</p> <p> TYPE: <code>Optional[MemcachedClientConfig]</code> DEFAULT: <code>None</code> </p> <code>time_threshold</code> <p>computations taking less time than this many seconds are not cached.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.3</code> </p> <code>allow_repeated_evaluations</code> <p>If <code>True</code>, repeated calls to a function with the same arguments will be allowed and outputs averaged until the running standard deviation of the mean stabilizes below <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>rtol_stderr</code> <p>relative tolerance for repeated evaluations. More precisely, memcached() will stop evaluating the function once the standard deviation of the mean is smaller than <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.1</code> </p> <code>min_repetitions</code> <p>minimum number of times that a function evaluation on the same arguments is repeated before returning cached values. Useful for stochastic functions only. If the model training is very noisy, set this number to higher values to reduce variance.</p> <p> TYPE: <code>int</code> DEFAULT: <code>3</code> </p> <code>ignore_args</code> <p>Do not take these keyword arguments into account when hashing the wrapped function for usage as key in memcached. This allows sharing the cache among different jobs for the same experiment run if the callable happens to have \"nuisance\" parameters like <code>job_id</code> which do not affect the result of the computation.</p> <p> TYPE: <code>Optional[Iterable[str]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Callable[[Callable[..., T], bytes | None], Callable[..., T]]</code> <p>A wrapped function</p> Source code in <code>src/pydvl/utils/caching.py</code> <pre><code>def memcached(\nclient_config: Optional[MemcachedClientConfig] = None,\ntime_threshold: float = 0.3,\nallow_repeated_evaluations: bool = False,\nrtol_stderr: float = 0.1,\nmin_repetitions: int = 3,\nignore_args: Optional[Iterable[str]] = None,\n) -&gt; Callable[[Callable[..., T], bytes | None], Callable[..., T]]:\n\"\"\"\n    Transparent, distributed memoization of function calls.\n    Given a function and its signature, memcached uses a distributed cache\n    that, for each set of inputs, keeps track of the average returned value,\n    with variance and number of times it was calculated.\n    If the function is deterministic, i.e. same input corresponds to the same\n    exact output, set `allow_repeated_evaluations` to `False`. If instead the\n    function is stochastic (like the training of a model depending on random\n    initializations), memcached() allows to set a minimum number of evaluations\n    to compute a running average, and a tolerance after which the function will\n    not be called anymore. In other words, the function will be recomputed\n    until the value has stabilized with a standard error smaller than\n    `rtol_stderr * running average`.\n    !!! Warning\n        Do not cache functions with state! See [Cache reuse](cache-reuse)\n    ??? Example\n        ```python\n        cached_fun = memcached(**asdict(cache_options))(heavy_computation)\n        ```\n    Args:\n        client_config: configuration for pymemcache's\n            [Client][pymemcache.client.base.Client].\n            Will be merged on top of the default configuration (see below).\n        time_threshold: computations taking less time than this many seconds are\n            not cached.\n        allow_repeated_evaluations: If `True`, repeated calls to a function\n            with the same arguments will be allowed and outputs averaged until the\n            running standard deviation of the mean stabilizes below\n            `rtol_stderr * mean`.\n        rtol_stderr: relative tolerance for repeated evaluations. More precisely,\n            [memcached()][pydvl.utils.caching.memcached] will stop evaluating the function once the\n            standard deviation of the mean is smaller than `rtol_stderr * mean`.\n        min_repetitions: minimum number of times that a function evaluation\n            on the same arguments is repeated before returning cached values. Useful\n            for stochastic functions only. If the model training is very noisy, set\n            this number to higher values to reduce variance.\n        ignore_args: Do not take these keyword arguments into account when\n            hashing the wrapped function for usage as key in memcached. This allows\n            sharing the cache among different jobs for the same experiment run if\n            the callable happens to have \"nuisance\" parameters like `job_id` which\n            do not affect the result of the computation.\n    Returns:\n        A wrapped function\n    \"\"\"\nif ignore_args is None:\nignore_args = []\n# Do I really need this?\ndef connect(config: MemcachedClientConfig):\n\"\"\"First tries to establish a connection, then tries setting and\n        getting a value.\"\"\"\ntry:\nclient = RetryingClient(\nClient(**asdict(config)),\nattempts=3,\nretry_delay=0.1,\nretry_for=[MemcacheUnexpectedCloseError],\n)\ntemp_key = str(uuid.uuid4())\nclient.set(temp_key, 7)\nassert client.get(temp_key) == 7\nclient.delete(temp_key, 0)\nreturn client\nexcept ConnectionRefusedError as e:\nlogger.error(  # type: ignore\nf\"@memcached: Timeout connecting \"\nf\"to {config.server} after \"\nf\"{config.connect_timeout} seconds: {str(e)}. Did you start memcached?\"\n)\nraise e\nexcept AssertionError as e:\nlogger.error(  # type: ignore\nf\"@memcached: Failure saving dummy value \"\nf\"to {config.server}: {str(e)}\"\n)\ndef wrapper(fun: Callable[..., T], signature: Optional[bytes] = None):\nif signature is None:\nsignature = serialize((fun.__code__.co_code, fun.__code__.co_consts))\n@wraps(fun, updated=[])  # don't try to use update() for a class\nclass Wrapped:\nconfig: MemcachedClientConfig\nstats: CacheStats\nclient: RetryingClient\ndef __init__(self, config: MemcachedClientConfig):\nself.config = config\nself.stats = CacheStats()\nself.client = connect(self.config)\nself._signature = signature\ndef __call__(self, *args, **kwargs) -&gt; T:\nkey_kwargs = {k: v for k, v in kwargs.items() if k not in ignore_args}  # type: ignore\narg_signature: bytes = serialize((args, list(key_kwargs.items())))\nkey = blake2b(self._signature + arg_signature).hexdigest().encode(\"ASCII\")  # type: ignore\nresult_dict: Dict[str, float] = self.get_key_value(key)\nif result_dict is None:\nresult_dict = {}\nstart = time()\nvalue = fun(*args, **kwargs)\nend = time()\nresult_dict[\"value\"] = value\nif end - start &gt;= time_threshold or allow_repeated_evaluations:\nresult_dict[\"count\"] = 1\nresult_dict[\"variance\"] = 0\nself.client.set(key, result_dict, noreply=True)\nself.stats.sets += 1\nself.stats.misses += 1\nelif allow_repeated_evaluations:\nself.stats.hits += 1\nvalue = result_dict[\"value\"]\ncount = result_dict[\"count\"]\nvariance = result_dict[\"variance\"]\nerror_on_average = (variance / count) ** (1 / 2)\nif (\nerror_on_average &gt; rtol_stderr * value\nor count &lt;= min_repetitions\n):\nnew_value = fun(*args, **kwargs)\nnew_avg, new_var = running_moments(\nvalue, variance, int(count), cast(float, new_value)\n)\nresult_dict[\"value\"] = new_avg\nresult_dict[\"count\"] = count + 1\nresult_dict[\"variance\"] = new_var\nself.client.set(key, result_dict, noreply=True)\nself.stats.sets += 1\nelse:\nself.stats.hits += 1\nreturn result_dict[\"value\"]  # type: ignore\ndef __getstate__(self):\n\"\"\"Enables pickling after a socket has been opened to the\n                memcached server, by removing the client from the stored\n                data.\"\"\"\nodict = self.__dict__.copy()\ndel odict[\"client\"]\nreturn odict\ndef __setstate__(self, d: dict):\n\"\"\"Restores a client connection after loading from a pickle.\"\"\"\nself.config = d[\"config\"]\nself.stats = d[\"stats\"]\nself.client = Client(**asdict(self.config))\nself._signature = signature\ndef get_key_value(self, key: bytes):\nresult = None\ntry:\nresult = self.client.get(key)\nexcept socket.timeout as e:\nself.stats.timeouts += 1\nwarnings.warn(f\"{type(self).__name__}: {str(e)}\", RuntimeWarning)\nexcept OSError as e:\nself.stats.errors += 1\nwarnings.warn(f\"{type(self).__name__}: {str(e)}\", RuntimeWarning)\nexcept AttributeError as e:\n# FIXME: this depends on _recv() failing on invalid sockets\n# See pymemcache.base.py,\nself.stats.reconnects += 1\nwarnings.warn(f\"{type(self).__name__}: {str(e)}\", RuntimeWarning)\nself.client = connect(self.config)\nreturn result\nWrapped.__doc__ = (\nf\"A wrapper around {fun.__name__}() with remote caching enabled.\\n\"\n+ (Wrapped.__doc__ or \"\")\n)\nWrapped.__name__ = f\"memcached_{fun.__name__}\"\npath = list(reversed(fun.__qualname__.split(\".\")))\npatched = [f\"memcached_{path[0]}\"] + path[1:]\nWrapped.__qualname__ = \".\".join(reversed(patched))\n# TODO: pick from some config file or something\nreturn Wrapped(client_config or MemcachedClientConfig())\nreturn wrapper\n</code></pre>"},{"location":"api/pydvl/utils/config/","title":"Config","text":""},{"location":"api/pydvl/utils/config/#pydvl.utils.config.ParallelConfig","title":"<code>ParallelConfig</code>  <code>dataclass</code>","text":"<p>Configuration for parallel computation backend.</p> PARAMETER  DESCRIPTION <code>backend</code> <p>Type of backend to use. Defaults to 'joblib'</p> <p> TYPE: <code>Literal['joblib', 'ray']</code> DEFAULT: <code>'joblib'</code> </p> <code>address</code> <p>Address of existing remote or local cluster to use.</p> <p> TYPE: <code>Optional[Union[str, Tuple[str, int]]]</code> DEFAULT: <code>None</code> </p> <code>n_cpus_local</code> <p>Number of CPUs to use when creating a local ray cluster. This has no effect when using an existing ray cluster.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>logging_level</code> <p>Logging level for the parallel backend's worker.</p> <p> TYPE: <code>int</code> DEFAULT: <code>WARNING</code> </p> <code>wait_timeout</code> <p>Timeout in seconds for waiting on futures.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p>"},{"location":"api/pydvl/utils/config/#pydvl.utils.config.MemcachedClientConfig","title":"<code>MemcachedClientConfig</code>  <code>dataclass</code>","text":"<p>Configuration of the memcached client.</p> PARAMETER  DESCRIPTION <code>server</code> <p>A tuple of (IP|domain name, port).</p> <p> TYPE: <code>Tuple[str, int]</code> DEFAULT: <code>('localhost', 11211)</code> </p> <code>connect_timeout</code> <p>How many seconds to wait before raising <code>ConnectionRefusedError</code> on failure to connect.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>timeout</code> <p>seconds to wait for send or recv calls on the socket connected to memcached.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>no_delay</code> <p>set the <code>TCP_NODELAY</code> flag, which may help with performance in some cases.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>serde</code> <p>a serializer / deserializer (\"serde\"). The default <code>PickleSerde</code> should work in most cases. See pymemcached's documentation for details.</p> <p> TYPE: <code>PickleSerde</code> DEFAULT: <code>PickleSerde(pickle_version=PICKLE_VERSION)</code> </p>"},{"location":"api/pydvl/utils/config/#pydvl.utils.config.MemcachedConfig","title":"<code>MemcachedConfig</code>  <code>dataclass</code>","text":"<p>Configuration for memcached(), providing memoization of function calls.</p> <p>Instances of this class are typically used as arguments for the construction of a Utility.</p> PARAMETER  DESCRIPTION <code>client_config</code> <p>Configuration for the connection to the memcached server.</p> <p> TYPE: <code>MemcachedClientConfig</code> DEFAULT: <code>field(default_factory=MemcachedClientConfig)</code> </p> <code>time_threshold</code> <p>computations taking less time than this many seconds are not cached.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.3</code> </p> <code>allow_repeated_evaluations</code> <p>If <code>True</code>, repeated calls to a function with the same arguments will be allowed and outputs averaged until the running standard deviation of the mean stabilises below <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>rtol_stderr</code> <p>relative tolerance for repeated evaluations. More precisely, memcached() will stop evaluating the function once the standard deviation of the mean is smaller than <code>rtol_stderr * mean</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.1</code> </p> <code>min_repetitions</code> <p>minimum number of times that a function evaluation on the same arguments is repeated before returning cached values. Useful for stochastic functions only. If the model training is very noisy, set this number to higher values to reduce variance.</p> <p> TYPE: <code>int</code> DEFAULT: <code>3</code> </p> <code>ignore_args</code> <p>Do not take these keyword arguments into account when hashing the wrapped function for usage as key in memcached.</p> <p> TYPE: <code>Optional[Iterable[str]]</code> DEFAULT: <code>None</code> </p>"},{"location":"api/pydvl/utils/dataset/","title":"Dataset","text":"<p>This module contains convenience classes to handle data and groups thereof.</p> <p>Shapley and Least Core value computations require evaluation of a scoring function (the utility). This is typically the performance of the model on a test set (as an approximation to its true expected performance). It is therefore convenient to keep both the training data and the test data together to be passed around to methods in shapley and least_core. This is done with Dataset.</p> <p>This abstraction layer also seamlessly grouping data points together if one is interested in computing their value as a group, see GroupedDataset.</p> <p>Objects of both types are used to construct a Utility object.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset","title":"<code>Dataset(x_train, y_train, x_test, y_test, feature_names=None, target_names=None, data_names=None, description=None, is_multi_output=False)</code>","text":"<p>A convenience class to handle datasets.</p> <p>It holds a dataset, split into training and test data, together with several labels on feature names, data point names and a description.</p> PARAMETER  DESCRIPTION <code>x_train</code> <p>training data</p> <p> TYPE: <code>Union[NDArray, DataFrame]</code> </p> <code>y_train</code> <p>labels for training data</p> <p> TYPE: <code>Union[NDArray, DataFrame]</code> </p> <code>x_test</code> <p>test data</p> <p> TYPE: <code>Union[NDArray, DataFrame]</code> </p> <code>y_test</code> <p>labels for test data</p> <p> TYPE: <code>Union[NDArray, DataFrame]</code> </p> <code>feature_names</code> <p>name of the features of input data</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>target_names</code> <p>names of the features of target data</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>data_names</code> <p>names assigned to data points. For example, if the dataset is a time series, each entry can be a timestamp which can be referenced directly instead of using a row number.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>description</code> <p>A textual description of the dataset.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>is_multi_output</code> <p>set to <code>False</code> if labels are scalars, or to <code>True</code> if they are vectors of dimension &gt; 1.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def __init__(\nself,\nx_train: Union[NDArray, pd.DataFrame],\ny_train: Union[NDArray, pd.DataFrame],\nx_test: Union[NDArray, pd.DataFrame],\ny_test: Union[NDArray, pd.DataFrame],\nfeature_names: Optional[Sequence[str]] = None,\ntarget_names: Optional[Sequence[str]] = None,\ndata_names: Optional[Sequence[str]] = None,\ndescription: Optional[str] = None,\n# FIXME: use same parameter name as in check_X_y()\nis_multi_output: bool = False,\n):\n\"\"\"Constructs a Dataset from data and labels.\n    Args:\n        x_train: training data\n        y_train: labels for training data\n        x_test: test data\n        y_test: labels for test data\n        feature_names: name of the features of input data\n        target_names: names of the features of target data\n        data_names: names assigned to data points.\n            For example, if the dataset is a time series, each entry can be a\n            timestamp which can be referenced directly instead of using a row\n            number.\n        description: A textual description of the dataset.\n        is_multi_output: set to `False` if labels are scalars, or to\n            `True` if they are vectors of dimension &gt; 1.\n    \"\"\"\nself.x_train, self.y_train = check_X_y(\nx_train, y_train, multi_output=is_multi_output\n)\nself.x_test, self.y_test = check_X_y(\nx_test, y_test, multi_output=is_multi_output\n)\nif x_train.shape[-1] != x_test.shape[-1]:\nraise ValueError(\nf\"Mismatching number of features: \"\nf\"{x_train.shape[-1]} and {x_test.shape[-1]}\"\n)\nif x_train.shape[0] != y_train.shape[0]:\nraise ValueError(\nf\"Mismatching number of samples: \"\nf\"{x_train.shape[-1]} and {x_test.shape[-1]}\"\n)\nif x_test.shape[0] != y_test.shape[0]:\nraise ValueError(\nf\"Mismatching number of samples: \"\nf\"{x_test.shape[-1]} and {y_test.shape[-1]}\"\n)\ndef make_names(s: str, a: np.ndarray) -&gt; List[str]:\nn = a.shape[1] if len(a.shape) &gt; 1 else 1\nreturn [f\"{s}{i:0{1 + int(np.log10(n))}d}\" for i in range(1, n + 1)]\nself.feature_names = feature_names\nself.target_names = target_names\nif self.feature_names is None:\nif isinstance(x_train, pd.DataFrame):\nself.feature_names = x_train.columns.tolist()\nelse:\nself.feature_names = make_names(\"x\", x_train)\nif self.target_names is None:\nif isinstance(y_train, pd.DataFrame):\nself.target_names = y_train.columns.tolist()\nelse:\nself.target_names = make_names(\"y\", y_train)\nif len(self.x_train.shape) &gt; 1:\nif (\nlen(self.feature_names) != self.x_train.shape[-1]\nor len(self.feature_names) != self.x_test.shape[-1]\n):\nraise ValueError(\"Mismatching number of features and names\")\nif len(self.y_train.shape) &gt; 1:\nif (\nlen(self.target_names) != self.y_train.shape[-1]\nor len(self.target_names) != self.y_test.shape[-1]\n):\nraise ValueError(\"Mismatching number of targets and names\")\nself.description = description or \"No description\"\nself._indices = np.arange(len(self.x_train), dtype=np.int_)\nself._data_names = (\nnp.array(data_names, dtype=object)\nif data_names is not None\nelse self._indices.astype(object)\n)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.indices","title":"<code>indices: NDArray[np.int_]</code>  <code>property</code>","text":"<p>Index of positions in data.x_train.</p> <p>Contiguous integers from 0 to len(Dataset).</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.data_names","title":"<code>data_names: NDArray[np.object_]</code>  <code>property</code>","text":"<p>Names of each individual datapoint.</p> <p>Used for reporting Shapley values.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.dim","title":"<code>dim: int</code>  <code>property</code>","text":"<p>Returns the number of dimensions of a sample.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.get_training_data","title":"<code>get_training_data(indices=None)</code>","text":"<p>Given a set of indices, returns the training data that refer to those indices.</p> <p>This is used mainly by Utility to retrieve subsets of the data from indices. It is typically not needed in algorithms.</p> PARAMETER  DESCRIPTION <code>indices</code> <p>Optional indices that will be used to select points from the training data. If <code>None</code>, the entire training data will be returned.</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>If <code>indices</code> is not <code>None</code>, the selected x and y arrays from the training data. Otherwise, the entire dataset.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_training_data(\nself, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n\"\"\"Given a set of indices, returns the training data that refer to those\n    indices.\n    This is used mainly by [Utility][pydvl.utils.utility.Utility] to retrieve\n    subsets of the data from indices. It is typically **not needed in\n    algorithms**.\n    Args:\n        indices: Optional indices that will be used to select points from\n            the training data. If `None`, the entire training data will be\n            returned.\n    Returns:\n        If `indices` is not `None`, the selected x and y arrays from the\n            training data. Otherwise, the entire dataset.\n    \"\"\"\nif indices is None:\nreturn self.x_train, self.y_train\nx = self.x_train[indices]\ny = self.y_train[indices]\nreturn x, y\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.get_test_data","title":"<code>get_test_data(indices=None)</code>","text":"<p>Returns the entire test set regardless of the passed indices.</p> <p>The passed indices will not be used because for data valuation we generally want to score the trained model on the entire test data.</p> <p>Additionally, the way this method is used in the Utility class, the passed indices will be those of the training data and would not work on the test data.</p> <p>There may be cases where it is desired to use parts of the test data. In those cases, it is recommended to inherit from Dataset and override get_test_data().</p> <p>For example, the following snippet shows how one could go about mapping the training data indices into test data indices inside get_test_data():</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; class DatasetWithTestDataIndices(Dataset):\n...    def get_test_data(self, indices=None):\n...        if indices is None:\n...            return self.x_test, self.y_test\n...        fraction = len(list(indices)) / len(self)\n...        mapped_indices = len(self.x_test) / len(self) * np.asarray(indices)\n...        mapped_indices = np.unique(mapped_indices.astype(int))\n...        return self.x_test[mapped_indices], self.y_test[mapped_indices]\n...\n&gt;&gt;&gt; X = np.random.rand(100, 10)\n&gt;&gt;&gt; y = np.random.randint(0, 2, 100)\n&gt;&gt;&gt; dataset = DatasetWithTestDataIndices.from_arrays(X, y)\n&gt;&gt;&gt; indices = np.random.choice(dataset.indices, 30, replace=False)\n&gt;&gt;&gt; _ = dataset.get_training_data(indices)\n&gt;&gt;&gt; _ = dataset.get_test_data(indices)\n</code></pre> PARAMETER  DESCRIPTION <code>indices</code> <p>Optional indices into the test data. This argument is unused left for compatibility with get_training_data().</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>The entire test data.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_test_data(\nself, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n\"\"\"Returns the entire test set regardless of the passed indices.\n    The passed indices will not be used because for data valuation\n    we generally want to score the trained model on the entire test data.\n    Additionally, the way this method is used in the\n    [Utility][pydvl.utils.utility.Utility] class, the passed indices will\n    be those of the training data and would not work on the test data.\n    There may be cases where it is desired to use parts of the test data.\n    In those cases, it is recommended to inherit from\n    [Dataset][pydvl.utils.dataset.Dataset] and override\n    [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data].\n    For example, the following snippet shows how one could go about\n    mapping the training data indices into test data indices\n    inside [get_test_data()][pydvl.utils.dataset.Dataset.get_test_data]:\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; import numpy as np\n        &gt;&gt;&gt; class DatasetWithTestDataIndices(Dataset):\n        ...    def get_test_data(self, indices=None):\n        ...        if indices is None:\n        ...            return self.x_test, self.y_test\n        ...        fraction = len(list(indices)) / len(self)\n        ...        mapped_indices = len(self.x_test) / len(self) * np.asarray(indices)\n        ...        mapped_indices = np.unique(mapped_indices.astype(int))\n        ...        return self.x_test[mapped_indices], self.y_test[mapped_indices]\n        ...\n        &gt;&gt;&gt; X = np.random.rand(100, 10)\n        &gt;&gt;&gt; y = np.random.randint(0, 2, 100)\n        &gt;&gt;&gt; dataset = DatasetWithTestDataIndices.from_arrays(X, y)\n        &gt;&gt;&gt; indices = np.random.choice(dataset.indices, 30, replace=False)\n        &gt;&gt;&gt; _ = dataset.get_training_data(indices)\n        &gt;&gt;&gt; _ = dataset.get_test_data(indices)\n        ```\n    Args:\n        indices: Optional indices into the test data. This argument is\n            unused left for compatibility with\n            [get_training_data()][pydvl.utils.dataset.Dataset.get_training_data].\n    Returns:\n        The entire test data.\n    \"\"\"\nreturn self.x_test, self.y_test\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.from_sklearn","title":"<code>from_sklearn(data, train_size=0.8, random_state=None, stratify_by_target=False, **kwargs)</code>  <code>classmethod</code>","text":"<p>Constructs a Dataset object from a sklearn.utils.Bunch, as returned by the <code>load_*</code> functions in scikit-learn toy datasets.</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; from sklearn.datasets import load_boston\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_boston())\n</code></pre> PARAMETER  DESCRIPTION <code>data</code> <p>scikit-learn Bunch object. The following attributes are supported:</p> <ul> <li><code>data</code>: covariates.</li> <li><code>target</code>: target variables (labels).</li> <li><code>feature_names</code> (optional): the feature names.</li> <li><code>target_names</code> (optional): the target names.</li> <li><code>DESCR</code> (optional): a description.</li> </ul> <p> TYPE: <code>Bunch</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code></p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the target variable as labels. Read more in scikit-learn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor. Use this to pass e.g. <code>is_multi_output</code>.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Object with the sklearn dataset</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_sklearn(\ncls,\ndata: Bunch,\ntrain_size: float = 0.8,\nrandom_state: Optional[int] = None,\nstratify_by_target: bool = False,\n**kwargs,\n) -&gt; \"Dataset\":\n\"\"\"Constructs a [Dataset][pydvl.utils.Dataset] object from a\n    [sklearn.utils.Bunch][], as returned by the `load_*`\n    functions in [scikit-learn toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html).\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; from sklearn.datasets import load_boston\n        &gt;&gt;&gt; dataset = Dataset.from_sklearn(load_boston())\n        ```\n    Args:\n        data: scikit-learn Bunch object. The following attributes are supported:\n            - `data`: covariates.\n            - `target`: target variables (labels).\n            - `feature_names` (**optional**): the feature names.\n            - `target_names` (**optional**): the target names.\n            - `DESCR` (**optional**): a description.\n        train_size: size of the training dataset. Used in `train_test_split`\n        random_state: seed for train / test split\n        stratify_by_target: If `True`, data is split in a stratified\n            fashion, using the target variable as labels. Read more in\n            [scikit-learn's user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `is_multi_output`.\n    Returns:\n        Object with the sklearn dataset\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\nx_train, x_test, y_train, y_test = train_test_split(\ndata.data,\ndata.target,\ntrain_size=train_size,\nrandom_state=random_state,\nstratify=data.target if stratify_by_target else None,\n)\nreturn cls(\nx_train,\ny_train,\nx_test,\ny_test,\nfeature_names=data.get(\"feature_names\"),\ntarget_names=data.get(\"target_names\"),\ndescription=data.get(\"DESCR\"),\n**kwargs,\n)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.Dataset.from_arrays","title":"<code>from_arrays(X, y, train_size=0.8, random_state=None, stratify_by_target=False, **kwargs)</code>  <code>classmethod</code>","text":"<p>Constructs a Dataset object from X and y numpy arrays  as returned by the <code>make_*</code> functions in sklearn generated datasets.</p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Dataset\n&gt;&gt;&gt; from sklearn.datasets import make_regression\n&gt;&gt;&gt; X, y = make_regression()\n&gt;&gt;&gt; dataset = Dataset.from_arrays(X, y)\n</code></pre> PARAMETER  DESCRIPTION <code>X</code> <p>numpy array of shape (n_samples, n_features)</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>numpy array of shape (n_samples,)</p> <p> TYPE: <code>NDArray</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code></p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the y variable as labels. Read more in sklearn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor. Use this to pass e.g. <code>feature_names</code> or <code>target_names</code>.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Object with the passed X and y arrays split across training and test sets.</p> <p>New in version 0.4.0</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_arrays(\ncls,\nX: NDArray,\ny: NDArray,\ntrain_size: float = 0.8,\nrandom_state: Optional[int] = None,\nstratify_by_target: bool = False,\n**kwargs,\n) -&gt; \"Dataset\":\n\"\"\"Constructs a [Dataset][pydvl.utils.Dataset] object from X and y numpy arrays  as\n    returned by the `make_*` functions in [sklearn generated datasets](https://scikit-learn.org/stable/datasets/sample_generators.html).\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from pydvl.utils import Dataset\n        &gt;&gt;&gt; from sklearn.datasets import make_regression\n        &gt;&gt;&gt; X, y = make_regression()\n        &gt;&gt;&gt; dataset = Dataset.from_arrays(X, y)\n        ```\n    Args:\n        X: numpy array of shape (n_samples, n_features)\n        y: numpy array of shape (n_samples,)\n        train_size: size of the training dataset. Used in `train_test_split`\n        random_state: seed for train / test split\n        stratify_by_target: If `True`, data is split in a stratified fashion,\n            using the y variable as labels. Read more in [sklearn's user\n            guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor. Use this to pass e.g. `feature_names`\n            or `target_names`.\n    Returns:\n        Object with the passed X and y arrays split across training and test sets.\n    !!! tip \"New in version 0.4.0\"\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\nx_train, x_test, y_train, y_test = train_test_split(\nX,\ny,\ntrain_size=train_size,\nrandom_state=random_state,\nstratify=y if stratify_by_target else None,\n)\nreturn cls(x_train, y_train, x_test, y_test, **kwargs)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset","title":"<code>GroupedDataset(x_train, y_train, x_test, y_test, data_groups, feature_names=None, target_names=None, group_names=None, description=None, **kwargs)</code>","text":"<p>             Bases: <code>Dataset</code></p> <p>Used for calculating Shapley values of subsets of the data considered as logical units. For instance, one can group by value of a categorical feature, by bin into which a continuous feature falls, or by label.</p> PARAMETER  DESCRIPTION <code>x_train</code> <p>training data</p> <p> TYPE: <code>NDArray</code> </p> <code>y_train</code> <p>labels of training data</p> <p> TYPE: <code>NDArray</code> </p> <code>x_test</code> <p>test data</p> <p> TYPE: <code>NDArray</code> </p> <code>y_test</code> <p>labels of test data</p> <p> TYPE: <code>NDArray</code> </p> <code>data_groups</code> <p>Iterable of the same length as <code>x_train</code> containing a group label for each training data point. The label can be of any type, e.g. <code>str</code> or <code>int</code>. Data points with the same label will then be grouped by this object and considered as one for effects of valuation.</p> <p> TYPE: <code>Sequence</code> </p> <code>feature_names</code> <p>names of the covariates' features.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>target_names</code> <p>names of the labels or targets y</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>group_names</code> <p>names of the groups. If not provided, the labels from <code>data_groups</code> will be used.</p> <p> TYPE: <code>Optional[Sequence[str]]</code> DEFAULT: <code>None</code> </p> <code>description</code> <p>A textual description of the dataset</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor.</p> <p> DEFAULT: <code>{}</code> </p> <p>Changed in version 0.6.0</p> <p>Added <code>group_names</code> and forwarding of <code>kwargs</code></p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def __init__(\nself,\nx_train: NDArray,\ny_train: NDArray,\nx_test: NDArray,\ny_test: NDArray,\ndata_groups: Sequence,\nfeature_names: Optional[Sequence[str]] = None,\ntarget_names: Optional[Sequence[str]] = None,\ngroup_names: Optional[Sequence[str]] = None,\ndescription: Optional[str] = None,\n**kwargs,\n):\n\"\"\"Class for grouping datasets.\n    Used for calculating Shapley values of subsets of the data considered\n    as logical units. For instance, one can group by value of a categorical\n    feature, by bin into which a continuous feature falls, or by label.\n    Args:\n        x_train: training data\n        y_train: labels of training data\n        x_test: test data\n        y_test: labels of test data\n        data_groups: Iterable of the same length as `x_train` containing\n            a group label for each training data point. The label can be of any\n            type, e.g. `str` or `int`. Data points with the same label will\n            then be grouped by this object and considered as one for effects of\n            valuation.\n        feature_names: names of the covariates' features.\n        target_names: names of the labels or targets y\n        group_names: names of the groups. If not provided, the labels\n            from `data_groups` will be used.\n        description: A textual description of the dataset\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor.\n    !!! tip \"Changed in version 0.6.0\"\n    Added `group_names` and forwarding of `kwargs`\n    \"\"\"\nsuper().__init__(\nx_train=x_train,\ny_train=y_train,\nx_test=x_test,\ny_test=y_test,\nfeature_names=feature_names,\ntarget_names=target_names,\ndescription=description,\n**kwargs,\n)\nif len(data_groups) != len(x_train):\nraise ValueError(\nf\"data_groups and x_train must have the same length.\"\nf\"Instead got {len(data_groups)=} and {len(x_train)=}\"\n)\nself.groups: OrderedDict[Any, List[int]] = OrderedDict(\n{k: [] for k in set(data_groups)}\n)\nfor idx, group in enumerate(data_groups):\nself.groups[group].append(idx)\nself.group_items = list(self.groups.items())\nself._indices = np.arange(len(self.groups.keys()))\nself._data_names = (\nnp.array(group_names, dtype=object)\nif group_names is not None\nelse np.array(list(self.groups.keys()), dtype=object)\n)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.indices","title":"<code>indices</code>  <code>property</code>","text":"<p>Indices of the groups.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.data_names","title":"<code>data_names</code>  <code>property</code>","text":"<p>Names of the groups.</p>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.get_training_data","title":"<code>get_training_data(indices=None)</code>","text":"<p>Returns the data and labels of all samples in the given groups.</p> PARAMETER  DESCRIPTION <code>indices</code> <p>group indices whose elements to return. If <code>None</code>, all data from all groups are returned.</p> <p> TYPE: <code>Optional[Iterable[int]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Tuple[NDArray, NDArray]</code> <p>Tuple of training data x and labels y.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>def get_training_data(\nself, indices: Optional[Iterable[int]] = None\n) -&gt; Tuple[NDArray, NDArray]:\n\"\"\"Returns the data and labels of all samples in the given groups.\n    Args:\n        indices: group indices whose elements to return. If `None`,\n            all data from all groups are returned.\n    Returns:\n        Tuple of training data x and labels y.\n    \"\"\"\nif indices is None:\nindices = self.indices\ndata_indices = [\nidx for group_id in indices for idx in self.group_items[group_id][1]\n]\nreturn super().get_training_data(data_indices)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.from_sklearn","title":"<code>from_sklearn(data, train_size=0.8, random_state=None, stratify_by_target=False, data_groups=None, **kwargs)</code>  <code>classmethod</code>","text":"<p>Constructs a GroupedDataset object from a sklearn.utils.Bunch as returned by the <code>load_*</code> functions in scikit-learn toy datasets and groups it.</p> Example <pre><code>&gt;&gt;&gt; from sklearn.datasets import load_iris\n&gt;&gt;&gt; from pydvl.utils import GroupedDataset\n&gt;&gt;&gt; iris = load_iris()\n&gt;&gt;&gt; data_groups = iris.data[:, 0] // 0.5\n&gt;&gt;&gt; dataset = GroupedDataset.from_sklearn(iris, data_groups=data_groups)\n</code></pre> PARAMETER  DESCRIPTION <code>data</code> <p>scikit-learn Bunch object. The following attributes are supported:</p> <ul> <li><code>data</code>: covariates.</li> <li><code>target</code>: target variables (labels).</li> <li><code>feature_names</code> (optional): the feature names.</li> <li><code>target_names</code> (optional): the target names.</li> <li><code>DESCR</code> (optional): a description.</li> </ul> <p> TYPE: <code>Bunch</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the target variable as labels. Read more in sklearn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>data_groups</code> <p>an array holding the group index or name for each data point. The length of this array must be equal to the number of data points in the dataset.</p> <p> TYPE: <code>Optional[Sequence]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments to pass to the Dataset constructor.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>GroupedDataset</code> <p>Dataset with the selected sklearn data</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_sklearn(\ncls,\ndata: Bunch,\ntrain_size: float = 0.8,\nrandom_state: Optional[int] = None,\nstratify_by_target: bool = False,\ndata_groups: Optional[Sequence] = None,\n**kwargs,\n) -&gt; \"GroupedDataset\":\n\"\"\"Constructs a [GroupedDataset][pydvl.utils.GroupedDataset] object from a\n    [sklearn.utils.Bunch][sklearn.utils.Bunch] as returned by the `load_*` functions in\n    [scikit-learn toy datasets](https://scikit-learn.org/stable/datasets/toy_dataset.html) and groups\n    it.\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from sklearn.datasets import load_iris\n        &gt;&gt;&gt; from pydvl.utils import GroupedDataset\n        &gt;&gt;&gt; iris = load_iris()\n        &gt;&gt;&gt; data_groups = iris.data[:, 0] // 0.5\n        &gt;&gt;&gt; dataset = GroupedDataset.from_sklearn(iris, data_groups=data_groups)\n        ```\n    Args:\n        data: scikit-learn Bunch object. The following attributes are supported:\n            - `data`: covariates.\n            - `target`: target variables (labels).\n            - `feature_names` (**optional**): the feature names.\n            - `target_names` (**optional**): the target names.\n            - `DESCR` (**optional**): a description.\n        train_size: size of the training dataset. Used in `train_test_split`.\n        random_state: seed for train / test split.\n        stratify_by_target: If `True`, data is split in a stratified\n            fashion, using the target variable as labels. Read more in\n            [sklearn's user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        data_groups: an array holding the group index or name for each\n            data point. The length of this array must be equal to the number of\n            data points in the dataset.\n        kwargs: Additional keyword arguments to pass to the\n            [Dataset][pydvl.utils.Dataset] constructor.\n    Returns:\n        Dataset with the selected sklearn data\n    \"\"\"\nif data_groups is None:\nraise ValueError(\n\"data_groups must be provided when constructing a GroupedDataset\"\n)\nx_train, x_test, y_train, y_test, data_groups_train, _ = train_test_split(\ndata.data,\ndata.target,\ndata_groups,\ntrain_size=train_size,\nrandom_state=random_state,\nstratify=data.target if stratify_by_target else None,\n)\ndataset = Dataset(\nx_train=x_train, y_train=y_train, x_test=x_test, y_test=y_test, **kwargs\n)\nreturn cls.from_dataset(dataset, data_groups_train)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.from_arrays","title":"<code>from_arrays(X, y, train_size=0.8, random_state=None, stratify_by_target=False, data_groups=None, **kwargs)</code>  <code>classmethod</code>","text":"<p>Constructs a GroupedDataset object from X and y numpy arrays as returned by the <code>make_*</code> functions in scikit-learn generated datasets.</p> Example <pre><code>&gt;&gt;&gt; from sklearn.datasets import make_classification\n&gt;&gt;&gt; from pydvl.utils import GroupedDataset\n&gt;&gt;&gt; X, y = make_classification(\n...     n_samples=100,\n...     n_features=4,\n...     n_informative=2,\n...     n_redundant=0,\n...     random_state=0,\n...     shuffle=False\n... )\n&gt;&gt;&gt; data_groups = X[:, 0] // 0.5\n&gt;&gt;&gt; dataset = GroupedDataset.from_arrays(X, y, data_groups=data_groups)\n</code></pre> PARAMETER  DESCRIPTION <code>X</code> <p>array of shape (n_samples, n_features)</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>array of shape (n_samples,)</p> <p> TYPE: <code>NDArray</code> </p> <code>train_size</code> <p>size of the training dataset. Used in <code>train_test_split</code>.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>random_state</code> <p>seed for train / test split.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>stratify_by_target</code> <p>If <code>True</code>, data is split in a stratified fashion, using the y variable as labels. Read more in sklearn's user guide.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>data_groups</code> <p>an array holding the group index or name for each data point. The length of this array must be equal to the number of data points in the dataset.</p> <p> TYPE: <code>Optional[Sequence]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments that will be passed to the Dataset constructor.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>Dataset</code> <p>Dataset with the passed X and y arrays split across training and test sets.</p> <p>New in version 0.4.0</p> <p>Changed in version 0.6.0</p> <p>Added kwargs to pass to the Dataset constructor.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_arrays(\ncls,\nX: NDArray,\ny: NDArray,\ntrain_size: float = 0.8,\nrandom_state: Optional[int] = None,\nstratify_by_target: bool = False,\ndata_groups: Optional[Sequence] = None,\n**kwargs,\n) -&gt; \"Dataset\":\n\"\"\"Constructs a [GroupedDataset][pydvl.utils.GroupedDataset] object from X and y numpy arrays\n    as returned by the `make_*` functions in\n    [scikit-learn generated datasets](https://scikit-learn.org/stable/datasets/sample_generators.html).\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; from sklearn.datasets import make_classification\n        &gt;&gt;&gt; from pydvl.utils import GroupedDataset\n        &gt;&gt;&gt; X, y = make_classification(\n        ...     n_samples=100,\n        ...     n_features=4,\n        ...     n_informative=2,\n        ...     n_redundant=0,\n        ...     random_state=0,\n        ...     shuffle=False\n        ... )\n        &gt;&gt;&gt; data_groups = X[:, 0] // 0.5\n        &gt;&gt;&gt; dataset = GroupedDataset.from_arrays(X, y, data_groups=data_groups)\n        ```\n    Args:\n        X: array of shape (n_samples, n_features)\n        y: array of shape (n_samples,)\n        train_size: size of the training dataset. Used in `train_test_split`.\n        random_state: seed for train / test split.\n        stratify_by_target: If `True`, data is split in a stratified\n            fashion, using the y variable as labels. Read more in\n            [sklearn's user guide](https://scikit-learn.org/stable/modules/cross_validation.html#stratification).\n        data_groups: an array holding the group index or name for each data\n            point. The length of this array must be equal to the number of\n            data points in the dataset.\n        kwargs: Additional keyword arguments that will be passed to the\n            [Dataset][pydvl.utils.Dataset] constructor.\n    Returns:\n        Dataset with the passed X and y arrays split across training and\n            test sets.\n    !!! tip \"New in version 0.4.0\"\n    !!! tip \"Changed in version 0.6.0\"\n        Added kwargs to pass to the [Dataset][pydvl.utils.Dataset] constructor.\n    \"\"\"\nif data_groups is None:\nraise ValueError(\n\"data_groups must be provided when constructing a GroupedDataset\"\n)\nx_train, x_test, y_train, y_test, data_groups_train, _ = train_test_split(\nX,\ny,\ndata_groups,\ntrain_size=train_size,\nrandom_state=random_state,\nstratify=y if stratify_by_target else None,\n)\ndataset = Dataset(\nx_train=x_train, y_train=y_train, x_test=x_test, y_test=y_test, **kwargs\n)\nreturn cls.from_dataset(dataset, data_groups_train)\n</code></pre>"},{"location":"api/pydvl/utils/dataset/#pydvl.utils.dataset.GroupedDataset.from_dataset","title":"<code>from_dataset(dataset, data_groups)</code>  <code>classmethod</code>","text":"<p>Creates a GroupedDataset object from the data a Dataset object and a mapping of data groups.</p> Example <pre><code>&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pydvl.utils import Dataset, GroupedDataset\n&gt;&gt;&gt; dataset = Dataset.from_arrays(\n...     X=np.asarray([[1, 2], [3, 4], [5, 6], [7, 8]]),\n...     y=np.asarray([0, 1, 0, 1]),\n... )\n&gt;&gt;&gt; dataset = GroupedDataset.from_dataset(dataset, data_groups=[0, 0, 1, 1])\n</code></pre> PARAMETER  DESCRIPTION <code>dataset</code> <p>The original data.</p> <p> TYPE: <code>Dataset</code> </p> <code>data_groups</code> <p>An array holding the group index or name for each data point. The length of this array must be equal to the number of data points in the dataset.</p> <p> TYPE: <code>Sequence[Any]</code> </p> RETURNS DESCRIPTION <code>GroupedDataset</code> <p>A GroupedDataset with the initial Dataset grouped by data_groups.</p> Source code in <code>src/pydvl/utils/dataset.py</code> <pre><code>@classmethod\ndef from_dataset(\ncls, dataset: Dataset, data_groups: Sequence[Any]\n) -&gt; \"GroupedDataset\":\n\"\"\"Creates a [GroupedDataset][pydvl.utils.GroupedDataset] object from the data a\n    [Dataset][pydvl.utils.Dataset] object and a mapping of data groups.\n    ??? Example\n        ```pycon\n        &gt;&gt;&gt; import numpy as np\n        &gt;&gt;&gt; from pydvl.utils import Dataset, GroupedDataset\n        &gt;&gt;&gt; dataset = Dataset.from_arrays(\n        ...     X=np.asarray([[1, 2], [3, 4], [5, 6], [7, 8]]),\n        ...     y=np.asarray([0, 1, 0, 1]),\n        ... )\n        &gt;&gt;&gt; dataset = GroupedDataset.from_dataset(dataset, data_groups=[0, 0, 1, 1])\n        ```\n    Args:\n        dataset: The original data.\n        data_groups: An array holding the group index or name for each data\n            point. The length of this array must be equal to the number of\n            data points in the dataset.\n    Returns:\n        A [GroupedDataset][pydvl.utils.GroupedDataset] with the initial\n            [Dataset][pydvl.utils.Dataset] grouped by data_groups.\n    \"\"\"\nreturn cls(\nx_train=dataset.x_train,\ny_train=dataset.y_train,\nx_test=dataset.x_test,\ny_test=dataset.y_test,\ndata_groups=data_groups,\nfeature_names=dataset.feature_names,\ntarget_names=dataset.target_names,\ndescription=dataset.description,\n)\n</code></pre>"},{"location":"api/pydvl/utils/functional/","title":"Functional","text":"<p>Supporting utilities for manipulating arguments of functions.</p>"},{"location":"api/pydvl/utils/functional/#pydvl.utils.functional.free_arguments","title":"<code>free_arguments(fun)</code>","text":"<p>Computes the set of free arguments for a function or functools.partial object.</p> <p>All arguments of a function are considered free unless they are set by a partial. For example, if <code>f = partial(g, a=1)</code>, then <code>a</code> is not a free argument of <code>f</code>.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>A callable or a [partial object][].</p> <p> TYPE: <code>Union[Callable, partial]</code> </p> RETURNS DESCRIPTION <code>Set[str]</code> <p>The set of free arguments of <code>fun</code>.</p> <p>New in version 0.7.0</p> Source code in <code>src/pydvl/utils/functional.py</code> <pre><code>def free_arguments(fun: Union[Callable, partial]) -&gt; Set[str]:\n\"\"\"Computes the set of free arguments for a function or\n    [functools.partial][] object.\n    All arguments of a function are considered free unless they are set by a\n    partial. For example, if `f = partial(g, a=1)`, then `a` is not a free\n    argument of `f`.\n    Args:\n        fun: A callable or a [partial object][].\n    Returns:\n        The set of free arguments of `fun`.\n    !!! tip \"New in version 0.7.0\"\n    \"\"\"\nargs_set_by_partial: Set[str] = set()\ndef _rec_unroll_partial_function_args(g: Union[Callable, partial]) -&gt; Callable:\n\"\"\"Stores arguments and recursively call itself if `g` is a\n        [functools.partial][] object. In the end, returns the initially wrapped\n        function.\n        This handles the construct `partial(_accept_additional_argument, *args,\n        **kwargs)` that is used by `maybe_add_argument`.\n        Args:\n            g: A partial or a function to unroll.\n        Returns:\n            Initial wrapped function.\n        \"\"\"\nnonlocal args_set_by_partial\nif isinstance(g, partial) and g.func == _accept_additional_argument:\narg = g.keywords[\"arg\"]\nif arg in args_set_by_partial:\nargs_set_by_partial.remove(arg)\nreturn _rec_unroll_partial_function_args(g.keywords[\"fun\"])\nelif isinstance(g, partial):\nargs_set_by_partial.update(g.keywords.keys())\nargs_set_by_partial.update(g.args)\nreturn _rec_unroll_partial_function_args(g.func)\nelse:\nreturn g\nwrapped_fn = _rec_unroll_partial_function_args(fun)\nsig = inspect.signature(wrapped_fn)\nreturn args_set_by_partial | set(sig.parameters.keys())\n</code></pre>"},{"location":"api/pydvl/utils/functional/#pydvl.utils.functional.maybe_add_argument","title":"<code>maybe_add_argument(fun, new_arg)</code>","text":"<p>Wraps a function to accept the given keyword parameter if it doesn't already.</p> <p>If <code>fun</code> already takes a keyword parameter of name <code>new_arg</code>, then it is returned as is. Otherwise, a wrapper is returned which merely ignores the argument.</p> PARAMETER  DESCRIPTION <code>fun</code> <p>The function to wrap</p> <p> TYPE: <code>Callable</code> </p> <code>new_arg</code> <p>The name of the argument that the new function will accept (and ignore).</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Callable</code> <p>A new function accepting one more keyword argument.</p> <p>Changed in version 0.7.0</p> <p>Ability to work with partials.</p> Source code in <code>src/pydvl/utils/functional.py</code> <pre><code>def maybe_add_argument(fun: Callable, new_arg: str) -&gt; Callable:\n\"\"\"Wraps a function to accept the given keyword parameter if it doesn't\n    already.\n    If `fun` already takes a keyword parameter of name `new_arg`, then it is\n    returned as is. Otherwise, a wrapper is returned which merely ignores the\n    argument.\n    Args:\n        fun: The function to wrap\n        new_arg: The name of the argument that the new function will accept\n            (and ignore).\n    Returns:\n        A new function accepting one more keyword argument.\n    !!! tip \"Changed in version 0.7.0\"\n        Ability to work with partials.\n    \"\"\"\nif new_arg in free_arguments(fun):\nreturn fun\nreturn partial(_accept_additional_argument, fun=fun, arg=new_arg)\n</code></pre>"},{"location":"api/pydvl/utils/numeric/","title":"Numeric","text":"<p>This module contains routines for numerical computations used across the library.</p>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.powerset","title":"<code>powerset(s)</code>","text":"<p>Returns an iterator for the power set of the argument.</p> <p>Subsets are generated in sequence by growing size. See  random_powerset() for random  sampling.</p> Example <pre><code>&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pydvl.utils.numeric import powerset\n&gt;&gt;&gt; list(powerset(np.array((1,2))))\n[(), (1,), (2,), (1, 2)]\n</code></pre> PARAMETER  DESCRIPTION <code>s</code> <p>The set to use</p> <p> TYPE: <code>NDArray[T]</code> </p> RETURNS DESCRIPTION <code>Iterator[Collection[T]]</code> <p>An iterator over all subsets of the set of indices <code>s</code>.</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def powerset(s: NDArray[T]) -&gt; Iterator[Collection[T]]:\n\"\"\"Returns an iterator for the power set of the argument.\n     Subsets are generated in sequence by growing size. See\n     [random_powerset()][pydvl.utils.numeric.random_powerset] for random\n     sampling.\n    ??? Example\n        ``` pycon\n        &gt;&gt;&gt; import numpy as np\n        &gt;&gt;&gt; from pydvl.utils.numeric import powerset\n        &gt;&gt;&gt; list(powerset(np.array((1,2))))\n        [(), (1,), (2,), (1, 2)]\n        ```\n    Args:\n         s: The set to use\n    Returns:\n        An iterator over all subsets of the set of indices `s`.\n    \"\"\"\nreturn chain.from_iterable(combinations(s, r) for r in range(len(s) + 1))\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.num_samples_permutation_hoeffding","title":"<code>num_samples_permutation_hoeffding(eps, delta, u_range)</code>","text":"<p>Lower bound on the number of samples required for MonteCarlo Shapley to obtain an (\u03b5,\u03b4)-approximation.</p> <p>That is: with probability 1-\u03b4, the estimated value for one data point will be \u03b5-close to the true quantity, if at least this many permutations are sampled.</p> PARAMETER  DESCRIPTION <code>eps</code> <p>\u03b5 &gt; 0</p> <p> TYPE: <code>float</code> </p> <code>delta</code> <p>0 &lt; \u03b4 &lt;= 1</p> <p> TYPE: <code>float</code> </p> <code>u_range</code> <p>Range of the Utility function</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>int</code> <p>Number of permutations required to guarantee \u03b5-correct Shapley values with probability 1-\u03b4</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def num_samples_permutation_hoeffding(eps: float, delta: float, u_range: float) -&gt; int:\n\"\"\"Lower bound on the number of samples required for MonteCarlo Shapley to\n    obtain an (\u03b5,\u03b4)-approximation.\n    That is: with probability 1-\u03b4, the estimated value for one data point will\n    be \u03b5-close to the true quantity, if at least this many permutations are\n    sampled.\n    Args:\n        eps: \u03b5 &gt; 0\n        delta: 0 &lt; \u03b4 &lt;= 1\n        u_range: Range of the [Utility][pydvl.utils.utility.Utility] function\n    Returns:\n        Number of _permutations_ required to guarantee \u03b5-correct Shapley\n            values with probability 1-\u03b4\n    \"\"\"\nreturn int(np.ceil(np.log(2 / delta) * 2 * u_range**2 / eps**2))\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_subset","title":"<code>random_subset(s, q=0.5, seed=None)</code>","text":"<p>Returns one subset at random from <code>s</code>.</p> PARAMETER  DESCRIPTION <code>s</code> <p>set to sample from</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>q</code> <p>Sampling probability for elements. The default 0.5 yields a uniform distribution over the power set of s.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.5</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>NDArray[T]</code> <p>The subset</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_subset(\ns: NDArray[T], q: float = 0.5, seed: Optional[Seed] = None\n) -&gt; NDArray[T]:\n\"\"\"Returns one subset at random from ``s``.\n    Args:\n        s: set to sample from\n        q: Sampling probability for elements. The default 0.5 yields a\n            uniform distribution over the power set of s.\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n    Returns:\n        The subset\n    \"\"\"\nrng = np.random.default_rng(seed)\nselection = rng.uniform(size=len(s)) &gt; q\nreturn s[selection]\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_powerset","title":"<code>random_powerset(s, n_samples=None, q=0.5, seed=None)</code>","text":"<p>Samples subsets from the power set of the argument, without pre-generating all subsets and in no order.</p> <p>See powerset if you wish to deterministically generate all subsets.</p> <p>To generate subsets, <code>len(s)</code> Bernoulli draws with probability <code>q</code> are drawn. The default value of <code>q = 0.5</code> provides a uniform distribution over the power set of <code>s</code>. Other choices can be used e.g. to implement owen_sampling_shapley.</p> PARAMETER  DESCRIPTION <code>s</code> <p>set to sample from</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>n_samples</code> <p>if set, stop the generator after this many steps. Defaults to <code>np.iinfo(np.int32).max</code></p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>q</code> <p>Sampling probability for elements. The default 0.5 yields a uniform distribution over the power set of s.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.5</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Generator[NDArray[T], None, None]</code> <p>Samples from the power set of <code>s</code>.</p> RAISES DESCRIPTION <code>ValueError</code> <p>if the element sampling probability is not in [0,1]</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_powerset(\ns: NDArray[T],\nn_samples: Optional[int] = None,\nq: float = 0.5,\nseed: Optional[Seed] = None,\n) -&gt; Generator[NDArray[T], None, None]:\n\"\"\"Samples subsets from the power set of the argument, without\n    pre-generating all subsets and in no order.\n    See [powerset][pydvl.utils.numeric.powerset] if you wish to deterministically generate all subsets.\n    To generate subsets, `len(s)` Bernoulli draws with probability `q` are\n    drawn. The default value of `q = 0.5` provides a uniform distribution over\n    the power set of `s`. Other choices can be used e.g. to implement\n    [owen_sampling_shapley][pydvl.value.shapley.owen.owen_sampling_shapley].\n    Args:\n        s: set to sample from\n        n_samples: if set, stop the generator after this many steps.\n            Defaults to `np.iinfo(np.int32).max`\n        q: Sampling probability for elements. The default 0.5 yields a\n            uniform distribution over the power set of s.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n    Returns:\n        Samples from the power set of `s`.\n    Raises:\n        ValueError: if the element sampling probability is not in [0,1]\n    \"\"\"\nif q &lt; 0 or q &gt; 1:\nraise ValueError(\"Element sampling probability must be in [0,1]\")\nrng = np.random.default_rng(seed)\ntotal = 1\nif n_samples is None:\nn_samples = np.iinfo(np.int32).max\nwhile total &lt;= n_samples:\nyield random_subset(s, q, seed=rng)\ntotal += 1\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_subset_of_size","title":"<code>random_subset_of_size(s, size, seed=None)</code>","text":"<p>Samples a random subset of given size uniformly from the powerset of <code>s</code>.</p> PARAMETER  DESCRIPTION <code>s</code> <p>Set to sample from</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>size</code> <p>Size of the subset to generate</p> <p> TYPE: <code>int</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>NDArray[T]</code> <p>The subset</p> <p>Raises     ValueError: If size &gt; len(s)</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_subset_of_size(\ns: NDArray[T], size: int, seed: Optional[Seed] = None\n) -&gt; NDArray[T]:\n\"\"\"Samples a random subset of given size uniformly from the powerset\n    of `s`.\n    Args:\n        s: Set to sample from\n        size: Size of the subset to generate\n        seed: Either an instance of a numpy random number generator or a seed for it.\n    Returns:\n        The subset\n    Raises\n        ValueError: If size &gt; len(s)\n    \"\"\"\nif size &gt; len(s):\nraise ValueError(\"Cannot sample subset larger than set\")\nrng = np.random.default_rng(seed)\nreturn rng.choice(s, size=size, replace=False)\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.random_matrix_with_condition_number","title":"<code>random_matrix_with_condition_number(n, condition_number, seed=None)</code>","text":"<p>Constructs a square matrix with a given condition number.</p> <p>Taken from: https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py</p> <p>Also see: https://math.stackexchange.com/questions/1351616/condition-number-of-ata.</p> PARAMETER  DESCRIPTION <code>n</code> <p>size of the matrix</p> <p> TYPE: <code>int</code> </p> <code>condition_number</code> <p>duh</p> <p> TYPE: <code>float</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>NDArray</code> <p>An (n,n) matrix with the requested condition number.</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def random_matrix_with_condition_number(\nn: int, condition_number: float, seed: Optional[Seed] = None\n) -&gt; NDArray:\n\"\"\"Constructs a square matrix with a given condition number.\n    Taken from:\n    [https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py](\n    https://gist.github.com/bstellato/23322fe5d87bb71da922fbc41d658079#file-random_mat_condition_number-py)\n    Also see:\n    [https://math.stackexchange.com/questions/1351616/condition-number-of-ata](\n    https://math.stackexchange.com/questions/1351616/condition-number-of-ata).\n    Args:\n        n: size of the matrix\n        condition_number: duh\n        seed: Either an instance of a numpy random number generator or a seed for it.\n    Returns:\n        An (n,n) matrix with the requested condition number.\n    \"\"\"\nif n &lt; 2:\nraise ValueError(\"Matrix size must be at least 2\")\nif condition_number &lt;= 1:\nraise ValueError(\"Condition number must be greater than 1\")\nrng = np.random.default_rng(seed)\nlog_condition_number = np.log(condition_number)\nexp_vec = np.arange(\n-log_condition_number / 4.0,\nlog_condition_number * (n + 1) / (4 * (n - 1)),\nlog_condition_number / (2.0 * (n - 1)),\n)\nexp_vec = exp_vec[:n]\ns: np.ndarray = np.exp(exp_vec)\nS = np.diag(s)\nU, _ = np.linalg.qr((rng.uniform(size=(n, n)) - 5.0) * 200)\nV, _ = np.linalg.qr((rng.uniform(size=(n, n)) - 5.0) * 200)\nP: np.ndarray = U.dot(S).dot(V.T)\nP = P.dot(P.T)\nreturn P\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.running_moments","title":"<code>running_moments(previous_avg, previous_variance, count, new_value)</code>","text":"<p>Uses Welford's algorithm to calculate the running average and variance of  a set of numbers.</p> <p>See Welford's algorithm in wikipedia</p> <p>Warning</p> <p>This is not really using Welford's correction for numerical stability for the variance. (FIXME)</p> <p>Todo</p> <p>This could be generalised to arbitrary moments. See this paper</p> PARAMETER  DESCRIPTION <code>previous_avg</code> <p>average value at previous step</p> <p> TYPE: <code>float | NDArray[float_]</code> </p> <code>previous_variance</code> <p>variance at previous step</p> <p> TYPE: <code>float | NDArray[float_]</code> </p> <code>count</code> <p>number of points seen so far</p> <p> TYPE: <code>int</code> </p> <code>new_value</code> <p>new value in the series of numbers</p> <p> TYPE: <code>float | NDArray[float_]</code> </p> RETURNS DESCRIPTION <code>Tuple[float | NDArray[float_], float | NDArray[float_]]</code> <p>new_average, new_variance, calculated with the new count</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def running_moments(\nprevious_avg: float | NDArray[np.float_],\nprevious_variance: float | NDArray[np.float_],\ncount: int,\nnew_value: float | NDArray[np.float_],\n) -&gt; Tuple[float | NDArray[np.float_], float | NDArray[np.float_]]:\n\"\"\"Uses Welford's algorithm to calculate the running average and variance of\n     a set of numbers.\n    See [Welford's algorithm in wikipedia](https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm)\n    !!! Warning\n        This is not really using Welford's correction for numerical stability\n        for the variance. (FIXME)\n    !!! Todo\n        This could be generalised to arbitrary moments. See [this paper](https://www.osti.gov/biblio/1028931)\n    Args:\n        previous_avg: average value at previous step\n        previous_variance: variance at previous step\n        count: number of points seen so far\n        new_value: new value in the series of numbers\n    Returns:\n        new_average, new_variance, calculated with the new count\n    \"\"\"\n# broadcasted operations seem not to be supported by mypy, so we ignore the type\nnew_average = (new_value + count * previous_avg) / (count + 1)  # type: ignore\nnew_variance = previous_variance + (\n(new_value - previous_avg) * (new_value - new_average) - previous_variance\n) / (count + 1)\nreturn new_average, new_variance\n</code></pre>"},{"location":"api/pydvl/utils/numeric/#pydvl.utils.numeric.top_k_value_accuracy","title":"<code>top_k_value_accuracy(y_true, y_pred, k=3)</code>","text":"<p>Computes the top-k accuracy for the estimated values by comparing indices of the highest k values.</p> PARAMETER  DESCRIPTION <code>y_true</code> <p>Exact/true value</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>y_pred</code> <p>Predicted/estimated value</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>k</code> <p>Number of the highest values taken into account</p> <p> TYPE: <code>int</code> DEFAULT: <code>3</code> </p> RETURNS DESCRIPTION <code>float</code> <p>Accuracy</p> Source code in <code>src/pydvl/utils/numeric.py</code> <pre><code>def top_k_value_accuracy(\ny_true: NDArray[np.float_], y_pred: NDArray[np.float_], k: int = 3\n) -&gt; float:\n\"\"\"Computes the top-k accuracy for the estimated values by comparing indices\n    of the highest k values.\n    Args:\n        y_true: Exact/true value\n        y_pred: Predicted/estimated value\n        k: Number of the highest values taken into account\n    Returns:\n        Accuracy\n    \"\"\"\ntop_k_exact_values = np.argsort(y_true)[-k:]\ntop_k_pred_values = np.argsort(y_pred)[-k:]\ntop_k_accuracy = len(np.intersect1d(top_k_exact_values, top_k_pred_values)) / k\nreturn top_k_accuracy\n</code></pre>"},{"location":"api/pydvl/utils/parallel/","title":"Parallel","text":""},{"location":"api/pydvl/utils/parallel/#pydvl.utils.parallel--this-module-is-deprecated","title":"This module is deprecated","text":"<p>Redirects</p> <p>Imports from this module will be redirected to pydvl.parallel only until v0.9.0. Please update your imports.</p>"},{"location":"api/pydvl/utils/progress/","title":"Progress","text":"<p>Warning</p> <p>This module is deprecated and will be removed in a future release. It implements a wrapper for the tqdm progress bar iterator for easy toggling, but this functionality is already provided by the <code>disable</code> argument of <code>tqdm</code>.</p>"},{"location":"api/pydvl/utils/progress/#pydvl.utils.progress.MockProgress","title":"<code>MockProgress(iterator)</code>","text":"<p>             Bases: <code>Iterator</code></p> <p>A Naive mock class to use with maybe_progress and tqdm. Mocked methods don't support return values. Mocked properties don't do anything</p> Source code in <code>src/pydvl/utils/progress.py</code> <pre><code>def __init__(self, iterator: Union[Iterator, Iterable]):\n# Since there is no _it in __dict__ at this point, doing here\n# self._it = iterator\n# results in a call to __getattr__() and the assignment fails, so we\n# use __dict__ instead\nself.__dict__[\"_it\"] = iterator\n</code></pre>"},{"location":"api/pydvl/utils/progress/#pydvl.utils.progress.maybe_progress","title":"<code>maybe_progress(it, display=False, **kwargs)</code>","text":"<p>Returns either a tqdm progress bar or a mock object which wraps the iterator as well, but ignores any accesses to methods or properties.</p> PARAMETER  DESCRIPTION <code>it</code> <p>the iterator to wrap</p> <p> TYPE: <code>Union[int, Iterable, Iterator]</code> </p> <code>display</code> <p>set to True to return a tqdm bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>kwargs</code> <p>Keyword arguments that will be forwarded to tqdm</p> <p> DEFAULT: <code>{}</code> </p> Source code in <code>src/pydvl/utils/progress.py</code> <pre><code>def maybe_progress(\nit: Union[int, Iterable, Iterator], display: bool = False, **kwargs\n) -&gt; Union[tqdm, MockProgress]:\n\"\"\"Returns either a tqdm progress bar or a mock object which wraps the\n    iterator as well, but ignores any accesses to methods or properties.\n    Args:\n        it: the iterator to wrap\n        display: set to True to return a tqdm bar\n        kwargs: Keyword arguments that will be forwarded to tqdm\n    \"\"\"\nif isinstance(it, int):\nit = range(it)  # type: ignore\nreturn tqdm(it, **kwargs) if display else MockProgress(it)\n</code></pre>"},{"location":"api/pydvl/utils/score/","title":"Score","text":"<p>This module provides a Scorer class that wraps scoring functions with additional information.</p> <p>Scorers are the fundamental building block of many data valuation methods. They are typically used by the Utility class to evaluate the quality of a model when trained on subsets of the training data.</p> <p>Scorers can be constructed in the same way as in scikit-learn: either from  known strings or from a callable. Greater values must be better. If they are not, a negated version can be used, see scikit-learn's make_scorer().</p> <p>Scorer provides additional information about the scoring function, like its range and default values, which can be used by some data valuation methods (like group_testing_shapley()) to estimate the number of samples required for a certain quality of approximation.</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.squashed_r2","title":"<code>squashed_r2 = compose_score(Scorer('r2'), _sigmoid, (0, 1), 'squashed r2')</code>  <code>module-attribute</code>","text":"<p>A scorer that squashes the R\u00b2 score into the range [0, 1] using a sigmoid.</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.squashed_variance","title":"<code>squashed_variance = compose_score(Scorer('explained_variance'), _sigmoid, (0, 1), 'squashed explained variance')</code>  <code>module-attribute</code>","text":"<p>A scorer that squashes the explained variance score into the range [0, 1] using a sigmoid.</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.ScorerCallable","title":"<code>ScorerCallable</code>","text":"<p>             Bases: <code>Protocol</code></p> <p>Signature for a scorer</p>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.Scorer","title":"<code>Scorer(scoring, default=np.nan, range=(-np.inf, np.inf), name=None)</code>","text":"<p>A scoring callable that takes a model, data, and labels and returns a scalar.</p> PARAMETER  DESCRIPTION <code>scoring</code> <p>Either a string or callable that can be passed to get_scorer.</p> <p> TYPE: <code>Union[str, ScorerCallable]</code> </p> <code>default</code> <p>score to be used when a model cannot be fit, e.g. when too little data is passed, or errors arise.</p> <p> TYPE: <code>float</code> DEFAULT: <code>nan</code> </p> <code>range</code> <p>numerical range of the score function. Some Monte Carlo methods can use this to estimate the number of samples required for a certain quality of approximation. If not provided, it can be read from the <code>scoring</code> object if it provides it, for instance if it was constructed with compose_score().</p> <p> TYPE: <code>Tuple</code> DEFAULT: <code>(-inf, inf)</code> </p> <code>name</code> <p>The name of the scorer. If not provided, the name of the function passed will be used.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <p>New in version 0.5.0</p> Source code in <code>src/pydvl/utils/score.py</code> <pre><code>def __init__(\nself,\nscoring: Union[str, ScorerCallable],\ndefault: float = np.nan,\nrange: Tuple = (-np.inf, np.inf),\nname: Optional[str] = None,\n):\nif name is None and isinstance(scoring, str):\nname = scoring\nself._scorer = get_scorer(scoring)\nself.default = default\n# TODO: auto-fill from known scorers ?\nself.range = np.array(range)\nself._name = getattr(self._scorer, \"__name__\", name or \"scorer\")\n</code></pre>"},{"location":"api/pydvl/utils/score/#pydvl.utils.score.compose_score","title":"<code>compose_score(scorer, transformation, range, name)</code>","text":"<p>Composes a scoring function with an arbitrary scalar transformation.</p> <p>Useful to squash unbounded scores into ranges manageable by data valuation methods.</p> <p>Example:</p> <pre><code>sigmoid = lambda x: 1/(1+np.exp(-x))\ncompose_score(Scorer(\"r2\"), sigmoid, range=(0,1), name=\"squashed r2\")\n</code></pre> PARAMETER  DESCRIPTION <code>scorer</code> <p>The object to be composed.</p> <p> TYPE: <code>Scorer</code> </p> <code>transformation</code> <p>A scalar transformation</p> <p> TYPE: <code>Callable[[float], float]</code> </p> <code>range</code> <p>The range of the transformation. This will be used e.g. by Utility for the range of the composed.</p> <p> TYPE: <code>Tuple[float, float]</code> </p> <code>name</code> <p>A string representation for the composition, for <code>str()</code>.</p> <p> TYPE: <code>str</code> </p> RETURNS DESCRIPTION <code>Scorer</code> <p>The composite Scorer.</p> Source code in <code>src/pydvl/utils/score.py</code> <pre><code>def compose_score(\nscorer: Scorer,\ntransformation: Callable[[float], float],\nrange: Tuple[float, float],\nname: str,\n) -&gt; Scorer:\n\"\"\"Composes a scoring function with an arbitrary scalar transformation.\n    Useful to squash unbounded scores into ranges manageable by data valuation\n    methods.\n    Example:\n    ```python\n    sigmoid = lambda x: 1/(1+np.exp(-x))\n    compose_score(Scorer(\"r2\"), sigmoid, range=(0,1), name=\"squashed r2\")\n    ```\n    Args:\n        scorer: The object to be composed.\n        transformation: A scalar transformation\n        range: The range of the transformation. This will be used e.g. by\n            [Utility][pydvl.utils.utility.Utility] for the range of the composed.\n        name: A string representation for the composition, for `str()`.\n    Returns:\n        The composite [Scorer][pydvl.utils.score.Scorer].\n    \"\"\"\nclass CompositeScorer(Scorer):\ndef __call__(self, model: SupervisedModel, X: NDArray, y: NDArray) -&gt; float:\nscore = self._scorer(model=model, X=X, y=y)\nreturn transformation(score)\nreturn CompositeScorer(scorer, range=range, name=name)\n</code></pre>"},{"location":"api/pydvl/utils/status/","title":"Status","text":""},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status","title":"<code>Status</code>","text":"<p>             Bases: <code>Enum</code></p> <p>Status of a computation.</p> <p>Statuses can be combined using bitwise or (<code>|</code>) and bitwise and (<code>&amp;</code>) to get the status of a combined computation. For example, if we have two computations, one that has converged and one that has failed, then the combined status is <code>Status.Converged | Status.Failed == Status.Converged</code>, but <code>Status.Converged &amp; Status.Failed == Status.Failed</code>.</p>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--or","title":"OR","text":"<p>The result of bitwise or-ing two valuation statuses with <code>|</code> is given by the following table:</p> P C F P P C P C C C C F P C F <p>where P = Pending, C = Converged, F = Failed.</p>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--and","title":"AND","text":"<p>The result of bitwise and-ing two valuation statuses with <code>&amp;</code> is given by the following table:</p> P C F P P P F C P C F F F F F <p>where P = Pending, C = Converged, F = Failed.</p>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--not","title":"NOT","text":"<p>The result of bitwise negation of a Status with <code>~</code> is <code>Failed</code> if the status is <code>Converged</code>, or <code>Converged</code> otherwise:</p> <pre><code>~P == C, ~C == F, ~F == C\n</code></pre>"},{"location":"api/pydvl/utils/status/#pydvl.utils.status.Status--boolean-casting","title":"Boolean casting","text":"<p>A Status evaluates to <code>True</code> iff it's <code>Converged</code> or <code>Failed</code>:</p> <pre><code>bool(Status.Pending) == False\nbool(Status.Converged) == True\nbool(Status.Failed) == True\n</code></pre> <p>Warning</p> <p>These truth values are inconsistent with the usual boolean operations. In particular the XOR of two instances of <code>Status</code> is not the same as the XOR of their boolean values.</p>"},{"location":"api/pydvl/utils/types/","title":"Types","text":"<p>This module contains types, protocols, decorators and generic function transformations. Some of it probably belongs elsewhere.</p>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.SupervisedModel","title":"<code>SupervisedModel</code>","text":"<p>             Bases: <code>Protocol</code></p> <p>This is the minimal Protocol that valuation methods require from models in order to work.</p> <p>All that is needed are the standard sklearn methods <code>fit()</code>, <code>predict()</code> and <code>score()</code>.</p>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.SupervisedModel.fit","title":"<code>fit(x, y)</code>","text":"<p>Fit the model to the data</p> PARAMETER  DESCRIPTION <code>x</code> <p>Independent variables</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>Dependent variable</p> <p> TYPE: <code>NDArray</code> </p> Source code in <code>src/pydvl/utils/types.py</code> <pre><code>def fit(self, x: NDArray, y: NDArray):\n\"\"\"Fit the model to the data\n    Args:\n        x: Independent variables\n        y: Dependent variable\n    \"\"\"\npass\n</code></pre>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.SupervisedModel.predict","title":"<code>predict(x)</code>","text":"<p>Compute predictions for the input</p> PARAMETER  DESCRIPTION <code>x</code> <p>Independent variables for which to compute predictions</p> <p> TYPE: <code>NDArray</code> </p> RETURNS DESCRIPTION <code>NDArray</code> <p>Predictions for the input</p> Source code in <code>src/pydvl/utils/types.py</code> <pre><code>def predict(self, x: NDArray) -&gt; NDArray:\n\"\"\"Compute predictions for the input\n    Args:\n        x: Independent variables for which to compute predictions\n    Returns:\n        Predictions for the input\n    \"\"\"\npass\n</code></pre>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.SupervisedModel.score","title":"<code>score(x, y)</code>","text":"<p>Compute the score of the model given test data</p> PARAMETER  DESCRIPTION <code>x</code> <p>Independent variables</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>Dependent variable</p> <p> TYPE: <code>NDArray</code> </p> RETURNS DESCRIPTION <code>float</code> <p>The score of the model on <code>(x, y)</code></p> Source code in <code>src/pydvl/utils/types.py</code> <pre><code>def score(self, x: NDArray, y: NDArray) -&gt; float:\n\"\"\"Compute the score of the model given test data\n    Args:\n        x: Independent variables\n        y: Dependent variable\n    Returns:\n        The score of the model on `(x, y)`\n    \"\"\"\npass\n</code></pre>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.NoPublicConstructor","title":"<code>NoPublicConstructor</code>","text":"<p>             Bases: <code>ABCMeta</code></p> <p>Metaclass that ensures a private constructor</p> <p>If a class uses this metaclass like this:</p> <pre><code>class SomeClass(metaclass=NoPublicConstructor):\n    pass\n</code></pre> <p>If you try to instantiate your class (<code>SomeClass()</code>), a <code>TypeError</code> will be thrown.</p> <p>Taken almost verbatim from: https://stackoverflow.com/a/64682734</p>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.NoPublicConstructor.create","title":"<code>create(*args, **kwargs)</code>","text":"<p>Create an instance of the class</p> Source code in <code>src/pydvl/utils/types.py</code> <pre><code>def create(cls, *args: Any, **kwargs: Any):\n\"\"\"Create an instance of the class\"\"\"\nreturn super().__call__(*args, **kwargs)\n</code></pre>"},{"location":"api/pydvl/utils/types/#pydvl.utils.types.ensure_seed_sequence","title":"<code>ensure_seed_sequence(seed=None)</code>","text":"<p>If the passed seed is a SeedSequence object then it is returned as is. If it is a Generator the internal protected seed sequence from the generator gets extracted. Otherwise, a new SeedSequence object is created from the passed (optional) seed.</p> PARAMETER  DESCRIPTION <code>seed</code> <p>Either an int, a Generator object a SeedSequence object or None.</p> <p> TYPE: <code>Optional[Union[Seed, SeedSequence]]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>SeedSequence</code> <p>A SeedSequence object.</p> <p>New in version 0.7.0</p> Source code in <code>src/pydvl/utils/types.py</code> <pre><code>def ensure_seed_sequence(\nseed: Optional[Union[Seed, SeedSequence]] = None\n) -&gt; SeedSequence:\n\"\"\"\n    If the passed seed is a SeedSequence object then it is returned as is. If it is\n    a Generator the internal protected seed sequence from the generator gets extracted.\n    Otherwise, a new SeedSequence object is created from the passed (optional) seed.\n    Args:\n        seed: Either an int, a Generator object a SeedSequence object or None.\n    Returns:\n        A SeedSequence object.\n    !!! tip \"New in version 0.7.0\"\n    \"\"\"\nif isinstance(seed, SeedSequence):\nreturn seed\nelif isinstance(seed, Generator):\nreturn cast(SeedSequence, seed.bit_generator.seed_seq)  # type: ignore\nelse:\nreturn SeedSequence(seed)\n</code></pre>"},{"location":"api/pydvl/utils/utility/","title":"Utility","text":"<p>This module contains classes to manage and learn utility functions for the computation of values. Please see the documentation on Computing Data Values for more information.</p> <p>Utility holds information about model, data and scoring function (the latter being what one usually understands under utility in the general definition of Shapley value). It is automatically cached across machines when the cache is configured and it is enabled upon construction.</p> <p>DataUtilityLearning adds support for learning the scoring function to avoid repeated re-training of the model to compute the score.</p> <p>This module also contains derived <code>Utility</code> classes for toy games that are used for testing and for demonstration purposes.</p>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility--references","title":"References","text":"<ol> <li> <p>Wang, T., Yang, Y. and Jia, R., 2021. Improving cooperative game theory-based data valuation via data utility learning. arXiv preprint arXiv:2107.06336.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.Utility","title":"<code>Utility(model, data, scorer=None, *, default_score=0.0, score_range=(-np.inf, np.inf), catch_errors=True, show_warnings=False, enable_cache=False, cache_options=None, clone_before_fit=True)</code>","text":"<p>Convenience wrapper with configurable memoization of the scoring function.</p> <p>An instance of <code>Utility</code> holds the triple of model, dataset and scoring function which determines the value of data points. This is used for the computation of all game-theoretic values like Shapley values and the Least Core.</p> <p>The Utility expect the model to fulfill the SupervisedModel interface i.e. to have <code>fit()</code>, <code>predict()</code>, and <code>score()</code> methods.</p> <p>When calling the utility, the model will be cloned if it is a Sci-Kit Learn model, otherwise a copy is created using copy.deepcopy</p> <p>Since evaluating the scoring function requires retraining the model and that can be time-consuming, this class wraps it and caches the results of each execution. Caching is available both locally and across nodes, but must always be enabled for your project first, see Setting up the cache.</p> ATTRIBUTE DESCRIPTION <code>model</code> <p>The supervised model.</p> <p> TYPE: <code>SupervisedModel</code> </p> <code>data</code> <p>An object containing the split data.</p> <p> TYPE: <code>Dataset</code> </p> <code>scorer</code> <p>A scoring function. If None, the <code>score()</code> method of the model will be used. See score for ways to create and compose scorers, in particular how to set default values and ranges.</p> <p> TYPE: <code>Scorer</code> </p> PARAMETER  DESCRIPTION <code>model</code> <p>Any supervised model. Typical choices can be found in the [sci-kit learn documentation][https://scikit-learn.org/stable/supervised_learning.html].</p> <p> TYPE: <code>SupervisedModel</code> </p> <code>data</code> <p>Dataset or GroupedDataset instance.</p> <p> TYPE: <code>Dataset</code> </p> <code>scorer</code> <p>A scoring object. If None, the <code>score()</code> method of the model will be used. See score for ways to create and compose scorers, in particular how to set default values and ranges. For convenience, a string can be passed, which will be used to construct a Scorer.</p> <p> TYPE: <code>Optional[Union[str, Scorer]]</code> DEFAULT: <code>None</code> </p> <code>default_score</code> <p>As a convenience when no <code>scorer</code> object is passed (where a default value can be provided), this argument also allows to set the default score for models that have not been fit, e.g. when too little data is passed, or errors arise.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.0</code> </p> <code>score_range</code> <p>As with <code>default_score</code>, this is a convenience argument for when no <code>scorer</code> argument is provided, to set the numerical range of the score function. Some Monte Carlo methods can use this to estimate the number of samples required for a certain quality of approximation.</p> <p> TYPE: <code>Tuple[float, float]</code> DEFAULT: <code>(-inf, inf)</code> </p> <code>catch_errors</code> <p>set to <code>True</code> to catch the errors when <code>fit()</code> fails. This could happen in several steps of the pipeline, e.g. when too little training data is passed, which happens often during Shapley value calculations. When this happens, the <code>default_score</code> is returned as a score and computation continues.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>show_warnings</code> <p>Set to <code>False</code> to suppress warnings thrown by <code>fit()</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>enable_cache</code> <p>If <code>True</code>, use memcached for memoization.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>cache_options</code> <p>Optional configuration object for memcached.</p> <p> TYPE: <code>Optional[MemcachedConfig]</code> DEFAULT: <code>None</code> </p> <code>clone_before_fit</code> <p>If <code>True</code>, the model will be cloned before calling <code>fit()</code>.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Utility, DataUtilityLearning, Dataset\n&gt;&gt;&gt; from sklearn.linear_model import LinearRegression, LogisticRegression\n&gt;&gt;&gt; from sklearn.datasets import load_iris\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_iris(), random_state=16)\n&gt;&gt;&gt; u = Utility(LogisticRegression(random_state=16), dataset)\n&gt;&gt;&gt; u(dataset.indices)\n0.9\n</code></pre> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __init__(\nself,\nmodel: SupervisedModel,\ndata: Dataset,\nscorer: Optional[Union[str, Scorer]] = None,\n*,\ndefault_score: float = 0.0,\nscore_range: Tuple[float, float] = (-np.inf, np.inf),\ncatch_errors: bool = True,\nshow_warnings: bool = False,\nenable_cache: bool = False,\ncache_options: Optional[MemcachedConfig] = None,\nclone_before_fit: bool = True,\n):\nself.model = self._clone_model(model)\nself.data = data\nif isinstance(scorer, str):\nscorer = Scorer(scorer, default=default_score, range=score_range)\nself.scorer = check_scoring(self.model, scorer)\nself.default_score = scorer.default if scorer is not None else default_score\n# TODO: auto-fill from known scorers ?\nself.score_range = scorer.range if scorer is not None else np.array(score_range)\nself.catch_errors = catch_errors\nself.show_warnings = show_warnings\nself.enable_cache = enable_cache\nself.cache_options: MemcachedConfig = cache_options or MemcachedConfig()\nself.clone_before_fit = clone_before_fit\nself._signature = serialize((hash(self.model), hash(data), hash(scorer)))\nself._initialize_utility_wrapper()\n</code></pre>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.Utility.signature","title":"<code>signature</code>  <code>property</code>","text":"<p>Signature used for caching model results.</p>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.Utility.cache_stats","title":"<code>cache_stats: Optional[CacheStats]</code>  <code>property</code>","text":"<p>Cache statistics are gathered when cache is enabled. See CacheStats for all fields returned.</p>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.Utility.__call__","title":"<code>__call__(indices)</code>","text":"PARAMETER  DESCRIPTION <code>indices</code> <p>a subset of valid indices for the <code>x_train</code> attribute of Dataset.</p> <p> TYPE: <code>Iterable[int]</code> </p> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __call__(self, indices: Iterable[int]) -&gt; float:\n\"\"\"\n    Args:\n        indices: a subset of valid indices for the\n            `x_train` attribute of [Dataset][pydvl.utils.dataset.Dataset].\n    \"\"\"\nutility: float = self._utility_wrapper(frozenset(indices))\nreturn utility\n</code></pre>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.DataUtilityLearning","title":"<code>DataUtilityLearning(u, training_budget, model)</code>","text":"<p>Implementation of Data Utility Learning (Wang et al., 2022)<sup>1</sup>.</p> <p>This object wraps a Utility and delegates calls to it, up until a given budget (number of iterations). Every tuple of input and output (a so-called utility sample) is stored. Once the budget is exhausted, <code>DataUtilityLearning</code> fits the given model to the utility samples. Subsequent calls will use the learned model to predict the utility instead of delegating.</p> PARAMETER  DESCRIPTION <code>u</code> <p>The Utility to learn.</p> <p> TYPE: <code>Utility</code> </p> <code>training_budget</code> <p>Number of utility samples to collect before fitting the given model.</p> <p> TYPE: <code>int</code> </p> <code>model</code> <p>A supervised regression model</p> <p> TYPE: <code>SupervisedModel</code> </p> Example <pre><code>&gt;&gt;&gt; from pydvl.utils import Utility, DataUtilityLearning, Dataset\n&gt;&gt;&gt; from sklearn.linear_model import LinearRegression, LogisticRegression\n&gt;&gt;&gt; from sklearn.datasets import load_iris\n&gt;&gt;&gt; dataset = Dataset.from_sklearn(load_iris())\n&gt;&gt;&gt; u = Utility(LogisticRegression(), dataset)\n&gt;&gt;&gt; wrapped_u = DataUtilityLearning(u, 3, LinearRegression())\n... # First 3 calls will be computed normally\n&gt;&gt;&gt; for i in range(3):\n...     _ = wrapped_u((i,))\n&gt;&gt;&gt; wrapped_u((1, 2, 3)) # Subsequent calls will be computed using the fit model for DUL\n0.0\n</code></pre> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __init__(\nself, u: Utility, training_budget: int, model: SupervisedModel\n) -&gt; None:\nself.utility = u\nself.training_budget = training_budget\nself.model = model\nself._current_iteration = 0\nself._is_model_fit = False\nself._utility_samples: Dict[FrozenSet, Tuple[NDArray[np.bool_], float]] = {}\n</code></pre>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.DataUtilityLearning.data","title":"<code>data: Dataset</code>  <code>property</code>","text":"<p>Returns the wrapped utility's Dataset.</p>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.MinerGameUtility","title":"<code>MinerGameUtility(n_miners, **kwargs)</code>","text":"<p>             Bases: <code>Utility</code></p> <p>Toy game utility that is used for testing and demonstration purposes.</p> <p>Consider a group of n miners, who have discovered large bars of gold.</p> <p>If two miners can carry one piece of gold, then the payoff of a coalition \\(S\\) is:</p> \\[{ v(S) = \\left\\{\\begin{array}{lll} \\mid S \\mid / 2 &amp; \\text{, if} &amp; \\mid S \\mid \\text{ is even} \\\\ ( \\mid S \\mid - 1)/2 &amp; \\text{, if} &amp; \\mid S \\mid \\text{ is odd} \\end{array}\\right. }\\] <p>If there are more than two miners and there is an even number of miners, then the core consists of the single payoff where each miner gets 1/2.</p> <p>If there is an odd number of miners, then the core is empty.</p> <p>Taken from Wikipedia</p> PARAMETER  DESCRIPTION <code>n_miners</code> <p>Number of miners that participate in the game.</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __init__(self, n_miners: int, **kwargs):\nif n_miners &lt;= 2:\nraise ValueError(f\"n_miners, {n_miners} should be &gt; 2\")\nself.n_miners = n_miners\nx = np.arange(n_miners)[..., np.newaxis]\n# The y values don't matter here\ny = np.zeros_like(x)\nself.data = Dataset(x_train=x, y_train=y, x_test=x, y_test=y)\n</code></pre>"},{"location":"api/pydvl/utils/utility/#pydvl.utils.utility.GlovesGameUtility","title":"<code>GlovesGameUtility(left, right, **kwargs)</code>","text":"<p>             Bases: <code>Utility</code></p> <p>Toy game utility that is used for testing and demonstration purposes.</p> <p>In this game, some players have a left glove and others a right glove. Single gloves have a worth of zero while pairs have a worth of 1.</p> <p>The payoff of a coalition \\(S\\) is:</p> \\[{ v(S) = \\min( \\mid S \\cap L \\mid, \\mid S \\cap R \\mid ) }\\] <p>Where \\(L\\), respectively \\(R\\), is the set of players with left gloves, respectively right gloves.</p> PARAMETER  DESCRIPTION <code>left</code> <p>Number of players with a left glove.</p> <p> TYPE: <code>int</code> </p> <code>right</code> <p>Number of player with a right glove.</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/utils/utility.py</code> <pre><code>def __init__(self, left: int, right: int, **kwargs):\nself.left = left\nself.right = right\nx = np.empty(left + right)[..., np.newaxis]\n# The y values don't matter here\ny = np.zeros_like(x)\nself.data = Dataset(x_train=x, y_train=y, x_test=x, y_test=y)\n</code></pre>"},{"location":"api/pydvl/value/","title":"Value","text":"<p>This module implements algorithms for the exact and approximate computation of values and semi-values.</p> <p>See Data valuation for an introduction to the concepts and methods implemented here.</p>"},{"location":"api/pydvl/value/result/","title":"Result","text":"<p>This module collects types and methods for the inspection of the results of valuation algorithms.</p> <p>The most important class is ValuationResult, which provides access to raw values, as well as convenient behaviour as a <code>Sequence</code> with extended indexing and updating abilities, and conversion to pandas DataFrames.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result--operating-on-results","title":"Operating on results","text":"<p>Results can be added together with the standard <code>+</code> operator. Because values are typically running averages of iterative algorithms, addition behaves like a weighted average of the two results, with the weights being the number of updates in each result: adding two results is the same as generating one result with the mean of the values of the two results as values. The variances are updated accordingly. See ValuationResult for details.</p> <p>Results can also be sorted by value, variance or number of updates, see sort(). The arrays of ValuationResult.values, ValuationResult.variances, ValuationResult.counts, ValuationResult.indices, ValuationResult.names are sorted in the same way.</p> <p>Indexing and slicing of results is supported and ValueItem objects are returned. These objects can be compared with the usual operators, which take only the ValueItem.value into account.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result--creating-result-objects","title":"Creating result objects","text":"<p>The most commonly used factory method is ValuationResult.zeros(), which creates a result object with all values, variances and counts set to zero. ValuationResult.empty() creates an empty result object, which can be used as a starting point for adding results together. Empty results are discarded when added to other results. Finally, ValuationResult.from_random() samples random values uniformly.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValueItem","title":"<code>ValueItem</code>  <code>dataclass</code>","text":"<p>             Bases: <code>Generic[IndexT, NameT]</code></p> <p>The result of a value computation for one datum.</p> <p><code>ValueItems</code> can be compared with the usual operators, forming a total order. Comparisons take only the <code>value</code> into account.</p> <p>Todo</p> <p>Maybe have a mode of comparing similar to <code>np.isclose</code>, or taking the <code>variance</code> into account.</p> ATTRIBUTE DESCRIPTION <code>index</code> <p>Index of the sample with this value in the original Dataset</p> <p> TYPE: <code>IndexT</code> </p> <code>name</code> <p>Name of the sample if it was provided. Otherwise, <code>str(index)</code></p> <p> TYPE: <code>NameT</code> </p> <code>value</code> <p>The value</p> <p> TYPE: <code>float</code> </p> <code>variance</code> <p>Variance of the value if it was computed with an approximate method</p> <p> TYPE: <code>Optional[float]</code> </p> <code>count</code> <p>Number of updates for this value</p> <p> TYPE: <code>Optional[int]</code> </p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValueItem.stderr","title":"<code>stderr: Optional[float]</code>  <code>property</code>","text":"<p>Standard error of the value.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult","title":"<code>ValuationResult(*, values, variances=None, counts=None, indices=None, data_names=None, algorithm='', status=Status.Pending, sort=False, **extra_values)</code>","text":"<p>             Bases: <code>Sequence</code>, <code>Iterable[ValueItem[IndexT, NameT]]</code>, <code>Generic[IndexT, NameT]</code></p> <p>Objects of this class hold the results of valuation algorithms.</p> <p>These include indices in the original Dataset, any data names (e.g. group names in GroupedDataset), the values themselves, and variance of the computation in the case of Monte Carlo methods. <code>ValuationResults</code> can be iterated over like any <code>Sequence</code>: <code>iter(valuation_result)</code> returns a generator of ValueItem in the order in which the object is sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult--indexing","title":"Indexing","text":"<p>Indexing can be position-based, when accessing any of the attributes values, variances, counts and indices, as well as when iterating over the object, or using the item access operator, both getter and setter. The \"position\" is either the original sequence in which the data was passed to the constructor, or the sequence in which the object is sorted, see below.</p> <p>Alternatively, indexing can be data-based, i.e. using the indices in the original dataset. This is the case for the methods get() and update().</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult--sorting","title":"Sorting","text":"<p>Results can be sorted in-place with sort(), or alternatively using python's standard <code>sorted()</code> and <code>reversed()</code> Note that sorting values affects how iterators and the object itself as <code>Sequence</code> behave: <code>values[0]</code> returns a ValueItem with the highest or lowest ranking point if this object is sorted by descending or ascending value, respectively. If unsorted, <code>values[0]</code> returns the <code>ValueItem</code> at position 0, which has data index <code>indices[0]</code> in the Dataset.</p> <p>The same applies to direct indexing of the <code>ValuationResult</code>: the index is positional, according to the sorting. It does not refer to the \"data index\". To sort according to data index, use sort() with <code>key=\"index\"</code>.</p> <p>In order to access ValueItem objects by their data index, use get().</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult--operating-on-results","title":"Operating on results","text":"<p>Results can be added to each other with the <code>+</code> operator. Means and variances are correctly updated, using the <code>counts</code> attribute.</p> <p>Results can also be updated with new values using update(). Means and variances are updated accordingly using the Welford algorithm.</p> <p>Empty objects behave in a special way, see empty().</p> PARAMETER  DESCRIPTION <code>values</code> <p>An array of values. If omitted, defaults to an empty array or to an array of zeros if <code>indices</code> are given.</p> <p> TYPE: <code>NDArray[float_]</code> </p> <code>indices</code> <p>An optional array of indices in the original dataset. If omitted, defaults to <code>np.arange(len(values))</code>. Warning: It is common to pass the indices of a Dataset here. Attention must be paid in a parallel context to copy them to the local process. Just do <code>indices=np.copy(data.indices)</code>.</p> <p> TYPE: <code>Optional[NDArray[IndexT]]</code> DEFAULT: <code>None</code> </p> <code>variances</code> <p>An optional array of variances in the computation of each value.</p> <p> TYPE: <code>Optional[NDArray[float_]]</code> DEFAULT: <code>None</code> </p> <code>counts</code> <p>An optional array with the number of updates for each value. Defaults to an array of ones.</p> <p> TYPE: <code>Optional[NDArray[int_]]</code> DEFAULT: <code>None</code> </p> <code>data_names</code> <p>Names for the data points. Defaults to index numbers if not set.</p> <p> TYPE: <code>Optional[Sequence[NameT] | NDArray[NameT]]</code> DEFAULT: <code>None</code> </p> <code>algorithm</code> <p>The method used.</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> <code>status</code> <p>The end status of the algorithm.</p> <p> TYPE: <code>Status</code> DEFAULT: <code>Pending</code> </p> <code>sort</code> <p>Whether to sort the indices by ascending value. See above how this affects usage as an iterable or sequence.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>extra_values</code> <p>Additional values that can be passed as keyword arguments. This can contain, for example, the least core value.</p> <p> DEFAULT: <code>{}</code> </p> RAISES DESCRIPTION <code>ValueError</code> <p>If input arrays have mismatching lengths.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __init__(\nself,\n*,\nvalues: NDArray[np.float_],\nvariances: Optional[NDArray[np.float_]] = None,\ncounts: Optional[NDArray[np.int_]] = None,\nindices: Optional[NDArray[IndexT]] = None,\ndata_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\nalgorithm: str = \"\",\nstatus: Status = Status.Pending,\nsort: bool = False,\n**extra_values,\n):\nif variances is not None and len(variances) != len(values):\nraise ValueError(\"Lengths of values and variances do not match\")\nif data_names is not None and len(data_names) != len(values):\nraise ValueError(\"Lengths of values and data_names do not match\")\nif indices is not None and len(indices) != len(values):\nraise ValueError(\"Lengths of values and indices do not match\")\nself._algorithm = algorithm\nself._status = Status(status)  # Just in case we are given a string\nself._values = values\nself._variances = np.zeros_like(values) if variances is None else variances\nself._counts = np.ones_like(values) if counts is None else counts\nself._sort_order = None\nself._extra_values = extra_values or {}\n# Yuk...\nif data_names is None:\nif indices is not None:\nself._names = np.copy(indices)\nelse:\nself._names = np.arange(len(self._values), dtype=np.int_)\nelif not isinstance(data_names, np.ndarray):\nself._names = np.array(data_names)\nelse:\nself._names = data_names.copy()\nif len(np.unique(self._names)) != len(self._names):\nraise ValueError(\"Data names must be unique\")\nif indices is None:\nindices = np.arange(len(self._values), dtype=np.int_)\nself._indices = indices\nself._positions = {idx: pos for pos, idx in enumerate(indices)}\nself._sort_positions: NDArray[np.int_] = np.arange(\nlen(self._values), dtype=np.int_\n)\nif sort:\nself.sort()\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.values","title":"<code>values: NDArray[np.float_]</code>  <code>property</code>","text":"<p>The values, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.variances","title":"<code>variances: NDArray[np.float_]</code>  <code>property</code>","text":"<p>The variances, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.stderr","title":"<code>stderr: NDArray[np.float_]</code>  <code>property</code>","text":"<p>The raw standard errors, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.counts","title":"<code>counts: NDArray[np.int_]</code>  <code>property</code>","text":"<p>The raw counts, possibly sorted.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.indices","title":"<code>indices: NDArray[IndexT]</code>  <code>property</code>","text":"<p>The indices for the values, possibly sorted.</p> <p>If the object is unsorted, then these are the same as declared at construction or <code>np.arange(len(values))</code> if none were passed.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.names","title":"<code>names: NDArray[NameT]</code>  <code>property</code>","text":"<p>The names for the values, possibly sorted. If the object is unsorted, then these are the same as declared at construction or <code>np.arange(len(values))</code> if none were passed.</p>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.sort","title":"<code>sort(reverse=False, key='value')</code>","text":"<p>Sorts the indices in place by <code>key</code>.</p> <p>Once sorted, iteration over the results, and indexing of all the properties ValuationResult.values, ValuationResult.variances, ValuationResult.counts, ValuationResult.indices and ValuationResult.names will follow the same order.</p> PARAMETER  DESCRIPTION <code>reverse</code> <p>Whether to sort in descending order by value.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>key</code> <p>The key to sort by. Defaults to ValueItem.value.</p> <p> TYPE: <code>Literal['value', 'variance', 'index', 'name']</code> DEFAULT: <code>'value'</code> </p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def sort(\nself,\nreverse: bool = False,\n# Need a \"Comparable\" type here\nkey: Literal[\"value\", \"variance\", \"index\", \"name\"] = \"value\",\n) -&gt; None:\n\"\"\"Sorts the indices in place by `key`.\n    Once sorted, iteration over the results, and indexing of all the\n    properties\n    [ValuationResult.values][pydvl.value.result.ValuationResult.values],\n    [ValuationResult.variances][pydvl.value.result.ValuationResult.variances],\n    [ValuationResult.counts][pydvl.value.result.ValuationResult.counts],\n    [ValuationResult.indices][pydvl.value.result.ValuationResult.indices]\n    and [ValuationResult.names][pydvl.value.result.ValuationResult.names]\n    will follow the same order.\n    Args:\n        reverse: Whether to sort in descending order by value.\n        key: The key to sort by. Defaults to\n            [ValueItem.value][pydvl.value.result.ValueItem].\n    \"\"\"\nkeymap = {\n\"index\": \"_indices\",\n\"value\": \"_values\",\n\"variance\": \"_variances\",\n\"name\": \"_names\",\n}\nself._sort_positions = np.argsort(getattr(self, keymap[key]))\nif reverse:\nself._sort_positions = self._sort_positions[::-1]\nself._sort_order = reverse\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.__getattr__","title":"<code>__getattr__(attr)</code>","text":"<p>Allows access to extra values as if they were properties of the instance.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __getattr__(self, attr: str) -&gt; Any:\n\"\"\"Allows access to extra values as if they were properties of the instance.\"\"\"\n# This is here to avoid a RecursionError when copying or pickling the object\nif attr == \"_extra_values\":\nraise AttributeError()\ntry:\nreturn self._extra_values[attr]\nexcept KeyError as e:\nraise AttributeError(\nf\"{self.__class__.__name__} object has no attribute {attr}\"\n) from e\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.__iter__","title":"<code>__iter__()</code>","text":"<p>Iterate over the results returning ValueItem objects. To sort in place before iteration, use sort().</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __iter__(self) -&gt; Iterator[ValueItem[IndexT, NameT]]:\n\"\"\"Iterate over the results returning [ValueItem][pydvl.value.result.ValueItem] objects.\n    To sort in place before iteration, use [sort()][pydvl.value.result.ValuationResult.sort].\n    \"\"\"\nfor pos in self._sort_positions:\nyield ValueItem(\nself._indices[pos],\nself._names[pos],\nself._values[pos],\nself._variances[pos],\nself._counts[pos],\n)\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.__add__","title":"<code>__add__(other)</code>","text":"<p>Adds two ValuationResults.</p> <p>The values must have been computed with the same algorithm. An exception to this is if one argument has empty values, in which case the other argument is returned.</p> <p>Warning</p> <p>Abusing this will introduce numerical errors.</p> <p>Means and standard errors are correctly handled. Statuses are added with bit-wise <code>&amp;</code>, see Status. <code>data_names</code> are taken from the left summand, or if unavailable from the right one. The <code>algorithm</code> string is carried over if both terms have the same one or concatenated.</p> <p>It is possible to add ValuationResults of different lengths, and with different or overlapping indices. The result will have the union of indices, and the values.</p> <p>Warning</p> <p>FIXME: Arbitrary <code>extra_values</code> aren't handled.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def __add__(\nself, other: ValuationResult[IndexT, NameT]\n) -&gt; ValuationResult[IndexT, NameT]:\n\"\"\"Adds two ValuationResults.\n    The values must have been computed with the same algorithm. An exception\n    to this is if one argument has empty values, in which case the other\n    argument is returned.\n    !!! Warning\n        Abusing this will introduce numerical errors.\n    Means and standard errors are correctly handled. Statuses are added with\n    bit-wise `&amp;`, see [Status][pydvl.value.result.Status].\n    `data_names` are taken from the left summand, or if unavailable from\n    the right one. The `algorithm` string is carried over if both terms\n    have the same one or concatenated.\n    It is possible to add ValuationResults of different lengths, and with\n    different or overlapping indices. The result will have the union of\n    indices, and the values.\n    !!! Warning\n        FIXME: Arbitrary `extra_values` aren't handled.\n    \"\"\"\n# empty results\nif len(self.values) == 0:\nreturn other\nif len(other.values) == 0:\nreturn self\nself._check_compatible(other)\nindices = np.union1d(self._indices, other._indices).astype(self._indices.dtype)\nthis_pos = np.searchsorted(indices, self._indices)\nother_pos = np.searchsorted(indices, other._indices)\nn: NDArray[np.int_] = np.zeros_like(indices, dtype=int)\nm: NDArray[np.int_] = np.zeros_like(indices, dtype=int)\nxn: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\nxm: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\nvn: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\nvm: NDArray[np.int_] = np.zeros_like(indices, dtype=float)\nn[this_pos] = self._counts\nxn[this_pos] = self._values\nvn[this_pos] = self._variances\nm[other_pos] = other._counts\nxm[other_pos] = other._values\nvm[other_pos] = other._variances\n# Sample mean of n+m samples from two means of n and m samples\nxnm = (n * xn + m * xm) / (n + m)\n# Sample variance of n+m samples from two sample variances of n and m samples\nvnm = (n * (vn + xn**2) + m * (vm + xm**2)) / (n + m) - xnm**2\nif np.any(vnm &lt; 0):\nif np.any(vnm &lt; -1e-6):\nlogger.warning(\n\"Numerical error in variance computation. \"\nf\"Negative sample variances clipped to 0 in {vnm}\"\n)\nvnm[np.where(vnm &lt; 0)] = 0\n# Merging of names:\n# If an index has the same name in both results, it must be the same.\n# If an index has a name in one result but not the other, the name is\n# taken from the result with the name.\nif self._names.dtype != other._names.dtype:\nif np.can_cast(other._names.dtype, self._names.dtype, casting=\"safe\"):\nother._names = other._names.astype(self._names.dtype)\nlogger.warning(\nf\"Casting ValuationResult.names from {other._names.dtype} to {self._names.dtype}\"\n)\nelse:\nraise TypeError(\nf\"Cannot cast ValuationResult.names from \"\nf\"{other._names.dtype} to {self._names.dtype}\"\n)\nboth_pos = np.intersect1d(this_pos, other_pos)\nif len(both_pos) &gt; 0:\nthis_names: NDArray = np.empty_like(indices, dtype=object)\nother_names: NDArray = np.empty_like(indices, dtype=object)\nthis_names[this_pos] = self._names\nother_names[other_pos] = other._names\nthis_shared_names = np.take(this_names, both_pos)\nother_shared_names = np.take(other_names, both_pos)\nif np.any(this_shared_names != other_shared_names):\nraise ValueError(f\"Mismatching names in ValuationResults\")\nnames = np.empty_like(indices, dtype=self._names.dtype)\nnames[this_pos] = self._names\nnames[other_pos] = other._names\nreturn ValuationResult(\nalgorithm=self.algorithm or other.algorithm or \"\",\nstatus=self.status &amp; other.status,\nindices=indices,\nvalues=xnm,\nvariances=vnm,\ncounts=n + m,\ndata_names=names,\n# FIXME: What to do with extra_values? This is not commutative:\n# extra_values=self._extra_values.update(other._extra_values),\n)\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.update","title":"<code>update(idx, new_value)</code>","text":"<p>Updates the result in place with a new value, using running mean and variance.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Data index of the value to update.</p> <p> TYPE: <code>int</code> </p> <code>new_value</code> <p>New value to add to the result.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>ValuationResult[IndexT, NameT]</code> <p>A reference to the same, modified result.</p> RAISES DESCRIPTION <code>IndexError</code> <p>If the index is not found.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def update(self, idx: int, new_value: float) -&gt; ValuationResult[IndexT, NameT]:\n\"\"\"Updates the result in place with a new value, using running mean\n    and variance.\n    Args:\n        idx: Data index of the value to update.\n        new_value: New value to add to the result.\n    Returns:\n        A reference to the same, modified result.\n    Raises:\n        IndexError: If the index is not found.\n    \"\"\"\ntry:\npos = self._positions[idx]\nexcept KeyError:\nraise IndexError(f\"Index {idx} not found in ValuationResult\")\nval, var = running_moments(\nself._values[pos], self._variances[pos], self._counts[pos], new_value\n)\nself[pos] = ValueItem(\nindex=cast(IndexT, idx),  # FIXME\nname=self._names[pos],\nvalue=val,\nvariance=var,\ncount=self._counts[pos] + 1,\n)\nreturn self\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.get","title":"<code>get(idx)</code>","text":"<p>Retrieves a ValueItem by data index, as opposed to sort index, like the indexing operator.</p> RAISES DESCRIPTION <code>IndexError</code> <p>If the index is not found.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def get(self, idx: Integral) -&gt; ValueItem:\n\"\"\"Retrieves a ValueItem by data index, as opposed to sort index, like\n    the indexing operator.\n    Raises:\n         IndexError: If the index is not found.\n    \"\"\"\ntry:\npos = self._positions[idx]\nexcept KeyError:\nraise IndexError(f\"Index {idx} not found in ValuationResult\")\nreturn ValueItem(\nself._indices[pos],\nself._names[pos],\nself._values[pos],\nself._variances[pos],\nself._counts[pos],\n)\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.to_dataframe","title":"<code>to_dataframe(column=None, use_names=False)</code>","text":"<p>Returns values as a dataframe.</p> PARAMETER  DESCRIPTION <code>column</code> <p>Name for the column holding the data value. Defaults to the name of the algorithm used.</p> <p> TYPE: <code>Optional[str]</code> DEFAULT: <code>None</code> </p> <code>use_names</code> <p>Whether to use data names instead of indices for the DataFrame's index.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>DataFrame</code> <p>A dataframe with two columns, one for the values, with name given as explained in <code>column</code>, and another with standard errors for approximate algorithms. The latter will be named <code>column+'_stderr'</code>.</p> <p>Raises:      ImportError: If pandas is not installed</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>def to_dataframe(\nself, column: Optional[str] = None, use_names: bool = False\n) -&gt; pandas.DataFrame:\n\"\"\"Returns values as a dataframe.\n    Args:\n        column: Name for the column holding the data value. Defaults to\n            the name of the algorithm used.\n        use_names: Whether to use data names instead of indices for the\n            DataFrame's index.\n    Returns:\n        A dataframe with two columns, one for the values, with name\n            given as explained in `column`, and another with standard errors for\n            approximate algorithms. The latter will be named `column+'_stderr'`.\n    Raises:\n         ImportError: If pandas is not installed\n    \"\"\"\nif not pandas:\nraise ImportError(\"Pandas required for DataFrame export\")\ncolumn = column or self._algorithm\ndf = pandas.DataFrame(\nself._values[self._sort_positions],\nindex=self._names[self._sort_positions]\nif use_names\nelse self._indices[self._sort_positions],\ncolumns=[column],\n)\ndf[column + \"_stderr\"] = self.stderr[self._sort_positions]\nreturn df\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.from_random","title":"<code>from_random(size, total=None, seed=None, **kwargs)</code>  <code>classmethod</code>","text":"<p>Creates a ValuationResult object and fills it with an array of random values from a uniform distribution in [-1,1]. The values can be made to sum up to a given total number (doing so will change their range).</p> PARAMETER  DESCRIPTION <code>size</code> <p>Number of values to generate</p> <p> TYPE: <code>int</code> </p> <code>total</code> <p>If set, the values are normalized to sum to this number (\"efficiency\" property of Shapley values).</p> <p> TYPE: <code>Optional[float]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional options to pass to the constructor of ValuationResult. Use to override status, names, etc.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>'ValuationResult'</code> <p>A valuation result with its status set to</p> <code>'ValuationResult'</code> <p>Status.Converged by default.</p> RAISES DESCRIPTION <code>ValueError</code> <p>If <code>size</code> is less than 1.</p> <p>Changed in version 0.6.0</p> <p>Added parameter <code>total</code>. Check for zero size</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>@classmethod\ndef from_random(\ncls,\nsize: int,\ntotal: Optional[float] = None,\nseed: Optional[Seed] = None,\n**kwargs,\n) -&gt; \"ValuationResult\":\n\"\"\"Creates a [ValuationResult][pydvl.value.result.ValuationResult] object and fills it with an array\n    of random values from a uniform distribution in [-1,1]. The values can\n    be made to sum up to a given total number (doing so will change their range).\n    Args:\n        size: Number of values to generate\n        total: If set, the values are normalized to sum to this number\n            (\"efficiency\" property of Shapley values).\n        kwargs: Additional options to pass to the constructor of\n            [ValuationResult][pydvl.value.result.ValuationResult]. Use to override status, names, etc.\n    Returns:\n        A valuation result with its status set to\n        [Status.Converged][pydvl.utils.status.Status] by default.\n    Raises:\n         ValueError: If `size` is less than 1.\n    !!! tip \"Changed in version 0.6.0\"\n        Added parameter `total`. Check for zero size\n    \"\"\"\nif size &lt; 1:\nraise ValueError(\"Size must be a positive integer\")\nrng = np.random.default_rng(seed)\nvalues = rng.uniform(low=-1, high=1, size=size)\nif total is not None:\nvalues *= total / np.sum(values)\noptions = dict(values=values, status=Status.Converged, algorithm=\"random\")\noptions.update(kwargs)\nreturn cls(**options)  # type: ignore\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.empty","title":"<code>empty(algorithm='', indices=None, data_names=None, n_samples=0)</code>  <code>classmethod</code>","text":"<p>Creates an empty ValuationResult object.</p> <p>Empty results are characterised by having an empty array of values. When another result is added to an empty one, the empty one is discarded.</p> PARAMETER  DESCRIPTION <code>algorithm</code> <p>Name of the algorithm used to compute the values</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>@classmethod\n@deprecated(\ntarget=True,\ndeprecated_in=\"0.6.0\",\nremove_in=\"0.8.0\",\nargs_mapping=dict(indices=None, data_names=None, n_samples=None),\ntemplate_mgs=\"`%(source_name)s` is deprecated for generating zero-filled \"\n\"results, use `ValuationResult.zeros()` instead.\",\n)\ndef empty(\ncls,\nalgorithm: str = \"\",\nindices: Optional[Sequence[IndexT] | NDArray[IndexT]] = None,\ndata_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\nn_samples: int = 0,\n) -&gt; ValuationResult:\n\"\"\"Creates an empty [ValuationResult][pydvl.value.result.ValuationResult] object.\n    Empty results are characterised by having an empty array of values. When\n    another result is added to an empty one, the empty one is discarded.\n    Args:\n        algorithm: Name of the algorithm used to compute the values\n    Returns:\n        Object with the results.\n    \"\"\"\nif indices is not None or data_names is not None or n_samples != 0:\nreturn cls.zeros(\nalgorithm=algorithm,\nindices=indices,\ndata_names=data_names,\nn_samples=n_samples,\n)\nreturn cls(algorithm=algorithm, status=Status.Pending, values=np.array([]))\n</code></pre>"},{"location":"api/pydvl/value/result/#pydvl.value.result.ValuationResult.zeros","title":"<code>zeros(algorithm='', indices=None, data_names=None, n_samples=0)</code>  <code>classmethod</code>","text":"<p>Creates an empty ValuationResult object.</p> <p>Empty results are characterised by having an empty array of values. When another result is added to an empty one, the empty one is ignored.</p> PARAMETER  DESCRIPTION <code>algorithm</code> <p>Name of the algorithm used to compute the values</p> <p> TYPE: <code>str</code> DEFAULT: <code>''</code> </p> <code>indices</code> <p>Data indices to use. A copy will be made. If not given, the indices will be set to the range <code>[0, n_samples)</code>.</p> <p> TYPE: <code>Optional[Sequence[IndexT] | NDArray[IndexT]]</code> DEFAULT: <code>None</code> </p> <code>data_names</code> <p>Data names to use. A copy will be made. If not given, the names will be set to the string representation of the indices.</p> <p> TYPE: <code>Optional[Sequence[NameT] | NDArray[NameT]]</code> DEFAULT: <code>None</code> </p> <code>n_samples</code> <p>Number of data points whose values are computed. If not given, the length of <code>indices</code> will be used.</p> <p> TYPE: <code>int</code> DEFAULT: <code>0</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/result.py</code> <pre><code>@classmethod\ndef zeros(\ncls,\nalgorithm: str = \"\",\nindices: Optional[Sequence[IndexT] | NDArray[IndexT]] = None,\ndata_names: Optional[Sequence[NameT] | NDArray[NameT]] = None,\nn_samples: int = 0,\n) -&gt; ValuationResult:\n\"\"\"Creates an empty [ValuationResult][pydvl.value.result.ValuationResult] object.\n    Empty results are characterised by having an empty array of values. When\n    another result is added to an empty one, the empty one is ignored.\n    Args:\n        algorithm: Name of the algorithm used to compute the values\n        indices: Data indices to use. A copy will be made. If not given,\n            the indices will be set to the range `[0, n_samples)`.\n        data_names: Data names to use. A copy will be made. If not given,\n            the names will be set to the string representation of the indices.\n        n_samples: Number of data points whose values are computed. If\n            not given, the length of `indices` will be used.\n    Returns:\n        Object with the results.\n    \"\"\"\nif indices is None:\nindices = np.arange(n_samples, dtype=np.int_)\nelse:\nindices = np.array(indices, dtype=np.int_)\nreturn cls(\nalgorithm=algorithm,\nstatus=Status.Pending,\nindices=indices,\ndata_names=np.array(data_names, dtype=object)\nif data_names is not None\nelse np.empty_like(indices, dtype=object),\nvalues=np.zeros(len(indices)),\nvariances=np.zeros(len(indices)),\ncounts=np.zeros(len(indices), dtype=np.int_),\n)\n</code></pre>"},{"location":"api/pydvl/value/sampler/","title":"Sampler","text":"<p>Samplers iterate over subsets of indices.</p> <p>The classes in this module are used to iterate over indices and subsets of their complement in the whole set, as required for the computation of marginal utility for semi-values. The elements returned when iterating over any subclass of PowersetSampler are tuples of the form <code>(idx, subset)</code>, where <code>idx</code> is the index of the element being added to the subset, and <code>subset</code> is the subset of the complement of <code>idx</code>. The classes in this module are used to iterate over an index set \\(I\\) as required for the computation of marginal utility for semi-values. The elements returned when iterating over any subclass of :class:<code>PowersetSampler</code> are tuples of the form \\((i, S)\\), where \\(i\\) is an index of interest, and \\(S \\subset I \\setminus \\{i\\}\\) is a subset of the complement of \\(i\\).</p> <p>The iteration happens in two nested loops. An outer loop iterates over \\(I\\), and an inner loop iterates over the powerset of \\(I \\setminus \\{i\\}\\). The outer iteration can be either sequential or at random.</p> <p>Note</p> <p>This is the natural mode of iteration for the combinatorial definition of semi-values, in particular Shapley value. For the computation using permutations, adhering to this interface is not ideal, but we stick to it for consistency.</p> <p>The samplers are used in the semivalues module to compute any semi-value, in particular Shapley and Beta values, and Banzhaf indices.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler--slicing-of-samplers","title":"Slicing of samplers","text":"<p>The samplers can be sliced for parallel computation. For those which are embarrassingly parallel, this is done by slicing the set of \"outer\" indices and returning new samplers over those slices. This includes all truly powerset-based samplers, such as DeterministicUniformSampler and UniformSampler. In contrast, slicing a PermutationSampler creates a new sampler which iterates over the same indices.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler","title":"<code>PowersetSampler(indices, index_iteration=IndexIteration.Sequential, outer_indices=None)</code>","text":"<p>             Bases: <code>ABC</code>, <code>Iterable[SampleT]</code>, <code>Generic[IndexT]</code></p> <p>Samplers are custom iterables over subsets of indices.</p> <p>Calling <code>iter()</code> on a sampler returns an iterator over tuples of the form \\((i, S)\\), where \\(i\\) is an index of interest, and \\(S \\subset I \\setminus \\{i\\}\\) is a subset of the complement of \\(i\\).</p> <p>This is done in two nested loops, where the outer loop iterates over the set of indices, and the inner loop iterates over subsets of the complement of the current index. The outer iteration can be either sequential or at random.</p> <p>Note</p> <p>Samplers are not iterators themselves, so that each call to <code>iter()</code> e.g. in a for loop creates a new iterator.</p> Example <pre><code>&gt;&gt;&gt;for idx, s in DeterministicUniformSampler(np.arange(2)):\n&gt;&gt;&gt;    print(s, end=\"\")\n[][2,][][1,]\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler--methods-required-in-subclasses","title":"Methods required in subclasses","text":"<p>Samplers must implement a weight() function to be used as a multiplier in Monte Carlo sums, so that the limit expectation coincides with the semi-value.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler--slicing-of-samplers","title":"Slicing of samplers","text":"<p>The samplers can be sliced for parallel computation. For those which are embarrassingly parallel, this is done by slicing the set of \"outer\" indices and returning new samplers over those slices.</p> <pre><code>index_iteration: the order in which indices are iterated over\nouter_indices: The set of items (indices) over which to iterate\nwhen sampling. Subsets are taken from the complement of each index\nin succession. For embarrassingly parallel computations, this set\nis sliced and the samplers are used to iterate over the slices.\n</code></pre> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(\nself,\nindices: NDArray[IndexT],\nindex_iteration: IndexIteration = IndexIteration.Sequential,\nouter_indices: NDArray[IndexT] | None = None,\n):\n\"\"\"\n    Args:\n        indices: The set of items (indices) to sample from.\n        index_iteration: the order in which indices are iterated over\n        outer_indices: The set of items (indices) over which to iterate\n            when sampling. Subsets are taken from the complement of each index\n            in succession. For embarrassingly parallel computations, this set\n            is sliced and the samplers are used to iterate over the slices.\n    \"\"\"\nself._indices = indices\nself._index_iteration = index_iteration\nself._outer_indices = outer_indices if outer_indices is not None else indices\nself._n = len(indices)\nself._n_samples = 0\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler.iterindices","title":"<code>iterindices()</code>","text":"<p>Iterates over indices in the order specified at construction.</p> this is probably not very useful, but I couldn't decide <p>which method is better</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def iterindices(self) -&gt; Iterator[IndexT]:\n\"\"\"Iterates over indices in the order specified at construction.\n    FIXME: this is probably not very useful, but I couldn't decide\n      which method is better\n    \"\"\"\nif self._index_iteration is PowersetSampler.IndexIteration.Sequential:\nfor idx in self._outer_indices:\nyield idx\nelif self._index_iteration is PowersetSampler.IndexIteration.Random:\nwhile True:\nyield np.random.choice(self._outer_indices, size=1).item()\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler.__len__","title":"<code>__len__()</code>","text":"<p>Returns the number of outer indices over which the sampler iterates.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __len__(self) -&gt; int:\n\"\"\"Returns the number of outer indices over which the sampler iterates.\"\"\"\nreturn len(self._outer_indices)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PowersetSampler.weight","title":"<code>weight(n, subset_len)</code>  <code>abstractmethod</code> <code>classmethod</code>","text":"<p>Factor by which to multiply Monte Carlo samples, so that the mean converges to the desired expression.</p> <p>By the Law of Large Numbers, the sample mean of \\(\\delta_i(S_j)\\) converges to the expectation under the distribution from which \\(S_j\\) is sampled.</p> \\[ \\frac{1}{m}  \\sum_{j = 1}^m \\delta_i (S_j) c (S_j) \\longrightarrow    \\underset{S \\sim \\mathcal{D}_{- i}}{\\mathbb{E}} [\\delta_i (S) c (    S)]\\] <p>We add a factor \\(c(S_j)\\) in order to have this expectation coincide with the desired expression.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>@classmethod\n@abc.abstractmethod\ndef weight(cls, n: int, subset_len: int) -&gt; float:\nr\"\"\"Factor by which to multiply Monte Carlo samples, so that the\n    mean converges to the desired expression.\n    By the Law of Large Numbers, the sample mean of $\\delta_i(S_j)$\n    converges to the expectation under the distribution from which $S_j$ is\n    sampled.\n    $$ \\frac{1}{m}  \\sum_{j = 1}^m \\delta_i (S_j) c (S_j) \\longrightarrow\n       \\underset{S \\sim \\mathcal{D}_{- i}}{\\mathbb{E}} [\\delta_i (S) c (\n       S)]$$\n    We add a factor $c(S_j)$ in order to have this expectation coincide with\n    the desired expression.\n    \"\"\"\n...\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.StochasticSamplerMixin","title":"<code>StochasticSamplerMixin(*args, seed=None, **kwargs)</code>","text":"<p>Mixin class for samplers which use a random number generator.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, *args, seed: Optional[Seed] = None, **kwargs):\nsuper().__init__(*args, **kwargs)\nself._rng = np.random.default_rng(seed)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicUniformSampler","title":"<code>DeterministicUniformSampler(indices, *args, **kwargs)</code>","text":"<p>             Bases: <code>PowersetSampler[IndexT]</code></p> <p>For every index \\(i\\), each subset of the complement <code>indices - {i}</code> is returned.</p> <p>Note</p> <p>Indices are always iterated over sequentially, irrespective of the value of <code>index_iteration</code> upon construction.</p> Example <pre><code>&gt;&gt;&gt; for idx, s in DeterministicUniformSampler(np.arange(2)):\n&gt;&gt;&gt;    print(f\"{idx} - {s}\", end=\", \")\n1 - [], 1 - [2], 2 - [], 2 - [1],\n</code></pre> PARAMETER  DESCRIPTION <code>indices</code> <p>The set of items (indices) to sample from.</p> <p> TYPE: <code>NDArray[IndexT]</code> </p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __init__(self, indices: NDArray[IndexT], *args, **kwargs):\n\"\"\"An iterator to perform uniform deterministic sampling of subsets.\n    For every index $i$, each subset of the complement `indices - {i}` is\n    returned.\n    !!! Note\n        Indices are always iterated over sequentially, irrespective of\n        the value of `index_iteration` upon construction.\n    ??? Example\n        ``` pycon\n        &gt;&gt;&gt; for idx, s in DeterministicUniformSampler(np.arange(2)):\n        &gt;&gt;&gt;    print(f\"{idx} - {s}\", end=\", \")\n        1 - [], 1 - [2], 2 - [], 2 - [1],\n        ```\n    Args:\n        indices: The set of items (indices) to sample from.\n    \"\"\"\n# Force sequential iteration\nkwargs.update({\"index_iteration\": PowersetSampler.IndexIteration.Sequential})\nsuper().__init__(indices, *args, **kwargs)\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.UniformSampler","title":"<code>UniformSampler</code>","text":"<p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[IndexT]</code></p> <p>An iterator to perform uniform random sampling of subsets.</p> <p>Iterating over every index \\(i\\), either in sequence or at random depending on the value of <code>index_iteration</code>, one subset of the complement <code>indices - {i}</code> is sampled with equal probability \\(2^{n-1}\\). The iterator never ends.</p> Example <p>The code <pre><code>for idx, s in UniformSampler(np.arange(3)):\nprint(f\"{idx} - {s}\", end=\", \")\n</code></pre> Produces the output: <pre><code>0 - [1 4], 1 - [2 3], 2 - [0 1 3], 3 - [], 4 - [2], 0 - [1 3 4], 1 - [0 2]\n(...)\n</code></pre></p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.UniformSampler.weight","title":"<code>weight(n, subset_len)</code>  <code>classmethod</code>","text":"<p>Correction coming from Monte Carlo integration so that the mean of the marginals converges to the value: the uniform distribution over the powerset of a set with n-1 elements has mass 2^{n-1} over each subset.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>@classmethod\ndef weight(cls, n: int, subset_len: int) -&gt; float:\n\"\"\"Correction coming from Monte Carlo integration so that the mean of\n    the marginals converges to the value: the uniform distribution over the\n    powerset of a set with n-1 elements has mass 2^{n-1} over each subset.\"\"\"\nreturn float(2 ** (n - 1)) if n &gt; 0 else 1.0\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.AntitheticSampler","title":"<code>AntitheticSampler</code>","text":"<p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[IndexT]</code></p> <p>An iterator to perform uniform random sampling of subsets, and their complements.</p> <p>Works as :class:<code>~pydvl.value.sampler.UniformSampler</code>, but for every tuple \\((i,S)\\), it subsequently returns \\((i,S^c)\\), where \\(S^c\\) is the complement of the set \\(S\\), including the index \\(i\\) itself.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PermutationSampler","title":"<code>PermutationSampler</code>","text":"<p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[IndexT]</code></p> <p>Sample permutations of indices and iterate through each returning increasing subsets, as required for the permutation definition of semi-values.</p> <p>This sampler does not implement the two loops described in PowersetSampler. Instead, for a permutation <code>(3,1,4,2)</code>, it returns in sequence the tuples of index and sets:  <code>(3, {})</code>, <code>(1, {3})</code>, <code>(4, {3,1})</code> and <code>(2, {3,1,4})</code>.</p> <p>Note that the full index set is never returned.</p> <p>Warning</p> <p>This sampler requires caching to be enabled or computation will be doubled wrt. a \"direct\" implementation of permutation MC</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.PermutationSampler.__getitem__","title":"<code>__getitem__(key)</code>","text":"<p>Permutation samplers cannot be split across indices, so we return a copy of the full sampler.</p> Source code in <code>src/pydvl/value/sampler.py</code> <pre><code>def __getitem__(self, key: slice | list[int]) -&gt; PowersetSampler[IndexT]:\n\"\"\"Permutation samplers cannot be split across indices, so we return\n    a copy of the full sampler.\"\"\"\nreturn super().__getitem__(slice(None))\n</code></pre>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.DeterministicPermutationSampler","title":"<code>DeterministicPermutationSampler</code>","text":"<p>             Bases: <code>PermutationSampler[IndexT]</code></p> <p>Samples all n! permutations of the indices deterministically, and iterates through them, returning sets as required for the permutation-based definition of semi-values.</p> <p>Warning</p> <p>This sampler requires caching to be enabled or computation will be doubled wrt. a \"direct\" implementation of permutation MC</p> <p>Warning</p> <p>This sampler is not parallelizable, as it always iterates over the whole set of permutations in the same order. Different processes would always return the same values for all indices.</p>"},{"location":"api/pydvl/value/sampler/#pydvl.value.sampler.RandomHierarchicalSampler","title":"<code>RandomHierarchicalSampler</code>","text":"<p>             Bases: <code>StochasticSamplerMixin</code>, <code>PowersetSampler[IndexT]</code></p> <p>For every index, sample a set size, then a set of that size.</p> <p>Todo</p> <p>This is unnecessary, but a step towards proper stratified sampling.</p>"},{"location":"api/pydvl/value/semivalues/","title":"Semivalues","text":"<p>This module provides the core functionality for the computation of generic semi-values. A semi-value is any valuation function with the form:</p> \\[v_\\text{semi}(i) = \\sum_{i=1}^n w(k) \\sum_{S \\subset D_{-i}^{(k)}} [U(S_{+i})-U(S)],\\] <p>where the coefficients \\(w(k)\\) satisfy the property:</p> \\[\\sum_{k=1}^n w(k) = 1.\\] Note <p>For implementation consistency, we slightly depart from the common definition of semi-values, which includes a factor \\(1/n\\) in the sum over subsets. Instead, we subsume this factor into the coefficient \\(w(k)\\).</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues--main-components","title":"Main components","text":"<p>The computation of a semi-value requires two components:</p> <ol> <li>A subset sampler that generates subsets of the set \\(D\\) of interest.</li> <li>A coefficient \\(w(k)\\) that assigns a weight to each subset size \\(k\\).</li> </ol> <p>Samplers can be found in sampler, and can be classified into two categories: powerset samplers and permutation samplers. Powerset samplers generate subsets of \\(D_{-i}\\), while the permutation sampler generates permutations of \\(D\\). The former conform to the above definition of semi-values, while the latter reformulates it as:</p> \\[ v(i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} \\tilde{w}( | \\sigma_{:i} | )[U(\\sigma_{:i} \\cup \\{i\\}) \u2212 U(\\sigma_{:i})], \\] <p>where \\(\\sigma_{:i}\\) denotes the set of indices in permutation sigma before the position where \\(i\\) appears (see Data valuation for details), and</p> \\[ \\tilde{w} (k) = n \\binom{n - 1}{k} w (k) \\] <p>is the weight correction due to the reformulation.</p> <p>Warning</p> <p>Both PermutationSampler and DeterministicPermutationSampler require caching to be enabled or computation will be doubled wrt. a 'direct' implementation of permutation MC.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues--computing-semi-values","title":"Computing semi-values","text":"<p>Samplers and coefficients can be arbitrarily mixed by means of the main entry point of this module, compute_generic_semivalues. There are several pre-defined coefficients, including the Shapley value of (Ghorbani and Zou, 2019)<sup>1</sup>, the Banzhaf index of (Wang and Jia)<sup>3</sup>, and the Beta coefficient of (Kwon and Zou, 2022)<sup>2</sup>. For each of these methods, there is a convenience wrapper function. Respectively, these are: compute_shapley_semivalues, compute_banzhaf_semivalues, and compute_beta_shapley_semivalues. instead.</p> <p>Parallelization and batching</p> <p>In order to ensure reproducibility and fine-grained control of parallelization, samples are generated in the main process and then distributed to worker processes for evaluation. For small sample sizes, this can lead to a significant overhead. To avoid this, we temporarily provide an additional argument <code>batch_size</code> to all methods which can improve performance with small models up to an order of magnitude. Note that this argument will be removed before version 1.0 in favour of a more general solution.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> <li> <p>Kwon, Y. and Zou, J., 2022. Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning. In: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022, Vol. 151. PMLR, Valencia, Spain.\u00a0\u21a9</p> </li> <li> <p>Wang, J.T. and Jia, R., 2022. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning. ArXiv preprint arXiv:2205.15466.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.SVCoefficient","title":"<code>SVCoefficient</code>","text":"<p>             Bases: <code>Protocol</code></p> <p>The protocol that coefficients for the computation of semi-values must fulfill.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.SVCoefficient.__call__","title":"<code>__call__(n, k)</code>","text":"<p>Computes the coefficient for a given subset size.</p> PARAMETER  DESCRIPTION <code>n</code> <p>Total number of elements in the set.</p> <p> TYPE: <code>int</code> </p> <code>k</code> <p>Size of the subset for which the coefficient is being computed</p> <p> TYPE: <code>int</code> </p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>def __call__(self, n: int, k: int) -&gt; float:\n\"\"\"Computes the coefficient for a given subset size.\n    Args:\n        n: Total number of elements in the set.\n        k: Size of the subset for which the coefficient is being computed\n    \"\"\"\n...\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.SemiValueMode","title":"<code>SemiValueMode</code>","text":"<p>             Bases: <code>str</code>, <code>Enum</code></p> <p>Enumeration of semi-value modes.</p> <p>Deprecation notice</p> <p>This enum and the associated methods are deprecated and will be removed in 0.8.0.</p>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_generic_semivalues","title":"<code>compute_generic_semivalues(sampler, u, coefficient, done, *, batch_size=1, n_jobs=1, config=ParallelConfig(), progress=False)</code>","text":"<p>Computes semi-values for a given utility function and subset sampler.</p> PARAMETER  DESCRIPTION <code>sampler</code> <p>The subset sampler to use for utility computations.</p> <p> TYPE: <code>PowersetSampler[IndexT]</code> </p> <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>coefficient</code> <p>The semi-value coefficient</p> <p> TYPE: <code>SVCoefficient</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per single parallel job.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(\ntarget=True,\ndeprecated_in=\"0.7.0\",\nremove_in=\"0.9.0\",\nargs_mapping={\"batch_size\": None},\ntemplate_mgs=\"batch_size is for experimental use and will be removed\"\n\"in future versions.\",\n)\ndef compute_generic_semivalues(\nsampler: PowersetSampler[IndexT],\nu: Utility,\ncoefficient: SVCoefficient,\ndone: StoppingCriterion,\n*,\nbatch_size: int = 1,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\n) -&gt; ValuationResult:\n\"\"\"Computes semi-values for a given utility function and subset sampler.\n    Args:\n        sampler: The subset sampler to use for utility computations.\n        u: Utility object with model, data, and scoring function.\n        coefficient: The semi-value coefficient\n        done: Stopping criterion.\n        batch_size: Number of marginal evaluations per single parallel job.\n        n_jobs: Number of parallel jobs to use.\n        config: Object configuring parallel computation, with cluster\n            address, number of cpus, etc.\n        progress: Whether to display a progress bar.\n    Returns:\n        Object with the results.\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n    \"\"\"\nfrom concurrent.futures import FIRST_COMPLETED, Future, wait\nfrom pydvl.utils import effective_n_jobs, init_executor, init_parallel_backend\nif isinstance(sampler, PermutationSampler) and not u.enable_cache:\nlog.warning(\n\"PermutationSampler requires caching to be enabled or computation \"\n\"will be doubled wrt. a 'direct' implementation of permutation MC\"\n)\nif batch_size != 1:\nwarnings.warn(\n\"Parameter `batch_size` is for experimental use and will be\"\n\" removed in future versions\",\nDeprecationWarning,\n)\nresult = ValuationResult.zeros(\nalgorithm=f\"semivalue-{str(sampler)}-{coefficient.__name__}\",  # type: ignore\nindices=u.data.indices,\ndata_names=u.data.data_names,\n)\nparallel_backend = init_parallel_backend(config)\nu = parallel_backend.put(u)\ncorrection = parallel_backend.put(\nlambda n, k: coefficient(n, k) * sampler.weight(n, k)\n)\nmax_workers = effective_n_jobs(n_jobs, config)\nn_submitted_jobs = 2 * max_workers  # number of jobs in the queue\nsampler_it = iter(sampler)\npbar = tqdm(disable=not progress, total=100, unit=\"%\")\nwith init_executor(\nmax_workers=max_workers, config=config, cancel_futures=True\n) as executor:\npending: set[Future] = set()\nwhile True:\npbar.n = 100 * done.completion()\npbar.refresh()\ncompleted, pending = wait(pending, timeout=1, return_when=FIRST_COMPLETED)\nfor future in completed:\nfor idx, marginal in future.result():\nresult.update(idx, marginal)\nif done(result):\nreturn result\n# Ensure that we always have n_submitted_jobs running\ntry:\nfor _ in range(n_submitted_jobs - len(pending)):\nsamples = tuple(islice(sampler_it, batch_size))\nif len(samples) == 0:\nraise StopIteration\npending.add(\nexecutor.submit(\n_marginal, u=u, coefficient=correction, samples=samples\n)\n)\nexcept StopIteration:\nif len(pending) == 0:\nreturn result\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_shapley_semivalues","title":"<code>compute_shapley_semivalues(u, *, done=MaxUpdates(100), sampler_t=PermutationSampler, batch_size=1, n_jobs=1, config=ParallelConfig(), progress=False, seed=None)</code>","text":"<p>Computes Shapley values for a given utility function.</p> <p>This is a convenience wrapper for compute_generic_semivalues with the Shapley coefficient. Use compute_shapley_values for a more flexible interface and additional methods, including TMCS.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> DEFAULT: <code>MaxUpdates(100)</code> </p> <code>sampler_t</code> <p>The sampler type to use. See the sampler module for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per single parallel job.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(\ntarget=True,\ndeprecated_in=\"0.7.0\",\nremove_in=\"0.9.0\",\nargs_mapping={\"batch_size\": None},\ntemplate_mgs=\"batch_size is for experimental use and will be removed\"\n\"in future versions.\",\n)\ndef compute_shapley_semivalues(\nu: Utility,\n*,\ndone: StoppingCriterion = MaxUpdates(100),\nsampler_t: Type[StochasticSampler] = PermutationSampler,\nbatch_size: int = 1,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\nseed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n\"\"\"Computes Shapley values for a given utility function.\n    This is a convenience wrapper for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    with the Shapley coefficient. Use\n    [compute_shapley_values][pydvl.value.shapley.common.compute_shapley_values]\n    for a more flexible interface and additional methods, including TMCS.\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: Stopping criterion.\n        sampler_t: The sampler type to use. See the\n            [sampler][pydvl.value.sampler] module for a list.\n        batch_size: Number of marginal evaluations per single parallel job.\n        n_jobs: Number of parallel jobs to use.\n        config: Object configuring parallel computation, with cluster\n            address, number of cpus, etc.\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n        progress: Whether to display a progress bar.\n    Returns:\n        Object with the results.\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n    \"\"\"\n# HACK: cannot infer return type because of useless IndexT, NameT\nreturn compute_generic_semivalues(  # type: ignore\nsampler_t(u.data.indices, seed=seed),\nu,\nshapley_coefficient,\ndone,\nbatch_size=batch_size,\nn_jobs=n_jobs,\nconfig=config,\nprogress=progress,\n)\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_banzhaf_semivalues","title":"<code>compute_banzhaf_semivalues(u, *, done=MaxUpdates(100), sampler_t=PermutationSampler, batch_size=1, n_jobs=1, config=ParallelConfig(), progress=False, seed=None)</code>","text":"<p>Computes Banzhaf values for a given utility function.</p> <p>This is a convenience wrapper for compute_generic_semivalues with the Banzhaf coefficient.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> DEFAULT: <code>MaxUpdates(100)</code> </p> <code>sampler_t</code> <p>The sampler type to use. See the sampler module for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per single parallel job.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(\ntarget=True,\ndeprecated_in=\"0.7.0\",\nremove_in=\"0.9.0\",\nargs_mapping={\"batch_size\": None},\ntemplate_mgs=\"batch_size is for experimental use and will be removed\"\n\"in future versions.\",\n)\ndef compute_banzhaf_semivalues(\nu: Utility,\n*,\ndone: StoppingCriterion = MaxUpdates(100),\nsampler_t: Type[StochasticSampler] = PermutationSampler,\nbatch_size: int = 1,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\nseed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n\"\"\"Computes Banzhaf values for a given utility function.\n    This is a convenience wrapper for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    with the Banzhaf coefficient.\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: Stopping criterion.\n        sampler_t: The sampler type to use. See the\n            [sampler][pydvl.value.sampler] module for a list.\n        batch_size: Number of marginal evaluations per single parallel job.\n        n_jobs: Number of parallel jobs to use.\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        progress: Whether to display a progress bar.\n    Returns:\n        Object with the results.\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n    \"\"\"\n# HACK: cannot infer return type because of useless IndexT, NameT\nreturn compute_generic_semivalues(  # type: ignore\nsampler_t(u.data.indices, seed=seed),\nu,\nbanzhaf_coefficient,\ndone,\nbatch_size=batch_size,\nn_jobs=n_jobs,\nconfig=config,\nprogress=progress,\n)\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_beta_shapley_semivalues","title":"<code>compute_beta_shapley_semivalues(u, *, alpha=1, beta=1, done=MaxUpdates(100), sampler_t=PermutationSampler, batch_size=1, n_jobs=1, config=ParallelConfig(), progress=False, seed=None)</code>","text":"<p>Computes Beta Shapley values for a given utility function.</p> <p>This is a convenience wrapper for compute_generic_semivalues with the Beta Shapley coefficient.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>alpha</code> <p>Alpha parameter of the Beta distribution.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1</code> </p> <code>beta</code> <p>Beta parameter of the Beta distribution.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> DEFAULT: <code>MaxUpdates(100)</code> </p> <code>sampler_t</code> <p>The sampler type to use. See the sampler module for a list.</p> <p> TYPE: <code>Type[StochasticSampler]</code> DEFAULT: <code>PermutationSampler</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per (parallelized) task.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(\ntarget=True,\ndeprecated_in=\"0.7.0\",\nremove_in=\"0.9.0\",\nargs_mapping={\"batch_size\": None},\ntemplate_mgs=\"batch_size is for experimental use and will be removed\"\n\"in future versions.\",\n)\ndef compute_beta_shapley_semivalues(\nu: Utility,\n*,\nalpha: float = 1,\nbeta: float = 1,\ndone: StoppingCriterion = MaxUpdates(100),\nsampler_t: Type[StochasticSampler] = PermutationSampler,\nbatch_size: int = 1,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\nseed: Optional[Seed] = None,\n) -&gt; ValuationResult:\n\"\"\"Computes Beta Shapley values for a given utility function.\n    This is a convenience wrapper for\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    with the Beta Shapley coefficient.\n    Args:\n        u: Utility object with model, data, and scoring function.\n        alpha: Alpha parameter of the Beta distribution.\n        beta: Beta parameter of the Beta distribution.\n        done: Stopping criterion.\n        sampler_t: The sampler type to use. See the\n            [sampler][pydvl.value.sampler] module for a list.\n        batch_size: Number of marginal evaluations per (parallelized) task.\n        n_jobs: Number of parallel jobs to use.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n        config: Object configuring parallel computation, with cluster address, number of\n            cpus, etc.\n        progress: Whether to display a progress bar.\n    Returns:\n        Object with the results.\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n    \"\"\"\n# HACK: cannot infer return type because of useless IndexT, NameT\nreturn compute_generic_semivalues(  # type: ignore\nsampler_t(u.data.indices, seed=seed),\nu,\nbeta_coefficient(alpha, beta),\ndone,\nbatch_size=batch_size,\nn_jobs=n_jobs,\nconfig=config,\nprogress=progress,\n)\n</code></pre>"},{"location":"api/pydvl/value/semivalues/#pydvl.value.semivalues.compute_semivalues","title":"<code>compute_semivalues(u, *, done=MaxUpdates(100), mode=SemiValueMode.Shapley, sampler_t=PermutationSampler[IndexT], batch_size=1, n_jobs=1, seed=None, **kwargs)</code>","text":"<p>Convenience entry point for most common semi-value computations.</p> <p>Deprecation warning</p> <p>This method is deprecated and will be replaced in 0.8.0 by the more general implementation of compute_generic_semivalues. Use compute_shapley_semivalues, compute_banzhaf_semivalues, or compute_beta_shapley_semivalues instead.</p> <p>The modes supported with this interface are the following. For greater flexibility use compute_generic_semivalues directly.</p> <ul> <li>SemiValueMode.Shapley:   Shapley values.</li> <li>SemiValueMode.BetaShapley:   Implements the Beta Shapley semi-value as introduced in   (Kwon and Zou, 2022)<sup>1</sup>.   Pass additional keyword arguments <code>alpha</code> and <code>beta</code> to set the   parameters of the Beta distribution (both default to 1).</li> <li>SemiValueMode.Banzhaf: Implements the Banzhaf   semi-value as introduced in (Wang and Jia, 2022)<sup>1</sup>.</li> </ul> <p>See [[data-valuation]] for an overview of valuation. - SemiValueMode.Banzhaf: Implements   the Banzhaf semi-value as introduced in [@wang_data_2022].</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion.</p> <p> TYPE: <code>StoppingCriterion</code> DEFAULT: <code>MaxUpdates(100)</code> </p> <code>mode</code> <p>The semi-value mode to use. See SemiValueMode for a list.</p> <p> TYPE: <code>SemiValueMode</code> DEFAULT: <code>Shapley</code> </p> <code>sampler_t</code> <p>The sampler type to use. See sampler for a list.</p> <p> TYPE: <code>Type[StochasticSampler[IndexT]]</code> DEFAULT: <code>PermutationSampler[IndexT]</code> </p> <code>batch_size</code> <p>Number of marginal evaluations per (parallelized) task.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>kwargs</code> <p>Additional keyword arguments passed to compute_generic_semivalues.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> <p>Deprecation notice</p> <p>Parameter <code>batch_size</code> is for experimental use and will be removed in future versions.</p> Source code in <code>src/pydvl/value/semivalues.py</code> <pre><code>@deprecated(target=True, deprecated_in=\"0.7.0\", remove_in=\"0.8.0\")\ndef compute_semivalues(\nu: Utility,\n*,\ndone: StoppingCriterion = MaxUpdates(100),\nmode: SemiValueMode = SemiValueMode.Shapley,\nsampler_t: Type[StochasticSampler[IndexT]] = PermutationSampler[IndexT],\nbatch_size: int = 1,\nn_jobs: int = 1,\nseed: Optional[Seed] = None,\n**kwargs,\n) -&gt; ValuationResult:\n\"\"\"Convenience entry point for most common semi-value computations.\n    !!! warning \"Deprecation warning\"\n        This method is deprecated and will be replaced in 0.8.0 by the more\n        general implementation of\n        [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].\n        Use\n        [compute_shapley_semivalues][pydvl.value.semivalues.compute_shapley_semivalues],\n        [compute_banzhaf_semivalues][pydvl.value.semivalues.compute_banzhaf_semivalues],\n        or\n        [compute_beta_shapley_semivalues][pydvl.value.semivalues.compute_beta_shapley_semivalues]\n        instead.\n    The modes supported with this interface are the following. For greater\n    flexibility use\n    [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues]\n    directly.\n    - [SemiValueMode.Shapley][pydvl.value.semivalues.SemiValueMode]:\n      Shapley values.\n    - [SemiValueMode.BetaShapley][pydvl.value.semivalues.SemiValueMode.BetaShapley]:\n      Implements the Beta Shapley semi-value as introduced in\n      (Kwon and Zou, 2022)&lt;sup&gt;&lt;a href=\"#kwon_beta_2022\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n      Pass additional keyword arguments `alpha` and `beta` to set the\n      parameters of the Beta distribution (both default to 1).\n    - [SemiValueMode.Banzhaf][SemiValueMode.Banzhaf]: Implements the Banzhaf\n      semi-value as introduced in (Wang and Jia, 2022)&lt;sup&gt;&lt;a href=\"#wang_data_2022\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    See [[data-valuation]] for an overview of valuation.\n    - [SemiValueMode.Banzhaf][pydvl.value.semivalues.SemiValueMode]: Implements\n      the Banzhaf semi-value as introduced in [@wang_data_2022].\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: Stopping criterion.\n        mode: The semi-value mode to use. See\n            [SemiValueMode][pydvl.value.semivalues.SemiValueMode] for a list.\n        sampler_t: The sampler type to use. See [sampler][pydvl.value.sampler]\n            for a list.\n        batch_size: Number of marginal evaluations per (parallelized) task.\n        n_jobs: Number of parallel jobs to use.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n        kwargs: Additional keyword arguments passed to\n            [compute_generic_semivalues][pydvl.value.semivalues.compute_generic_semivalues].\n    Returns:\n        Object with the results.\n    !!! warning \"Deprecation notice\"\n        Parameter `batch_size` is for experimental use and will be removed in\n        future versions.\n    \"\"\"\nif mode == SemiValueMode.Shapley:\ncoefficient = shapley_coefficient\nelif mode == SemiValueMode.BetaShapley:\nalpha = kwargs.pop(\"alpha\", 1)\nbeta = kwargs.pop(\"beta\", 1)\ncoefficient = beta_coefficient(alpha, beta)\nelif mode == SemiValueMode.Banzhaf:\ncoefficient = banzhaf_coefficient\nelse:\nraise ValueError(f\"Unknown mode {mode}\")\ncoefficient = cast(SVCoefficient, coefficient)\n# HACK: cannot infer return type because of useless IndexT, NameT\nreturn compute_generic_semivalues(  # type: ignore\nsampler_t(u.data.indices, seed=seed),\nu,\ncoefficient,\ndone,\nn_jobs=n_jobs,\nbatch_size=batch_size,\n**kwargs,\n)\n</code></pre>"},{"location":"api/pydvl/value/stopping/","title":"Stopping","text":"<p>Stopping criteria for value computations.</p> <p>This module provides a basic set of stopping criteria, like MaxUpdates, MaxTime, or HistoryDeviation among others. These can behave in different ways depending on the context. For example, MaxUpdates limits the number of updates to values, which depending on the algorithm may mean a different number of utility evaluations or imply other computations like solving a linear or quadratic program.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--creating-stopping-criteria","title":"Creating stopping criteria","text":"<p>The easiest way is to declare a function implementing the interface StoppingCriterionCallable and wrap it with make_criterion(). This creates a StoppingCriterion object that can be composed with other stopping criteria.</p> <p>Alternatively, and in particular if reporting of completion is required, one can inherit from this class and implement the abstract methods _check and completion.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--composing-stopping-criteria","title":"Composing stopping criteria","text":"<p>Objects of type StoppingCriterion can be composed with the binary operators <code>&amp;</code> (and), and <code>|</code> (or), following the truth tables of Status. The unary operator <code>~</code> (not) is also supported. See StoppingCriterion for details on how these operations affect the behavior of the stopping criteria.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterionCallable","title":"<code>StoppingCriterionCallable</code>","text":"<p>             Bases: <code>Protocol</code></p> <p>Signature for a stopping criterion</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion","title":"<code>StoppingCriterion(modify_result=True)</code>","text":"<p>             Bases: <code>ABC</code></p> <p>A composable callable object to determine whether a computation must stop.</p> <p>A <code>StoppingCriterion</code> is a callable taking a ValuationResult and returning a Status. It also keeps track of individual convergence of values with converged, and reports the overall completion of the computation with completion.</p> <p>Instances of <code>StoppingCriterion</code> can be composed with the binary operators <code>&amp;</code> (and), and <code>|</code> (or), following the truth tables of Status. The unary operator <code>~</code> (not) is also supported. These boolean operations act according to the following rules:</p> <ul> <li>The results of _check are   combined with the operator. See Status for   the truth tables.</li> <li>The results of   converged are combined   with the operator (returning another boolean array).</li> <li>The completion   method returns the min, max, or the complement to 1 of the completions of   the operands, for AND, OR and NOT respectively. This is required for cases   where one of the criteria does not keep track of the convergence of single   values, e.g. MaxUpdates, because   completion by   default returns the mean of the boolean convergence array.</li> </ul>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion--subclassing","title":"Subclassing","text":"<p>Subclassing this class requires implementing a _check method that returns a Status object based on a given ValuationResult. This method should update the attribute _converged, which is a boolean array indicating whether the value for each index has converged. When this does not make sense for a particular stopping criterion, completion should be overridden to provide an overall completion value, since its default implementation attempts to compute the mean of _converged.</p> PARAMETER  DESCRIPTION <code>modify_result</code> <p>If <code>True</code> the status of the input ValuationResult is modified in place after the call.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, modify_result: bool = True):\nself.modify_result = modify_result\nself._converged = np.full(0, False)\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion.converged","title":"<code>converged: NDArray[np.bool_]</code>  <code>property</code>","text":"<p>Returns a boolean array indicating whether the values have converged for each data point.</p> <p>Inheriting classes must set the <code>_converged</code> attribute in their _check.</p> RETURNS DESCRIPTION <code>NDArray[bool_]</code> <p>A boolean array indicating whether the values have converged for</p> <code>NDArray[bool_]</code> <p>each data point.</p>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion.completion","title":"<code>completion()</code>","text":"<p>Returns a value between 0 and 1 indicating the completion of the computation.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def completion(self) -&gt; float:\n\"\"\"Returns a value between 0 and 1 indicating the completion of the\n    computation.\n    \"\"\"\nif self.converged.size == 0:\nreturn 0.0\nreturn float(np.mean(self.converged).item())\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.StoppingCriterion.__call__","title":"<code>__call__(result)</code>","text":"<p>Calls _check, maybe updating the result.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __call__(self, result: ValuationResult) -&gt; Status:\n\"\"\"Calls [_check][pydvl.value.stopping.StoppingCriterion._check], maybe updating the result.\"\"\"\nif len(result) == 0:\nlogger.warning(\n\"At least one iteration finished but no results where generated. \"\n\"Please check that your scorer and utility return valid numbers.\"\n)\nstatus = self._check(result)\nif self.modify_result:  # FIXME: this is not nice\nresult._status = status\nreturn status\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.AbsoluteStandardError","title":"<code>AbsoluteStandardError(threshold, fraction=1.0, burn_in=4, modify_result=True)</code>","text":"<p>             Bases: <code>StoppingCriterion</code></p> <p>Determine convergence based on the standard error of the values.</p> <p>If \\(s_i\\) is the standard error for datum \\(i\\) and \\(v_i\\) its value, then this criterion returns Converged if \\(s_i &lt; \\epsilon\\) for all \\(i\\) and a threshold value \\(\\epsilon \\gt 0\\).</p> PARAMETER  DESCRIPTION <code>threshold</code> <p>A value is considered to have converged if the standard error is below this value. A way of choosing it is to pick some percentage of the range of the values. For Shapley values this is the difference between the maximum and minimum of the utility function (to see this substitute the maximum and minimum values of the utility into the marginal contribution formula).</p> <p> TYPE: <code>float</code> </p> <code>fraction</code> <p>The fraction of values that must have converged for the criterion to return Converged.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1.0</code> </p> <code>burn_in</code> <p>The number of iterations to ignore before checking for convergence. This is required because computations typically start with zero variance, as a result of using empty(). The default is set to an arbitrary minimum which is usually enough but may need to be increased.</p> <p> TYPE: <code>int</code> DEFAULT: <code>4</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(\nself,\nthreshold: float,\nfraction: float = 1.0,\nburn_in: int = 4,\nmodify_result: bool = True,\n):\nsuper().__init__(modify_result=modify_result)\nself.threshold = threshold\nself.fraction = fraction\nself.burn_in = burn_in\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxChecks","title":"<code>MaxChecks(n_checks, modify_result=True)</code>","text":"<p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate as soon as the number of checks exceeds the threshold.</p> <p>A \"check\" is one call to the criterion.</p> PARAMETER  DESCRIPTION <code>n_checks</code> <p>Threshold: if <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[int]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, n_checks: Optional[int], modify_result: bool = True):\nsuper().__init__(modify_result=modify_result)\nif n_checks is not None and n_checks &lt; 1:\nraise ValueError(\"n_iterations must be at least 1 or None\")\nself.n_checks = n_checks\nself._count = 0\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxUpdates","title":"<code>MaxUpdates(n_updates, modify_result=True)</code>","text":"<p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate if any number of value updates exceeds or equals the given threshold.</p> <p>Note</p> <p>If you want to ensure that all values have been updated, you probably want MinUpdates instead.</p> <p>This checks the <code>counts</code> field of a ValuationResult, i.e. the number of times that each index has been updated. For powerset samplers, the maximum of this number coincides with the maximum number of subsets sampled. For permutation samplers, it coincides with the number of permutations sampled.</p> PARAMETER  DESCRIPTION <code>n_updates</code> <p>Threshold: if <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[int]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, n_updates: Optional[int], modify_result: bool = True):\nsuper().__init__(modify_result=modify_result)\nif n_updates is not None and n_updates &lt; 1:\nraise ValueError(\"n_updates must be at least 1 or None\")\nself.n_updates = n_updates\nself.last_max = 0\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MinUpdates","title":"<code>MinUpdates(n_updates, modify_result=True)</code>","text":"<p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate as soon as all value updates exceed or equal the given threshold.</p> <p>This checks the <code>counts</code> field of a ValuationResult, i.e. the number of times that each index has been updated. For powerset samplers, the minimum of this number is a lower bound for the number of subsets sampled. For permutation samplers, it lower-bounds the amount of permutations sampled.</p> PARAMETER  DESCRIPTION <code>n_updates</code> <p>Threshold: if <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[int]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, n_updates: Optional[int], modify_result: bool = True):\nsuper().__init__(modify_result=modify_result)\nself.n_updates = n_updates\nself.last_min = 0\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.MaxTime","title":"<code>MaxTime(seconds, modify_result=True)</code>","text":"<p>             Bases: <code>StoppingCriterion</code></p> <p>Terminate if the computation time exceeds the given number of seconds.</p> <p>Checks the elapsed time since construction</p> PARAMETER  DESCRIPTION <code>seconds</code> <p>Threshold: The computation is terminated if the elapsed time between object construction and a _check exceeds this value. If <code>None</code>, no _check is performed, effectively creating a (never) stopping criterion that always returns <code>Pending</code>.</p> <p> TYPE: <code>Optional[float]</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(self, seconds: Optional[float], modify_result: bool = True):\nsuper().__init__(modify_result=modify_result)\nself.max_seconds = seconds or np.inf\nif self.max_seconds &lt;= 0:\nraise ValueError(\"Number of seconds for MaxTime must be positive or None\")\nself.start = time()\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.HistoryDeviation","title":"<code>HistoryDeviation(n_steps, rtol, pin_converged=True, modify_result=True)</code>","text":"<p>             Bases: <code>StoppingCriterion</code></p> <p>A simple check for relative distance to a previous step in the computation.</p> <p>The method used by (Ghorbani and Zou, 2019)<sup>1</sup> computes the relative distances between the current values \\(v_i^t\\) and the values at the previous checkpoint \\(v_i^{t-\\tau}\\). If the sum is below a given threshold, the computation is terminated.</p> \\[\\sum_{i=1}^n \\frac{\\left| v_i^t - v_i^{t-\\tau} \\right|}{v_i^t} &lt; \\epsilon.\\] <p>When the denominator is zero, the summand is set to the value of \\(v_i^{ t-\\tau}\\).</p> <p>This implementation is slightly generalised to allow for different number of updates to individual indices, as happens with powerset samplers instead of permutations. Every subset of indices that is found to converge can be pinned to that state. Once all indices have converged the method has converged.</p> <p>Warning</p> <p>This criterion is meant for the reproduction of the results in the paper, but we do not recommend using it in practice.</p> PARAMETER  DESCRIPTION <code>n_steps</code> <p>Checkpoint values every so many updates and use these saved values to compare.</p> <p> TYPE: <code>int</code> </p> <code>rtol</code> <p>Relative tolerance for convergence (\\(\\epsilon\\) in the formula).</p> <p> TYPE: <code>float</code> </p> <code>pin_converged</code> <p>If <code>True</code>, once an index has converged, it is pinned</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def __init__(\nself,\nn_steps: int,\nrtol: float,\npin_converged: bool = True,\nmodify_result: bool = True,\n):\nsuper().__init__(modify_result=modify_result)\nif n_steps &lt; 1:\nraise ValueError(\"n_steps must be at least 1\")\nif rtol &lt;= 0 or rtol &gt;= 1:\nraise ValueError(\"rtol must be in (0, 1)\")\nself.n_steps = n_steps\nself.rtol = rtol\nself.update_op = np.logical_or if pin_converged else np.logical_and\nself._memory = None  # type: ignore\n</code></pre>"},{"location":"api/pydvl/value/stopping/#pydvl.value.stopping.make_criterion","title":"<code>make_criterion(fun, converged=None, completion=None, name=None)</code>","text":"<p>Create a new StoppingCriterion from a function. Use this to enable simpler functions to be composed with bitwise operators</p> PARAMETER  DESCRIPTION <code>fun</code> <p>The callable to wrap.</p> <p> TYPE: <code>StoppingCriterionCallable</code> </p> <code>converged</code> <p>A callable that returns a boolean array indicating what values have converged.</p> <p> TYPE: <code>Callable[[], NDArray[bool_]] | None</code> DEFAULT: <code>None</code> </p> <code>completion</code> <p>A callable that returns a value between 0 and 1 indicating the rate of completion of the computation. If not provided, the fraction of converged values is used.</p> <p> TYPE: <code>Callable[[], float] | None</code> DEFAULT: <code>None</code> </p> <code>name</code> <p>The name of the new criterion. If <code>None</code>, the <code>__name__</code> of the function is used.</p> <p> TYPE: <code>str | None</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>Type[StoppingCriterion]</code> <p>A new subclass of StoppingCriterion.</p> Source code in <code>src/pydvl/value/stopping.py</code> <pre><code>def make_criterion(\nfun: StoppingCriterionCallable,\nconverged: Callable[[], NDArray[np.bool_]] | None = None,\ncompletion: Callable[[], float] | None = None,\nname: str | None = None,\n) -&gt; Type[StoppingCriterion]:\n\"\"\"Create a new [StoppingCriterion][pydvl.value.stopping.StoppingCriterion] from a function.\n    Use this to enable simpler functions to be composed with bitwise operators\n    Args:\n        fun: The callable to wrap.\n        converged: A callable that returns a boolean array indicating what\n            values have converged.\n        completion: A callable that returns a value between 0 and 1 indicating\n            the rate of completion of the computation. If not provided, the fraction\n            of converged values is used.\n        name: The name of the new criterion. If `None`, the `__name__` of\n            the function is used.\n    Returns:\n        A new subclass of [StoppingCriterion][pydvl.value.stopping.StoppingCriterion].\n    \"\"\"\nclass WrappedCriterion(StoppingCriterion):\ndef __init__(self, modify_result: bool = True):\nsuper().__init__(modify_result=modify_result)\nself._name = name or getattr(fun, \"__name__\", \"WrappedCriterion\")\ndef _check(self, result: ValuationResult) -&gt; Status:\nreturn fun(result)\n@property\ndef converged(self) -&gt; NDArray[np.bool_]:\nif converged is None:\nreturn super().converged\nreturn converged()\n@property\ndef name(self):\nreturn self._name\ndef completion(self) -&gt; float:\nif completion is None:\nreturn super().completion()\nreturn completion()\nreturn WrappedCriterion\n</code></pre>"},{"location":"api/pydvl/value/least_core/","title":"Least core","text":"<p>New in version 0.4.0</p> <p>This package holds all routines for the computation of Least Core data values.</p> <p>Please refer to Data valuation for an overview.</p> <p>In addition to the standard interface via compute_least_core_values(), because computing the Least Core values requires the solution of a linear and a quadratic problem after computing all the utility values, there is the possibility of performing each step separately. This is useful when running multiple experiments: use lc_prepare_problem() or mclc_prepare_problem() to prepare a list of problems to solve, then solve them in parallel with lc_solve_problems().</p> <p>Note that mclc_prepare_problem() is parallelized itself, so preparing the problems should be done in sequence in this case. The solution of the linear systems can then be done in parallel.</p>"},{"location":"api/pydvl/value/least_core/#pydvl.value.least_core.LeastCoreMode","title":"<code>LeastCoreMode</code>","text":"<p>             Bases: <code>Enum</code></p> <p>Available Least Core algorithms.</p>"},{"location":"api/pydvl/value/least_core/#pydvl.value.least_core.compute_least_core_values","title":"<code>compute_least_core_values(u, *, n_jobs=1, n_iterations=None, mode=LeastCoreMode.MonteCarlo, non_negative_subsidy=False, solver_options=None, **kwargs)</code>","text":"<p>Umbrella method to compute Least Core values with any of the available algorithms.</p> <p>See Data valuation for an overview.</p> <p>The following algorithms are available. Note that the exact method can only work with very small datasets and is thus intended only for testing.</p> <ul> <li><code>exact</code>: uses the complete powerset of the training set for the constraints   combinatorial_exact_shapley().</li> <li><code>montecarlo</code>:  uses the approximate Monte Carlo Least Core algorithm.   Implemented in montecarlo_least_core().</li> </ul> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_jobs</code> <p>Number of jobs to run in parallel. Only used for Monte Carlo Least Core.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>n_iterations</code> <p>Number of subsets to sample and evaluate the utility on. Only used for Monte Carlo Least Core.</p> <p> TYPE: <code>Optional[int]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>Algorithm to use. See LeastCoreMode for available options.</p> <p> TYPE: <code>LeastCoreMode</code> DEFAULT: <code>MonteCarlo</code> </p> <code>non_negative_subsidy</code> <p>If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>solver_options</code> <p>Optional dictionary of options passed to the solvers.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the computed values.</p> <p>New in version 0.5.0</p> Source code in <code>src/pydvl/value/least_core/__init__.py</code> <pre><code>def compute_least_core_values(\nu: Utility,\n*,\nn_jobs: int = 1,\nn_iterations: Optional[int] = None,\nmode: LeastCoreMode = LeastCoreMode.MonteCarlo,\nnon_negative_subsidy: bool = False,\nsolver_options: Optional[dict] = None,\n**kwargs,\n) -&gt; ValuationResult:\n\"\"\"Umbrella method to compute Least Core values with any of the available\n    algorithms.\n    See [Data valuation][computing-data-values] for an overview.\n    The following algorithms are available. Note that the exact method can only\n    work with very small datasets and is thus intended only for testing.\n    - `exact`: uses the complete powerset of the training set for the constraints\n      [combinatorial_exact_shapley()][pydvl.value.shapley.naive.combinatorial_exact_shapley].\n    - `montecarlo`:  uses the approximate Monte Carlo Least Core algorithm.\n      Implemented in [montecarlo_least_core()][pydvl.value.least_core.montecarlo.montecarlo_least_core].\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_jobs: Number of jobs to run in parallel. Only used for Monte Carlo\n            Least Core.\n        n_iterations: Number of subsets to sample and evaluate the utility on.\n            Only used for Monte Carlo Least Core.\n        mode: Algorithm to use. See\n            [LeastCoreMode][pydvl.value.least_core.LeastCoreMode] for available\n            options.\n        non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Optional dictionary of options passed to the solvers.\n    Returns:\n        Object with the computed values.\n    !!! tip \"New in version 0.5.0\"\n    \"\"\"\nprogress: bool = kwargs.pop(\"progress\", False)\n# TODO: remove this before releasing version 0.7.0\nif kwargs:\nwarnings.warn(\nDeprecationWarning(\n\"Passing solver options as kwargs was deprecated in 0.6.0, will \"\n\"be removed in 0.7.0. `Use solver_options` instead.\"\n)\n)\nif solver_options is None:\nsolver_options = kwargs\nelse:\nsolver_options.update(kwargs)\nif mode == LeastCoreMode.MonteCarlo:\n# TODO fix progress showing and maybe_progress in remote case\nprogress = False\nif n_iterations is None:\nraise ValueError(\"n_iterations cannot be None for Monte Carlo Least Core\")\nreturn montecarlo_least_core(\nu=u,\nn_iterations=n_iterations,\nn_jobs=n_jobs,\nprogress=progress,\nnon_negative_subsidy=non_negative_subsidy,\nsolver_options=solver_options,\n**kwargs,\n)\nelif mode == LeastCoreMode.Exact:\nreturn exact_least_core(\nu=u,\nprogress=progress,\nnon_negative_subsidy=non_negative_subsidy,\nsolver_options=solver_options,\n)\nraise ValueError(f\"Invalid value encountered in {mode=}\")\n</code></pre>"},{"location":"api/pydvl/value/least_core/common/","title":"Common","text":""},{"location":"api/pydvl/value/least_core/common/#pydvl.value.least_core.common.lc_solve_problem","title":"<code>lc_solve_problem(problem, *, u, algorithm, non_negative_subsidy=False, solver_options=None, **options)</code>","text":"<p>Solves a linear problem as prepared by mclc_prepare_problem(). Useful for parallel execution of multiple experiments by running this as a remote task.</p> <p>See exact_least_core() or montecarlo_least_core() for argument descriptions.</p> Source code in <code>src/pydvl/value/least_core/common.py</code> <pre><code>def lc_solve_problem(\nproblem: LeastCoreProblem,\n*,\nu: Utility,\nalgorithm: str,\nnon_negative_subsidy: bool = False,\nsolver_options: Optional[dict] = None,\n**options,\n) -&gt; ValuationResult:\n\"\"\"Solves a linear problem as prepared by\n    [mclc_prepare_problem()][pydvl.value.least_core.montecarlo.mclc_prepare_problem].\n    Useful for parallel execution of multiple experiments by running this as a\n    remote task.\n    See [exact_least_core()][pydvl.value.least_core.naive.exact_least_core] or\n    [montecarlo_least_core()][pydvl.value.least_core.montecarlo.montecarlo_least_core] for\n    argument descriptions.\n    \"\"\"\nn = len(u.data)\nif np.any(np.isnan(problem.utility_values)):\nwarnings.warn(\nf\"Calculation returned \"\nf\"{np.sum(np.isnan(problem.utility_values))} NaN \"\nf\"values out of {problem.utility_values.size}\",\nRuntimeWarning,\n)\n# TODO: remove this before releasing version 0.7.0\nif options:\nwarnings.warn(\nDeprecationWarning(\n\"Passing solver options as kwargs was deprecated in \"\n\"0.6.0, will be removed in 0.7.0. `Use solver_options` \"\n\"instead.\"\n)\n)\nif solver_options is None:\nsolver_options = options\nelse:\nsolver_options.update(options)\nif solver_options is None:\nsolver_options = {}\nif \"solver\" not in solver_options:\nsolver_options[\"solver\"] = cp.SCS\nif \"max_iters\" not in solver_options and solver_options[\"solver\"] == cp.SCS:\nsolver_options[\"max_iters\"] = 10000\nlogger.debug(\"Removing possible duplicate values in lower bound array\")\nb_lb = problem.utility_values\nA_lb, unique_indices = np.unique(problem.A_lb, return_index=True, axis=0)\nb_lb = b_lb[unique_indices]\nlogger.debug(\"Building equality constraint\")\nA_eq = np.ones((1, n))\n# We might have already computed the total utility one or more times.\n# This is the index of the row(s) in A_lb with all ones.\ntotal_utility_indices = np.where(A_lb.sum(axis=1) == n)[0]\nif len(total_utility_indices) == 0:\nb_eq = np.array([u(u.data.indices)])\nelse:\nb_eq = b_lb[total_utility_indices]\n# Remove the row(s) corresponding to the total utility\n# from the lower bound constraints\n# because given the equality constraint\n# it is the same as using the constraint e &gt;= 0\n# (i.e. setting non_negative_subsidy = True).\nmask: NDArray[np.bool_] = np.ones_like(b_lb, dtype=bool)\nmask[total_utility_indices] = False\nb_lb = b_lb[mask]\nA_lb = A_lb[mask]\n# Remove the row(s) corresponding to the empty subset\n# because, given u(\u2205) = (which is almost always the case,\n# it is the same as using the constraint e &gt;= 0\n# (i.e. setting non_negative_subsidy = True).\nemptyset_utility_indices = np.where(A_lb.sum(axis=1) == 0)[0]\nif len(emptyset_utility_indices) &gt; 0:\nmask = np.ones_like(b_lb, dtype=bool)\nmask[emptyset_utility_indices] = False\nb_lb = b_lb[mask]\nA_lb = A_lb[mask]\n_, subsidy = _solve_least_core_linear_program(\nA_eq=A_eq,\nb_eq=b_eq,\nA_lb=A_lb,\nb_lb=b_lb,\nnon_negative_subsidy=non_negative_subsidy,\nsolver_options=solver_options,\n)\nvalues: Optional[NDArray[np.float_]]\nif subsidy is None:\nlogger.debug(\"No values were found\")\nstatus = Status.Failed\nvalues = np.empty(n)\nvalues[:] = np.nan\nsubsidy = np.nan\nelse:\nvalues = _solve_egalitarian_least_core_quadratic_program(\nsubsidy,\nA_eq=A_eq,\nb_eq=b_eq,\nA_lb=A_lb,\nb_lb=b_lb,\nsolver_options=solver_options,\n)\nif values is None:\nlogger.debug(\"No values were found\")\nstatus = Status.Failed\nvalues = np.empty(n)\nvalues[:] = np.nan\nsubsidy = np.nan\nelse:\nstatus = Status.Converged\nreturn ValuationResult(\nalgorithm=algorithm,\nstatus=status,\nvalues=values,\nsubsidy=subsidy,\nstderr=None,\ndata_names=u.data.data_names,\n)\n</code></pre>"},{"location":"api/pydvl/value/least_core/common/#pydvl.value.least_core.common.lc_solve_problems","title":"<code>lc_solve_problems(problems, u, algorithm, config=ParallelConfig(), n_jobs=1, non_negative_subsidy=True, solver_options=None, **options)</code>","text":"<p>Solves a list of linear problems in parallel.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility.</p> <p> TYPE: <code>Utility</code> </p> <code>problems</code> <p>Least Core problems to solve, as returned by mclc_prepare_problem().</p> <p> TYPE: <code>Sequence[LeastCoreProblem]</code> </p> <code>algorithm</code> <p>Name of the valuation algorithm.</p> <p> TYPE: <code>str</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to run.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>non_negative_subsidy</code> <p>If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>solver_options</code> <p>Additional options to pass to the solver.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>List[ValuationResult]</code> <p>List of solutions.</p> Source code in <code>src/pydvl/value/least_core/common.py</code> <pre><code>def lc_solve_problems(\nproblems: Sequence[LeastCoreProblem],\nu: Utility,\nalgorithm: str,\nconfig: ParallelConfig = ParallelConfig(),\nn_jobs: int = 1,\nnon_negative_subsidy: bool = True,\nsolver_options: Optional[dict] = None,\n**options,\n) -&gt; List[ValuationResult]:\n\"\"\"Solves a list of linear problems in parallel.\n    Args:\n        u: Utility.\n        problems: Least Core problems to solve, as returned by\n            [mclc_prepare_problem()][pydvl.value.least_core.montecarlo.mclc_prepare_problem].\n        algorithm: Name of the valuation algorithm.\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        n_jobs: Number of parallel jobs to run.\n        non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Additional options to pass to the solver.\n    Returns:\n        List of solutions.\n    \"\"\"\ndef _map_func(\nproblems: List[LeastCoreProblem], *args, **kwargs\n) -&gt; List[ValuationResult]:\nreturn [lc_solve_problem(p, *args, **kwargs) for p in problems]\nmap_reduce_job: MapReduceJob[\n\"LeastCoreProblem\", \"List[ValuationResult]\"\n] = MapReduceJob(\ninputs=problems,\nmap_func=_map_func,\nmap_kwargs=dict(\nu=u,\nalgorithm=algorithm,\nnon_negative_subsidy=non_negative_subsidy,\nsolver_options=solver_options,\n**options,\n),\nreduce_func=lambda x: list(itertools.chain(*x)),\nconfig=config,\nn_jobs=n_jobs,\n)\nsolutions = map_reduce_job()\nreturn solutions\n</code></pre>"},{"location":"api/pydvl/value/least_core/montecarlo/","title":"Montecarlo","text":""},{"location":"api/pydvl/value/least_core/montecarlo/#pydvl.value.least_core.montecarlo.montecarlo_least_core","title":"<code>montecarlo_least_core(u, n_iterations, *, n_jobs=1, config=ParallelConfig(), non_negative_subsidy=False, solver_options=None, options=None, progress=False)</code>","text":"<p>Computes approximate Least Core values using a Monte Carlo approach.</p> \\[ \\begin{array}{lll} \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\ \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\ &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp; , \\forall S \\in \\{S_1, S_2, \\dots, S_m \\overset{\\mathrm{iid}}{\\sim} U(2^N) \\} \\end{array} \\] <p>Where:</p> <ul> <li>\\(U(2^N)\\) is the uniform distribution over the powerset of \\(N\\).</li> <li>\\(m\\) is the number of subsets that will be sampled and whose utility will   be computed and used to compute the data values.</li> </ul> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_iterations</code> <p>total number of iterations to use</p> <p> TYPE: <code>int</code> </p> <code>n_jobs</code> <p>number of jobs across which to distribute the computation</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>non_negative_subsidy</code> <p>If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>solver_options</code> <p>Dictionary of options that will be used to select a solver and to configure it. Refer to cvxpy's documentation for all possible options.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> <code>options</code> <p>(Deprecated) Dictionary of solver options. Use solver_options instead.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, shows a tqdm progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values and the least core value.</p> Source code in <code>src/pydvl/value/least_core/montecarlo.py</code> <pre><code>def montecarlo_least_core(\nu: Utility,\nn_iterations: int,\n*,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nnon_negative_subsidy: bool = False,\nsolver_options: Optional[dict] = None,\noptions: Optional[dict] = None,\nprogress: bool = False,\n) -&gt; ValuationResult:\nr\"\"\"Computes approximate Least Core values using a Monte Carlo approach.\n    $$\n    \\begin{array}{lll}\n    \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\\n    \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\\n    &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp; ,\n    \\forall S \\in \\{S_1, S_2, \\dots, S_m \\overset{\\mathrm{iid}}{\\sim} U(2^N) \\}\n    \\end{array}\n    $$\n    Where:\n    * $U(2^N)$ is the uniform distribution over the powerset of $N$.\n    * $m$ is the number of subsets that will be sampled and whose utility will\n      be computed and used to compute the data values.\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_iterations: total number of iterations to use\n        n_jobs: number of jobs across which to distribute the computation\n        config: Object configuring parallel computation, with cluster\n            address, number of cpus, etc.\n        non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Dictionary of options that will be used to select a solver\n            and to configure it. Refer to [cvxpy's\n            documentation](https://www.cvxpy.org/tutorial/advanced/index.html#setting-solver-options)\n            for all possible options.\n        options: (Deprecated) Dictionary of solver options. Use solver_options\n            instead.\n        progress: If True, shows a tqdm progress bar\n    Returns:\n        Object with the data values and the least core value.\n    \"\"\"\n# TODO: remove this before releasing version 0.7.0\nif options:\nwarnings.warn(\nDeprecationWarning(\n\"Passing solver options as kwargs was deprecated in \"\n\"0.6.0, will be removed in 0.7.0. `Use solver_options` \"\n\"instead.\"\n)\n)\nif solver_options is None:\nsolver_options = options\nelse:\nsolver_options.update(options)\nproblem = mclc_prepare_problem(\nu, n_iterations, n_jobs=n_jobs, config=config, progress=progress\n)\nreturn lc_solve_problem(\nproblem,\nu=u,\nalgorithm=\"montecarlo_least_core\",\nnon_negative_subsidy=non_negative_subsidy,\nsolver_options=solver_options,\n)\n</code></pre>"},{"location":"api/pydvl/value/least_core/montecarlo/#pydvl.value.least_core.montecarlo.mclc_prepare_problem","title":"<code>mclc_prepare_problem(u, n_iterations, *, n_jobs=1, config=ParallelConfig(), progress=False)</code>","text":"<p>Prepares a linear problem by sampling subsets of the data. Use this to separate the problem preparation from the solving with lc_solve_problem(). Useful for parallel execution of multiple experiments.</p> <p>See montecarlo_least_core for argument descriptions.</p> Source code in <code>src/pydvl/value/least_core/montecarlo.py</code> <pre><code>def mclc_prepare_problem(\nu: Utility,\nn_iterations: int,\n*,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\n) -&gt; LeastCoreProblem:\n\"\"\"Prepares a linear problem by sampling subsets of the data. Use this to\n    separate the problem preparation from the solving with\n    [lc_solve_problem()][pydvl.value.least_core.common.lc_solve_problem]. Useful\n    for parallel execution of multiple experiments.\n    See\n    [montecarlo_least_core][pydvl.value.least_core.montecarlo.montecarlo_least_core]\n    for argument descriptions.\n    \"\"\"\nn = len(u.data)\nif n_iterations &lt; n:\nwarnings.warn(\nf\"Number of iterations '{n_iterations}' is smaller the size of the dataset '{n}'. \"\nf\"This is not optimal because in the worst case we need at least '{n}' constraints \"\n\"to satisfy the individual rationality condition.\"\n)\nif n_iterations &gt; 2**n:\nwarnings.warn(\nf\"Passed n_iterations is greater than the number subsets! \"\nf\"Setting it to 2^{n}\",\nRuntimeWarning,\n)\nn_iterations = 2**n\niterations_per_job = max(1, n_iterations // effective_n_jobs(n_jobs, config))\nmap_reduce_job: MapReduceJob[\"Utility\", \"LeastCoreProblem\"] = MapReduceJob(\ninputs=u,\nmap_func=_montecarlo_least_core,\nreduce_func=_reduce_func,\nmap_kwargs=dict(n_iterations=iterations_per_job, progress=progress),\nn_jobs=n_jobs,\nconfig=config,\n)\nreturn map_reduce_job()\n</code></pre>"},{"location":"api/pydvl/value/least_core/naive/","title":"Naive","text":""},{"location":"api/pydvl/value/least_core/naive/#pydvl.value.least_core.naive.exact_least_core","title":"<code>exact_least_core(u, *, non_negative_subsidy=False, solver_options=None, options=None, progress=True)</code>","text":"<p>Computes the exact Least Core values.</p> <p>Note</p> <p>If the training set contains more than 20 instances a warning is printed because the computation is very expensive. This method is mostly used for internal testing and simple use cases. Please refer to the Monte Carlo method for practical applications.</p> <p>The least core is the solution to the following Linear Programming problem:</p> \\[ \\begin{array}{lll} \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\ \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\ &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp;, \\forall S \\subseteq N \\\\ \\end{array} \\] <p>Where \\(N = \\{1, 2, \\dots, n\\}\\) are the training set's indices.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function non_negative_subsidy: If True, the least core subsidy \\(e\\) is constrained to be non-negative.</p> <p> TYPE: <code>Utility</code> </p> <code>solver_options</code> <p>Dictionary of options that will be used to select a solver and to configure it. Refer to the cvxpy's documentation for all possible options.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> <code>options</code> <p>(Deprecated) Dictionary of solver options. Use <code>solver_options</code> instead.</p> <p> TYPE: <code>Optional[dict]</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, shows a tqdm progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values and the least core value.</p> Source code in <code>src/pydvl/value/least_core/naive.py</code> <pre><code>def exact_least_core(\nu: Utility,\n*,\nnon_negative_subsidy: bool = False,\nsolver_options: Optional[dict] = None,\noptions: Optional[dict] = None,\nprogress: bool = True,\n) -&gt; ValuationResult:\nr\"\"\"Computes the exact Least Core values.\n    !!! Note\n        If the training set contains more than 20 instances a warning is printed\n        because the computation is very expensive. This method is mostly used for\n        internal testing and simple use cases. Please refer to the\n        [Monte Carlo method][pydvl.value.least_core.montecarlo.montecarlo_least_core]\n        for practical applications.\n    The least core is the solution to the following Linear Programming problem:\n    $$\n    \\begin{array}{lll}\n    \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\\n    \\text{subject to} &amp; \\displaystyle\\sum_{i\\in N} x_{i} = v(N) &amp; \\\\\n    &amp; \\displaystyle\\sum_{i\\in S} x_{i} + e \\geq v(S) &amp;, \\forall S \\subseteq N \\\\\n    \\end{array}\n    $$\n    Where $N = \\{1, 2, \\dots, n\\}$ are the training set's indices.\n    Args:\n        u: Utility object with model, data, and scoring function\n            non_negative_subsidy: If True, the least core subsidy $e$ is constrained\n            to be non-negative.\n        solver_options: Dictionary of options that will be used to select a solver\n            and to configure it. Refer to the [cvxpy's\n            documentation](https://www.cvxpy.org/tutorial/advanced/index.html#setting-solver-options)\n            for all possible options.\n        options: (Deprecated) Dictionary of solver options. Use `solver_options`\n            instead.\n        progress: If True, shows a tqdm progress bar\n    Returns:\n        Object with the data values and the least core value.\n    \"\"\"\nn = len(u.data)\nif n &gt; 20:  # Arbitrary choice, will depend on time required, caching, etc.\nwarnings.warn(f\"Large dataset! Computation requires 2^{n} calls to model.fit()\")\n# TODO: remove this before releasing version 0.7.0\nif options:\nwarnings.warn(\nDeprecationWarning(\n\"Passing solver options as kwargs was deprecated in \"\n\"0.6.0, will \"\n\"be removed in 0.7.0. `Use solver_options` instead.\"\n)\n)\nif solver_options is None:\nsolver_options = options\nelse:\nsolver_options.update(options)\nproblem = lc_prepare_problem(u, progress=progress)\nreturn lc_solve_problem(\nproblem=problem,\nu=u,\nalgorithm=\"exact_least_core\",\nnon_negative_subsidy=non_negative_subsidy,\nsolver_options=solver_options,\n)\n</code></pre>"},{"location":"api/pydvl/value/least_core/naive/#pydvl.value.least_core.naive.lc_prepare_problem","title":"<code>lc_prepare_problem(u, progress=False)</code>","text":"<p>Prepares a linear problem with all subsets of the data Use this to separate the problem preparation from the solving with lc_solve_problem(). Useful for parallel execution of multiple experiments.</p> <p>See exact_least_core() for argument descriptions.</p> Source code in <code>src/pydvl/value/least_core/naive.py</code> <pre><code>def lc_prepare_problem(u: Utility, progress: bool = False) -&gt; LeastCoreProblem:\n\"\"\"Prepares a linear problem with all subsets of the data\n    Use this to separate the problem preparation from the solving with\n    [lc_solve_problem()][pydvl.value.least_core.common.lc_solve_problem]. Useful for\n    parallel execution of multiple experiments.\n    See [exact_least_core()][pydvl.value.least_core.naive.exact_least_core] for argument\n    descriptions.\n    \"\"\"\nn = len(u.data)\nlogger.debug(\"Building vectors and matrices for linear programming problem\")\npowerset_size = 2**n\nA_lb = np.zeros((powerset_size, n))\nlogger.debug(\"Iterating over all subsets\")\nutility_values = np.zeros(powerset_size)\nfor i, subset in enumerate(\nmaybe_progress(\npowerset(u.data.indices), progress, total=powerset_size - 1, position=0\n)\n):\nindices: NDArray[np.bool_] = np.zeros(n, dtype=bool)\nindices[list(subset)] = True\nA_lb[i, indices] = 1\nutility_values[i] = u(subset)\nreturn LeastCoreProblem(utility_values, A_lb)\n</code></pre>"},{"location":"api/pydvl/value/loo/","title":"Loo","text":""},{"location":"api/pydvl/value/loo/loo/","title":"Loo","text":""},{"location":"api/pydvl/value/loo/loo/#pydvl.value.loo.loo.compute_loo","title":"<code>compute_loo(u, *, n_jobs=1, config=ParallelConfig(), progress=True)</code>","text":"<p>Computes leave one out value:</p> \\[v(i) = u(D) - u(D \\setminus \\{i\\}) \\] PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>progress</code> <p>If True, display a progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>If True, display a progress bar</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>New in version 0.7.0</p> <p>Renamed from <code>naive_loo</code> and added parallel computation.</p> Source code in <code>src/pydvl/value/loo/loo.py</code> <pre><code>def compute_loo(\nu: Utility,\n*,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = True,\n) -&gt; ValuationResult:\nr\"\"\"Computes leave one out value:\n    $$v(i) = u(D) - u(D \\setminus \\{i\\}) $$\n    Args:\n        u: Utility object with model, data, and scoring function\n        progress: If True, display a progress bar\n        n_jobs: Number of parallel jobs to use\n        config: Object configuring parallel computation, with cluster\n            address, number of cpus, etc.\n        progress: If True, display a progress bar\n    Returns:\n        Object with the data values.\n    !!! tip \"New in version 0.7.0\"\n        Renamed from `naive_loo` and added parallel computation.\n    \"\"\"\nif len(u.data) &lt; 3:\nraise ValueError(\"Dataset must have at least 2 elements\")\nresult = ValuationResult.zeros(\nalgorithm=\"loo\",\nindices=u.data.indices,\ndata_names=u.data.data_names,\n)\nall_indices = set(u.data.indices)\ntotal_utility = u(u.data.indices)\ndef fun(idx: int) -&gt; tuple[int, float]:\nreturn idx, total_utility - u(all_indices.difference({idx}))\nmax_workers = effective_n_jobs(n_jobs, config)\nn_submitted_jobs = 2 * max_workers  # number of jobs in the queue\n# NOTE: this could be done with a simple executor.map(), but we want to\n# display a progress bar\nwith init_executor(\nmax_workers=max_workers, config=config, cancel_futures=True\n) as executor:\npending: set[Future] = set()\nindex_it = iter(u.data.indices)\npbar = tqdm(disable=not progress, total=100, unit=\"%\")\nwhile True:\npbar.n = 100 * sum(result.counts) / len(u.data)\npbar.refresh()\ncompleted, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED)\nfor future in completed:\nidx, marginal = future.result()\nresult.update(idx, marginal)\n# Ensure that we always have n_submitted_jobs running\ntry:\nfor _ in range(n_submitted_jobs - len(pending)):\npending.add(executor.submit(fun, next(index_it)))\nexcept StopIteration:\nif len(pending) == 0:\nreturn result\n</code></pre>"},{"location":"api/pydvl/value/loo/naive/","title":"Naive","text":""},{"location":"api/pydvl/value/loo/naive/#pydvl.value.loo.naive.naive_loo","title":"<code>naive_loo(u, *, progress=True, **kwargs)</code>","text":"<p>Deprecated. Use compute_loo instead.</p> Source code in <code>src/pydvl/value/loo/naive.py</code> <pre><code>@deprecated(\ntarget=compute_loo,\ndeprecated_in=\"0.7.0\",\nremove_in=\"0.8.0\",\nargs_extra=dict(n_jobs=1),\n)\ndef naive_loo(u: Utility, *, progress: bool = True, **kwargs) -&gt; ValuationResult:\n\"\"\"Deprecated. Use [compute_loo][pydvl.value.loo.compute_loo] instead.\"\"\"\npass  # type: ignore\n</code></pre>"},{"location":"api/pydvl/value/oob/","title":"Oob","text":""},{"location":"api/pydvl/value/oob/oob/","title":"Oob","text":""},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob--references","title":"References","text":"<ol> <li> <p>Kwon et al. Data-OOB: Out-of-bag Estimate as a Simple and Efficient Data Value. In: Published at ICML 2023\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob.compute_data_oob","title":"<code>compute_data_oob(u, n_est=10, max_samples=0.8, n_jobs=None, loss=None, *, progress=False)</code>","text":"<p>Computes Data out of bag values</p> <p>This implements the method described in (Kwon and Zou, 2023) <sup>1</sup>. It fits several base estimators provided through u.model through a bagging process. The point value corresponds to the average loss of estimators which were not fit on it.</p> <p>\\(w_{bj}\\in Z\\) is the number of times the j-th datum \\((x_j, y_j)\\) is selected in the b-th bootstrap dataset.</p> \\[\\psi((x_i,y_i),\\Theta_B):=\\frac{\\sum_{b=1}^{B}\\mathbb{1}(w_{bi}=0)T(y_i, \\hat{f}_b(x_i))}{\\sum_{b=1}^{B} \\mathbb{1} (w_{bi}=0)}\\] <p>With:</p> \\[ T: Y \\times Y \\rightarrow \\mathbb{R} \\] <p>T is a score function that represents the goodness of a weak learner \\(\\hat{f}_b\\) at the i-th datum \\((x_i, y_i)\\).</p> <p>There is a need to tune n_est and max_samples jointly to ensure all samples are at least 1 time oob, otherwise the result could include a nan value for that datum.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>n_est</code> <p>Number of estimator used in the bagging procedure.</p> <p> TYPE: <code>int</code> DEFAULT: <code>10</code> </p> <code>max_samples</code> <p>The fraction of samples to draw to train each base estimator.</p> <p> TYPE: <code>float</code> DEFAULT: <code>0.8</code> </p> <code>n_jobs</code> <p>The number of jobs to run in parallel used in the bagging procedure for both fit and predict.</p> <p> TYPE: <code>int</code> DEFAULT: <code>None</code> </p> <code>loss</code> <p>A function taking as parameters model prediction and corresponding data labels(preds, y) and returning an array of point-wise errors.</p> <p> TYPE: <code>Callable</code> DEFAULT: <code>None</code> </p> <code>progress</code> <p>If True, display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> Source code in <code>src/pydvl/value/oob/oob.py</code> <pre><code>def compute_data_oob(\nu: Utility,\nn_est: int = 10,\nmax_samples: float = 0.8,\nn_jobs: int = None,\nloss: Callable = None,\n*,\nprogress: bool = False,\n) -&gt; ValuationResult:\nr\"\"\"Computes Data out of bag values\n    This implements the method described in (Kwon and Zou, 2023) &lt;sup&gt;&lt;a href=\"kwon_data_2023\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    It fits several base estimators provided through u.model through a bagging process. The point value corresponds to the average loss of estimators which were not fit on it.\n    $w_{bj}\\in Z$ is the number of times the j-th datum $(x_j, y_j)$ is selected in the b-th bootstrap dataset.\n    $$\\psi((x_i,y_i),\\Theta_B):=\\frac{\\sum_{b=1}^{B}\\mathbb{1}(w_{bi}=0)T(y_i, \\hat{f}_b(x_i))}{\\sum_{b=1}^{B}\n    \\mathbb{1}\n    (w_{bi}=0)}$$\n    With:\n    $$\n    T: Y \\times Y\n    \\rightarrow \\mathbb{R}\n    $$\n    T is a score function that represents the goodness of a weak learner $\\hat{f}_b$ at the i-th datum $(x_i, y_i)$.\n    There is a need to tune n_est and max_samples jointly to ensure all samples are at least 1 time oob, otherwise the result could include a nan value for that datum.\n    Args:\n        u: Utility object with model, data, and scoring function.\n        n_est: Number of estimator used in the bagging procedure.\n        max_samples: The fraction of samples to draw to train each base estimator.\n        n_jobs: The number of jobs to run in parallel used in the bagging\n            procedure for both fit and predict.\n        loss: A function taking as parameters model prediction and corresponding\n            data labels(preds, y) and returning an array of point-wise errors.\n        progress: If True, display a progress bar.\n    Returns:\n        Object with the data values.\n    \"\"\"\nresult: ValuationResult[np.int_, np.object_] = ValuationResult.empty(\nalgorithm=\"data_oob\", indices=u.data.indices, data_names=u.data.data_names\n)\nif is_classifier(u.model):\nbag = BaggingClassifier(\nu.model, n_estimators=n_est, max_samples=max_samples, n_jobs=n_jobs\n)\nif loss is None:\nloss = point_wise_accuracy\nelif is_regressor(u.model):\nbag = BaggingRegressor(\nu.model, n_estimators=n_est, max_samples=max_samples, n_jobs=n_jobs\n)\nif loss is None:\nloss = neg_l2_distance\nelse:\nraise Exception(\n\"Model has to be a classifier or a regressor in sklearn format.\"\n)\nbag.fit(u.data.x_train, u.data.y_train)\nfor est, samples in maybe_progress(\nzip(bag.estimators_, bag.estimators_samples_), progress, total=n_est\n):  # The bottleneck is the bag fitting not this part so TQDM is not very useful here\noob_idx = np.setxor1d(u.data.indices, np.unique(samples))\narray_loss = loss(\npreds=est.predict(u.data.x_train[oob_idx]), y=u.data.y_train[oob_idx]\n)\nresult += ValuationResult(\nalgorithm=\"data_oob\",\nindices=oob_idx,\nvalues=array_loss,\ncounts=np.ones_like(array_loss, dtype=u.data.indices.dtype),\n)\nreturn result\n</code></pre>"},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob.point_wise_accuracy","title":"<code>point_wise_accuracy(preds, y)</code>","text":"<p>Computes point wise accuracy</p> PARAMETER  DESCRIPTION <code>preds</code> <p>Model prediction on</p> <p> TYPE: <code>NDArray</code> </p> <code>y</code> <p>data labels corresponding to the model predictions</p> <p> TYPE: <code>NDArray</code> </p> RETURNS DESCRIPTION <code>NDArray</code> <p>Array of point wise accuracy</p> Source code in <code>src/pydvl/value/oob/oob.py</code> <pre><code>def point_wise_accuracy(preds: NDArray, y: NDArray) -&gt; NDArray:\nr\"\"\"Computes point wise accuracy\n    Args:\n        preds: Model prediction on\n        y:  data labels corresponding to the model predictions\n    Returns:\n        Array of point wise accuracy\n    \"\"\"\nreturn np.array(preds == y, dtype=np.int_)\n</code></pre>"},{"location":"api/pydvl/value/oob/oob/#pydvl.value.oob.oob.neg_l2_distance","title":"<code>neg_l2_distance(preds, y)</code>","text":"<p>Computes negative l2 distance between label and model prediction</p> PARAMETER  DESCRIPTION <code>preds</code> <p>Model prediction on</p> <p> TYPE: <code>NDArray[T]</code> </p> <code>y</code> <p>data labels corresponding to the model predictions</p> <p> TYPE: <code>NDArray[T]</code> </p> RETURNS DESCRIPTION <code>NDArray[T]</code> <p>Array with point wise negative l2 distance between label and model prediction</p> Source code in <code>src/pydvl/value/oob/oob.py</code> <pre><code>def neg_l2_distance(preds: NDArray[T], y: NDArray[T]) -&gt; NDArray[T]:\nr\"\"\"Computes negative l2 distance between label and model prediction\n    Args:\n        preds: Model prediction on\n        y:  data labels corresponding to the model predictions\n    Returns:\n        Array with point wise negative l2 distance between label and model prediction\n    \"\"\"\nreturn -np.square(\nnp.array(\npreds - y,\ndtype=np.float64,\n)\n)\n</code></pre>"},{"location":"api/pydvl/value/shapley/","title":"Shapley","text":"<p>This package holds all routines for the computation of Shapley Data value. Users will want to use compute_shapley_values or compute_semivalues as interfaces to most methods defined in the modules.</p> <p>Please refer to the guide on data valuation for an overview of all methods.</p>"},{"location":"api/pydvl/value/shapley/common/","title":"Common","text":""},{"location":"api/pydvl/value/shapley/common/#pydvl.value.shapley.common.compute_shapley_values","title":"<code>compute_shapley_values(u, *, done=MaxUpdates(100), mode=ShapleyMode.TruncatedMontecarlo, n_jobs=1, seed=None, **kwargs)</code>","text":"<p>Umbrella method to compute Shapley values with any of the available algorithms.</p> <p>See [[data-valuation]] for an overview.</p> <p>The following algorithms are available. Note that the exact methods can only work with very small datasets and are thus intended only for testing. Some algorithms also accept additional arguments, please refer to the documentation of each particular method.</p> <ul> <li><code>combinatorial_exact</code>: uses the combinatorial implementation of data   Shapley. Implemented in   combinatorial_exact_shapley().</li> <li><code>combinatorial_montecarlo</code>:  uses the approximate Monte Carlo   implementation of combinatorial data Shapley. Implemented in   combinatorial_montecarlo_shapley().</li> <li><code>permutation_exact</code>: uses the permutation-based implementation of data   Shapley. Computation is not parallelized. Implemented in   permutation_exact_shapley().</li> <li><code>permutation_montecarlo</code>: uses the approximate Monte Carlo   implementation of permutation data Shapley. Accepts a   TruncationPolicy to stop   computing marginals. Implemented in   permutation_montecarlo_shapley().</li> <li><code>owen_sampling</code>: Uses the Owen continuous extension of the utility   function to the unit cube. Implemented in   owen_sampling_shapley(). This   method does not take a StoppingCriterion   but instead requires a parameter <code>q_max</code> for the number of subdivisions   of the unit interval to use for integration, and another parameter   <code>n_samples</code> for the number of subsets to sample for each \\(q\\).</li> <li><code>owen_halved</code>: Same as 'owen_sampling' but uses correlated samples in the   expectation. Implemented in   owen_sampling_shapley().   This method  requires an additional parameter <code>q_max</code> for the number of   subdivisions of the interval [0,0.5] to use for integration, and another   parameter <code>n_samples</code> for the number of subsets to sample for each \\(q\\).</li> <li><code>group_testing</code>: estimates differences of Shapley values and solves a   constraint satisfaction problem. High sample complexity, not recommended.   Implemented in group_testing_shapley(). This   method does not take a StoppingCriterion   but instead requires a parameter <code>n_samples</code> for the number of   iterations to run.</li> </ul> <p>Additionally, one can use model-specific methods:</p> <ul> <li><code>knn</code>: Exact method for K-Nearest neighbour models. Implemented in   knn_shapley().</li> </ul> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Object used to determine when to stop the computation for Monte Carlo methods. The default is to stop after 100 iterations. See the available criteria in stopping. It is possible to combine several of them using boolean operators. Some methods ignore this argument, others require specific subtypes.</p> <p> TYPE: <code>StoppingCriterion</code> DEFAULT: <code>MaxUpdates(100)</code> </p> <code>n_jobs</code> <p>Number of parallel jobs (available only to some methods)</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>mode</code> <p>Choose which shapley algorithm to use. See ShapleyMode for a list of allowed value.</p> <p> TYPE: <code>ShapleyMode</code> DEFAULT: <code>TruncatedMontecarlo</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the results.</p> Source code in <code>src/pydvl/value/shapley/common.py</code> <pre><code>def compute_shapley_values(\nu: Utility,\n*,\ndone: StoppingCriterion = MaxUpdates(100),\nmode: ShapleyMode = ShapleyMode.TruncatedMontecarlo,\nn_jobs: int = 1,\nseed: Optional[Seed] = None,\n**kwargs,\n) -&gt; ValuationResult:\n\"\"\"Umbrella method to compute Shapley values with any of the available\n    algorithms.\n    See [[data-valuation]] for an overview.\n    The following algorithms are available. Note that the exact methods can only\n    work with very small datasets and are thus intended only for testing. Some\n    algorithms also accept additional arguments, please refer to the\n    documentation of each particular method.\n    - `combinatorial_exact`: uses the combinatorial implementation of data\n      Shapley. Implemented in\n      [combinatorial_exact_shapley()][pydvl.value.shapley.naive.combinatorial_exact_shapley].\n    - `combinatorial_montecarlo`:  uses the approximate Monte Carlo\n      implementation of combinatorial data Shapley. Implemented in\n      [combinatorial_montecarlo_shapley()][pydvl.value.shapley.montecarlo.combinatorial_montecarlo_shapley].\n    - `permutation_exact`: uses the permutation-based implementation of data\n      Shapley. Computation is **not parallelized**. Implemented in\n      [permutation_exact_shapley()][pydvl.value.shapley.naive.permutation_exact_shapley].\n    - `permutation_montecarlo`: uses the approximate Monte Carlo\n      implementation of permutation data Shapley. Accepts a\n      [TruncationPolicy][pydvl.value.shapley.truncated.TruncationPolicy] to stop\n      computing marginals. Implemented in\n      [permutation_montecarlo_shapley()][pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley].\n    - `owen_sampling`: Uses the Owen continuous extension of the utility\n      function to the unit cube. Implemented in\n      [owen_sampling_shapley()][pydvl.value.shapley.owen.owen_sampling_shapley]. This\n      method does not take a [StoppingCriterion][pydvl.value.stopping.StoppingCriterion]\n      but instead requires a parameter `q_max` for the number of subdivisions\n      of the unit interval to use for integration, and another parameter\n      `n_samples` for the number of subsets to sample for each $q$.\n    - `owen_halved`: Same as 'owen_sampling' but uses correlated samples in the\n      expectation. Implemented in\n      [owen_sampling_shapley()][pydvl.value.shapley.owen.owen_sampling_shapley].\n      This method  requires an additional parameter `q_max` for the number of\n      subdivisions of the interval [0,0.5] to use for integration, and another\n      parameter `n_samples` for the number of subsets to sample for each $q$.\n    - `group_testing`: estimates differences of Shapley values and solves a\n      constraint satisfaction problem. High sample complexity, not recommended.\n      Implemented in [group_testing_shapley()][pydvl.value.shapley.gt.group_testing_shapley]. This\n      method does not take a [StoppingCriterion][pydvl.value.stopping.StoppingCriterion]\n      but instead requires a parameter `n_samples` for the number of\n      iterations to run.\n    Additionally, one can use model-specific methods:\n    - `knn`: Exact method for K-Nearest neighbour models. Implemented in\n      [knn_shapley()][pydvl.value.shapley.knn.knn_shapley].\n    Args:\n        u: [Utility][pydvl.utils.utility.Utility] object with model, data, and\n            scoring function.\n        done: Object used to determine when to stop the computation for Monte\n            Carlo methods. The default is to stop after 100 iterations. See the\n            available criteria in [stopping][pydvl.value.stopping]. It is\n            possible to combine several of them using boolean operators. Some\n            methods ignore this argument, others require specific subtypes.\n        n_jobs: Number of parallel jobs (available only to some methods)\n        seed: Either an instance of a numpy random number generator or a seed\n            for it.\n        mode: Choose which shapley algorithm to use. See\n            [ShapleyMode][pydvl.value.shapley.ShapleyMode] for a list of allowed\n            value.\n    Returns:\n        Object with the results.\n    \"\"\"\nprogress: bool = kwargs.pop(\"progress\", False)\nif mode not in list(ShapleyMode):\nraise ValueError(f\"Invalid value encountered in {mode=}\")\nif mode in (\nShapleyMode.PermutationMontecarlo,\nShapleyMode.ApproShapley,\nShapleyMode.TruncatedMontecarlo,\n):\ntruncation = kwargs.pop(\"truncation\", NoTruncation())\nreturn permutation_montecarlo_shapley(  # type: ignore\nu=u, done=done, truncation=truncation, n_jobs=n_jobs, seed=seed, **kwargs\n)\nelif mode == ShapleyMode.CombinatorialMontecarlo:\nreturn combinatorial_montecarlo_shapley(\nu, done=done, n_jobs=n_jobs, seed=seed, progress=progress\n)\nelif mode == ShapleyMode.CombinatorialExact:\nreturn combinatorial_exact_shapley(u, n_jobs=n_jobs, progress=progress)\nelif mode == ShapleyMode.PermutationExact:\nreturn permutation_exact_shapley(u, progress=progress)\nelif mode == ShapleyMode.Owen or mode == ShapleyMode.OwenAntithetic:\nif kwargs.get(\"n_samples\") is None:\nraise ValueError(\"n_samples cannot be None for Owen methods\")\nif kwargs.get(\"max_q\") is None:\nraise ValueError(\"Owen Sampling requires max_q for the outer integral\")\nmethod = (\nOwenAlgorithm.Standard\nif mode == ShapleyMode.Owen\nelse OwenAlgorithm.Antithetic\n)\nreturn owen_sampling_shapley(\nu,\nn_samples=int(kwargs.get(\"n_samples\", -1)),\nmax_q=int(kwargs.get(\"max_q\", -1)),\nmethod=method,\nn_jobs=n_jobs,\nseed=seed,\n)\nelif mode == ShapleyMode.KNN:\nreturn knn_shapley(u, progress=progress)\nelif mode == ShapleyMode.GroupTesting:\nn_samples = kwargs.pop(\"n_samples\")\nif n_samples is None:\nraise ValueError(\"n_samples cannot be None for Group Testing\")\nepsilon = kwargs.pop(\"epsilon\")\nif epsilon is None:\nraise ValueError(\"Group Testing requires error bound epsilon\")\ndelta = kwargs.pop(\"delta\", 0.05)\nreturn group_testing_shapley(\nu,\nepsilon=float(epsilon),\ndelta=delta,\nn_samples=int(n_samples),\nn_jobs=n_jobs,\nprogress=progress,\nseed=seed,\n**kwargs,\n)\nelse:\nraise ValueError(f\"Invalid value encountered in {mode=}\")\n</code></pre>"},{"location":"api/pydvl/value/shapley/gt/","title":"Gt","text":"<p>This module implements Group Testing for the approximation of Shapley values, as introduced in (Jia, R. et al., 2019)<sup>1</sup>. The sampling of index subsets is done in such a way that an approximation to the true Shapley values can be computed with guarantees.</p> <p>Warning</p> <p>This method is very inefficient. Potential improvements to the implementation notwithstanding, convergence seems to be very slow (in terms of evaluations of the utility required). We recommend other Monte Carlo methods instead.</p> <p>You can read more in the documentation.</p> <p>New in version 0.4.0</p>"},{"location":"api/pydvl/value/shapley/gt/#pydvl.value.shapley.gt--references","title":"References","text":"<ol> <li> <p>Jia, R. et al., 2019. Towards Efficient Data Valuation Based on the Shapley Value. In: Proceedings of the 22nd International Conference on Artificial Intelligence and Statistics, pp. 1167\u20131176. PMLR.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/gt/#pydvl.value.shapley.gt.num_samples_eps_delta","title":"<code>num_samples_eps_delta(eps, delta, n, utility_range)</code>","text":"<p>Implements the formula in Theorem 3 of (Jia, R. et al., 2019)<sup>1</sup> which gives a lower bound on the number of samples required to obtain an (\u03b5/\u221an,\u03b4/(N(N-1))-approximation to all pair-wise differences of Shapley values, wrt. \\(\\ell_2\\) norm.</p> PARAMETER  DESCRIPTION <code>eps</code> <p>\u03b5</p> <p> TYPE: <code>float</code> </p> <code>delta</code> <p>\u03b4</p> <p> TYPE: <code>float</code> </p> <code>n</code> <p>Number of data points</p> <p> TYPE: <code>int</code> </p> <code>utility_range</code> <p>Range of the Utility function</p> <p> TYPE: <code>float</code> </p> <p>Returns:     Number of samples from \\(2^{[n]}\\) guaranteeing \u03b5/\u221an-correct Shapley         pair-wise differences of values with probability 1-\u03b4/(N(N-1)).</p> <p>New in version 0.4.0</p> Source code in <code>src/pydvl/value/shapley/gt.py</code> <pre><code>def num_samples_eps_delta(\neps: float, delta: float, n: int, utility_range: float\n) -&gt; int:\nr\"\"\"Implements the formula in Theorem 3 of (Jia, R. et al., 2019)&lt;sup&gt;&lt;a href=\"#jia_efficient_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;\n    which gives a lower bound on the number of samples required to obtain an\n    (\u03b5/\u221an,\u03b4/(N(N-1))-approximation to all pair-wise differences of Shapley\n    values, wrt. $\\ell_2$ norm.\n    Args:\n        eps: \u03b5\n        delta: \u03b4\n        n: Number of data points\n        utility_range: Range of the [Utility][pydvl.utils.utility.Utility] function\n    Returns:\n        Number of samples from $2^{[n]}$ guaranteeing \u03b5/\u221an-correct Shapley\n            pair-wise differences of values with probability 1-\u03b4/(N(N-1)).\n    !!! tip \"New in version 0.4.0\"\n    \"\"\"\nconstants = _constants(n=n, epsilon=eps, delta=delta, utility_range=utility_range)\nreturn int(constants.T)\n</code></pre>"},{"location":"api/pydvl/value/shapley/gt/#pydvl.value.shapley.gt.group_testing_shapley","title":"<code>group_testing_shapley(u, n_samples, epsilon, delta, *, n_jobs=1, config=ParallelConfig(), progress=False, seed=None, **options)</code>","text":"<p>Implements group testing for approximation of Shapley values as described in (Jia, R. et al., 2019)<sup>1</sup>.</p> <p>Warning</p> <p>This method is very inefficient. It requires several orders of magnitude more evaluations of the utility than others in montecarlo. It also uses several intermediate objects like the results from the runners and the constraint matrices which can become rather large.</p> <p>By picking a specific distribution over subsets, the differences in Shapley values can be approximated with a Monte Carlo sum. These are then used to solve for the individual values in a feasibility problem.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_samples</code> <p>Number of tests to perform. Use num_samples_eps_delta to estimate this.</p> <p> TYPE: <code>int</code> </p> <code>epsilon</code> <p>From the (\u03b5,\u03b4) sample bound. Use the same as for the estimation of <code>n_iterations</code>.</p> <p> TYPE: <code>float</code> </p> <code>delta</code> <p>From the (\u03b5,\u03b4) sample bound. Use the same as for the estimation of <code>n_iterations</code>.</p> <p> TYPE: <code>float</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use. Each worker performs a chunk of all tests (i.e. utility evaluations).</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> <code>options</code> <p>Additional options to pass to cvxpy.Problem.solve(). E.g. to change the solver (which defaults to <code>cvxpy.SCS</code>) pass <code>solver=cvxpy.CVXOPT</code>.</p> <p> DEFAULT: <code>{}</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>New in version 0.4.0</p> <p>Changed in version 0.5.0</p> <p>Changed the solver to cvxpy instead of scipy's linprog. Added the ability to pass arbitrary options to it.</p> Source code in <code>src/pydvl/value/shapley/gt.py</code> <pre><code>def group_testing_shapley(\nu: Utility,\nn_samples: int,\nepsilon: float,\ndelta: float,\n*,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\nseed: Optional[Seed] = None,\n**options,\n) -&gt; ValuationResult:\n\"\"\"Implements group testing for approximation of Shapley values as described\n    in (Jia, R. et al., 2019)&lt;sup&gt;&lt;a href=\"#jia_efficient_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    !!! Warning\n        This method is very inefficient. It requires several orders of magnitude\n        more evaluations of the utility than others in\n        [montecarlo][pydvl.value.shapley.montecarlo]. It also uses several intermediate\n        objects like the results from the runners and the constraint matrices\n        which can become rather large.\n    By picking a specific distribution over subsets, the differences in Shapley\n    values can be approximated with a Monte Carlo sum. These are then used to\n    solve for the individual values in a feasibility problem.\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_samples: Number of tests to perform. Use\n            [num_samples_eps_delta][pydvl.value.shapley.gt.num_samples_eps_delta]\n            to estimate this.\n        epsilon: From the (\u03b5,\u03b4) sample bound. Use the same as for the\n            estimation of `n_iterations`.\n        delta: From the (\u03b5,\u03b4) sample bound. Use the same as for the\n            estimation of `n_iterations`.\n        n_jobs: Number of parallel jobs to use. Each worker performs a chunk\n            of all tests (i.e. utility evaluations).\n        config: Object configuring parallel computation, with cluster\n            address, number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n        options: Additional options to pass to\n            [cvxpy.Problem.solve()](https://www.cvxpy.org/tutorial/advanced/index.html#solve-method-options).\n            E.g. to change the solver (which defaults to `cvxpy.SCS`) pass\n            `solver=cvxpy.CVXOPT`.\n    Returns:\n        Object with the data values.\n    !!! tip \"New in version 0.4.0\"\n    !!! tip \"Changed in version 0.5.0\"\n        Changed the solver to cvxpy instead of scipy's linprog. Added the ability\n        to pass arbitrary options to it.\n    \"\"\"\nn = len(u.data.indices)\nconst = _constants(\nn=n,\nepsilon=epsilon,\ndelta=delta,\nutility_range=u.score_range.max() - u.score_range.min(),\n)\nT = n_samples\nif T &lt; const.T:\nlog.warning(\nf\"n_samples of {T} are below the required {const.T} for the \"\nf\"\u03b5={epsilon:.02f} guarantee at \u03b4={1 - delta:.02f} probability\"\n)\nsamples_per_job = max(1, n_samples // effective_n_jobs(n_jobs, config))\ndef reducer(\nresults_it: Iterable[Tuple[NDArray, NDArray]]\n) -&gt; Tuple[NDArray, NDArray]:\nreturn np.concatenate(list(x[0] for x in results_it)).astype(\nnp.float_\n), np.concatenate(list(x[1] for x in results_it)).astype(np.int_)\nseed_sequence = ensure_seed_sequence(seed)\nmap_reduce_seed_sequence, cvxpy_seed = tuple(seed_sequence.spawn(2))\nmap_reduce_job: MapReduceJob[Utility, Tuple[NDArray, NDArray]] = MapReduceJob(\nu,\nmap_func=_group_testing_shapley,\nreduce_func=reducer,\nmap_kwargs=dict(n_samples=samples_per_job, progress=progress),\nconfig=config,\nn_jobs=n_jobs,\n)\nuu, betas = map_reduce_job(seed=map_reduce_seed_sequence)\n# Matrix of estimated differences. See Eqs. (3) and (4) in the paper.\nC = np.zeros(shape=(n, n))\nfor i in range(n):\nfor j in range(i + 1, n):\nC[i, j] = np.dot(uu, betas[:, i] - betas[:, j])\nC *= const.Z / T\ntotal_utility = u(u.data.indices)\n###########################################################################\n# Solution of the constraint problem with cvxpy\nv = cp.Variable(n)\nconstraints = [cp.sum(v) == total_utility]\nfor i in range(n):\nfor j in range(i + 1, n):\nconstraints.append(v[i] - v[j] &lt;= epsilon + C[i, j])\nconstraints.append(v[j] - v[i] &lt;= epsilon - C[i, j])\nproblem = cp.Problem(cp.Minimize(0), constraints)\nsolver = options.pop(\"solver\", cp.SCS)\nproblem.solve(solver=solver, **options)\nif problem.status != \"optimal\":\nlog.warning(f\"cvxpy returned status {problem.status}\")\nvalues = (\nnp.nan * np.ones_like(u.data.indices)\nif not hasattr(v.value, \"__len__\")\nelse v.value\n)\nstatus = Status.Failed\nelse:\nvalues = v.value\nstatus = Status.Converged\nreturn ValuationResult(\nalgorithm=\"group_testing_shapley\",\nstatus=status,\nvalues=values,\ndata_names=u.data.data_names,\nsolver_status=problem.status,\n)\n</code></pre>"},{"location":"api/pydvl/value/shapley/knn/","title":"Knn","text":"<p>This module contains Shapley computations for K-Nearest Neighbours.</p> <p>Todo</p> <p>Implement approximate KNN computation for sublinear complexity</p>"},{"location":"api/pydvl/value/shapley/knn/#pydvl.value.shapley.knn--references","title":"References","text":"<ol> <li> <p>Jia, R. et al., 2019. Efficient Task-Specific Data Valuation for Nearest Neighbor Algorithms. In: Proceedings of the VLDB Endowment, Vol. 12, No. 11, pp. 1610\u20131623.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/knn/#pydvl.value.shapley.knn.knn_shapley","title":"<code>knn_shapley(u, *, progress=True)</code>","text":"<p>Computes exact Shapley values for a KNN classifier.</p> <p>This implements the method described in (Jia, R. et al., 2019)<sup>1</sup>. It exploits the local structure of K-Nearest Neighbours to reduce the number of calls to the utility function to a constant number per index, thus reducing computation time to \\(O(n)\\).</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility with a KNN model to extract parameters from. The object will not be modified nor used other than to call get_params()</p> <p> TYPE: <code>Utility</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> RAISES DESCRIPTION <code>TypeError</code> <p>If the model in the utility is not a sklearn.neighbors.KNeighborsClassifier.</p> <p>New in version 0.1.0</p> Source code in <code>src/pydvl/value/shapley/knn.py</code> <pre><code>def knn_shapley(u: Utility, *, progress: bool = True) -&gt; ValuationResult:\n\"\"\"Computes exact Shapley values for a KNN classifier.\n    This implements the method described in (Jia, R. et al., 2019)&lt;sup&gt;&lt;a href=\"#jia_efficient_2019a\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    It exploits the local structure of K-Nearest Neighbours to reduce the number\n    of calls to the utility function to a constant number per index, thus\n    reducing computation time to $O(n)$.\n    Args:\n        u: Utility with a KNN model to extract parameters from. The object\n            will not be modified nor used other than to call [get_params()](\n            &lt;https://scikit-learn.org/stable/modules/generated/sklearn.base.BaseEstimator.html#sklearn.base.BaseEstimator.get_params&gt;)\n        progress: Whether to display a progress bar.\n    Returns:\n        Object with the data values.\n    Raises:\n        TypeError: If the model in the utility is not a\n            [sklearn.neighbors.KNeighborsClassifier][].\n    !!! tip \"New in version 0.1.0\"\n    \"\"\"\nif not isinstance(u.model, KNeighborsClassifier):\nraise TypeError(\"KNN Shapley requires a K-Nearest Neighbours model\")\ndefaults: Dict[str, Union[int, str]] = {\n\"algorithm\": \"ball_tree\" if u.data.dim &gt;= 20 else \"kd_tree\",\n\"metric\": \"minkowski\",\n\"p\": 2,\n}\ndefaults.update(u.model.get_params())\n# HACK: NearestNeighbors doesn't support this. There will be more...\ndel defaults[\"weights\"]\nn_neighbors: int = int(defaults[\"n_neighbors\"])\ndefaults[\"n_neighbors\"] = len(u.data)  # We want all training points sorted\nassert n_neighbors &lt; len(u.data)\n# assert data.target_dim == 1\nnns = NearestNeighbors(**defaults).fit(u.data.x_train)\n# closest to farthest\n_, indices = nns.kneighbors(u.data.x_test)\nvalues: NDArray[np.float_] = np.zeros_like(u.data.indices, dtype=np.float_)\nn = len(u.data)\nyt = u.data.y_train\niterator = enumerate(zip(u.data.y_test, indices), start=1)\nfor j, (y, ii) in maybe_progress(iterator, progress):\nvalue_at_x = int(yt[ii[-1]] == y) / n\nvalues[ii[-1]] += (value_at_x - values[ii[-1]]) / j\nfor i in range(n - 2, n_neighbors, -1):  # farthest to closest\nvalue_at_x = (\nvalues[ii[i + 1]] + (int(yt[ii[i]] == y) - int(yt[ii[i + 1]] == y)) / i\n)\nvalues[ii[i]] += (value_at_x - values[ii[i]]) / j\nfor i in range(n_neighbors, -1, -1):  # farthest to closest\nvalue_at_x = (\nvalues[ii[i + 1]]\n+ (int(yt[ii[i]] == y) - int(yt[ii[i + 1]] == y)) / n_neighbors\n)\nvalues[ii[i]] += (value_at_x - values[ii[i]]) / j\nreturn ValuationResult(\nalgorithm=\"knn_shapley\",\nstatus=Status.Converged,\nvalues=values,\ndata_names=u.data.data_names,\n)\n</code></pre>"},{"location":"api/pydvl/value/shapley/montecarlo/","title":"Montecarlo","text":"<p>Monte Carlo approximations to Shapley Data values.</p> <p>Warning</p> <p>You probably want to use the common interface provided by compute_shapley_values() instead of directly using the functions in this module.</p> <p>Because exact computation of Shapley values requires \\(\\mathcal{O}(2^n)\\) re-trainings of the model, several Monte Carlo approximations are available. The first two sample from the powerset of the training data directly: combinatorial_montecarlo_shapley() and owen_sampling_shapley(). The latter uses a reformulation in terms of a continuous extension of the utility.</p> <p>Alternatively, employing another reformulation of the expression above as a sum over permutations, one has the implementation in permutation_montecarlo_shapley(), or using an early stopping strategy to reduce computation truncated_montecarlo_shapley().</p> <p>Also see</p> <p>It is also possible to use group_testing_shapley() to reduce the number of evaluations of the utility. The method is however typically outperformed by others in this module.</p> <p>Also see</p> <p>Additionally, you can consider grouping your data points using GroupedDataset and computing the values of the groups instead. This is not to be confused with \"group testing\" as implemented in group_testing_shapley(): any of the algorithms mentioned above, including Group Testing, can work to valuate groups of samples as units.</p>"},{"location":"api/pydvl/value/shapley/montecarlo/#pydvl.value.shapley.montecarlo--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/montecarlo/#pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley","title":"<code>permutation_montecarlo_shapley(u, done, *, truncation=NoTruncation(), n_jobs=1, config=ParallelConfig(), progress=False, seed=None)</code>","text":"<p>Computes an approximate Shapley value by sampling independent permutations of the index set, approximating the sum:</p> \\[ v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} \\tilde{w}( | \\sigma_{:i} | )[u(\\sigma_{:i} \\cup \\{i\\}) \u2212 u(\\sigma_{:i})], \\] <p>where \\(\\sigma_{:i}\\) denotes the set of indices in permutation sigma before the position where \\(i\\) appears (see [[data-valuation]] for details).</p> <p>This implements the method described in (Ghorbani and Zou, 2019)<sup>1</sup> with a double stopping criterion.</p> <p>.. todo::    Think of how to add Robin-Gelman or some other more principled stopping    criterion.</p> <p>Instead of naively implementing the expectation, we sequentially add points to coalitions from a permutation and incrementally compute marginal utilities. We stop computing marginals for a given permutation based on a TruncationPolicy. (Ghorbani and Zou, 2019)<sup>1</sup> mention two policies: one that stops after a certain fraction of marginals are computed, implemented in FixedTruncation, and one that stops if the last computed utility (\"score\") is close to the total utility using the standard deviation of the utility as a measure of proximity, implemented in BootstrapTruncation.</p> <p>We keep sampling permutations and updating all shapley values until the StoppingCriterion returns <code>True</code>.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>function checking whether computation must stop.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>truncation</code> <p>An optional callable which decides whether to interrupt processing a permutation and set all subsequent marginals to zero. Typically used to stop computation when the marginal is small.</p> <p> TYPE: <code>TruncationPolicy</code> DEFAULT: <code>NoTruncation()</code> </p> <code>n_jobs</code> <p>number of jobs across which to distribute the computation.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display a progress bar.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Seed</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> Source code in <code>src/pydvl/value/shapley/montecarlo.py</code> <pre><code>@deprecated(\ntarget=True,\ndeprecated_in=\"0.7.0\",\nremove_in=\"0.8.0\",\nargs_mapping=dict(\ncoordinator_update_period=None, worker_update_period=None, progress=None\n),\n)\ndef permutation_montecarlo_shapley(\nu: Utility,\ndone: StoppingCriterion,\n*,\ntruncation: TruncationPolicy = NoTruncation(),\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\nseed: Seed = None,\n) -&gt; ValuationResult:\nr\"\"\"Computes an approximate Shapley value by sampling independent\n    permutations of the index set, approximating the sum:\n    $$\n    v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)}\n    \\tilde{w}( | \\sigma_{:i} | )[u(\\sigma_{:i} \\cup \\{i\\}) \u2212 u(\\sigma_{:i})],\n    $$\n    where $\\sigma_{:i}$ denotes the set of indices in permutation sigma before\n    the position where $i$ appears (see [[data-valuation]] for details).\n    This implements the method described in (Ghorbani and Zou, 2019)&lt;sup&gt;&lt;a href=\"#ghorbani_data_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;\n    with a double stopping criterion.\n    .. todo::\n       Think of how to add Robin-Gelman or some other more principled stopping\n       criterion.\n    Instead of naively implementing the expectation, we sequentially add points\n    to coalitions from a permutation and incrementally compute marginal utilities.\n    We stop computing marginals for a given permutation based on a\n    [TruncationPolicy][pydvl.value.shapley.truncated.TruncationPolicy].\n    (Ghorbani and Zou, 2019)&lt;sup&gt;&lt;a href=\"#ghorbani_data_2019\"&gt;1&lt;/a&gt;&lt;/sup&gt;\n    mention two policies: one that stops after a certain\n    fraction of marginals are computed, implemented in\n    [FixedTruncation][pydvl.value.shapley.truncated.FixedTruncation],\n    and one that stops if the last computed utility (\"score\") is close to the\n    total utility using the standard deviation of the utility as a measure of\n    proximity, implemented in\n    [BootstrapTruncation][pydvl.value.shapley.truncated.BootstrapTruncation].\n    We keep sampling permutations and updating all shapley values\n    until the [StoppingCriterion][pydvl.value.stopping.StoppingCriterion] returns\n    `True`.\n    Args:\n        u: Utility object with model, data, and scoring function.\n        done: function checking whether computation must stop.\n        truncation: An optional callable which decides whether to interrupt\n            processing a permutation and set all subsequent marginals to\n            zero. Typically used to stop computation when the marginal is small.\n        n_jobs: number of jobs across which to distribute the computation.\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        progress: Whether to display a progress bar.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n    Returns:\n        Object with the data values.\n    \"\"\"\nalgorithm = \"permutation_montecarlo_shapley\"\nparallel_backend = init_parallel_backend(config)\nu = parallel_backend.put(u)\nmax_workers = effective_n_jobs(n_jobs, config)\nn_submitted_jobs = 2 * max_workers  # number of jobs in the executor's queue\nseed_sequence = ensure_seed_sequence(seed)\nresult = ValuationResult.zeros(\nalgorithm=algorithm, indices=u.data.indices, data_names=u.data.data_names\n)\npbar = tqdm(disable=not progress, total=100, unit=\"%\")\nwith init_executor(\nmax_workers=max_workers, config=config, cancel_futures=CancellationPolicy.ALL\n) as executor:\npending: set[Future] = set()\nwhile True:\npbar.n = 100 * done.completion()\npbar.refresh()\ncompleted, pending = wait(\npending, timeout=config.wait_timeout, return_when=FIRST_COMPLETED\n)\nfor future in completed:\nresult += future.result()\n# we could check outside the loop, but that means more\n# submissions if the stopping criterion is unstable\nif done(result):\nreturn result\n# Ensure that we always have n_submitted_jobs in the queue or running\nn_remaining_slots = n_submitted_jobs - len(pending)\nseeds = seed_sequence.spawn(n_remaining_slots)\nfor i in range(n_remaining_slots):\nfuture = executor.submit(\n_permutation_montecarlo_one_step,\nu,\ntruncation,\nalgorithm,\nseed=seeds[i],\n)\npending.add(future)\n</code></pre>"},{"location":"api/pydvl/value/shapley/montecarlo/#pydvl.value.shapley.montecarlo.combinatorial_montecarlo_shapley","title":"<code>combinatorial_montecarlo_shapley(u, done, *, n_jobs=1, config=ParallelConfig(), progress=False, seed=None)</code>","text":"<p>Computes an approximate Shapley value using the combinatorial definition:</p> \\[v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}} \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)]\\] <p>This consists of randomly sampling subsets of the power set of the training indices in u.data, and computing their marginal utilities. See Data valuation for details.</p> <p>Note that because sampling is done with replacement, the approximation is poor even for \\(2^{m}\\) subsets with \\(m&gt;n\\), even though there are \\(2^{n-1}\\) subsets for each \\(i\\). Prefer permutation_montecarlo_shapley().</p> <p>Parallelization is done by splitting the set of indices across processes and computing the sum over subsets \\(S \\subseteq N \\setminus \\{i\\}\\) separately.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Stopping criterion for the computation.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>n_jobs</code> <p>number of parallel jobs across which to distribute the computation. Each worker receives a chunk of indices</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> Source code in <code>src/pydvl/value/shapley/montecarlo.py</code> <pre><code>def combinatorial_montecarlo_shapley(\nu: Utility,\ndone: StoppingCriterion,\n*,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\nseed: Optional[Seed] = None,\n) -&gt; ValuationResult:\nr\"\"\"Computes an approximate Shapley value using the combinatorial\n    definition:\n    $$v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}}\n    \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)]$$\n    This consists of randomly sampling subsets of the power set of the training\n    indices in [u.data][pydvl.utils.utility.Utility], and computing their\n    marginal utilities. See [Data valuation][computing-data-values] for details.\n    Note that because sampling is done with replacement, the approximation is\n    poor even for $2^{m}$ subsets with $m&gt;n$, even though there are $2^{n-1}$\n    subsets for each $i$. Prefer\n    [permutation_montecarlo_shapley()][pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley].\n    Parallelization is done by splitting the set of indices across processes and\n    computing the sum over subsets $S \\subseteq N \\setminus \\{i\\}$ separately.\n    Args:\n        u: Utility object with model, data, and scoring function\n        done: Stopping criterion for the computation.\n        n_jobs: number of parallel jobs across which to distribute the\n            computation. Each worker receives a chunk of\n            [indices][pydvl.utils.dataset.Dataset.indices]\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n    Returns:\n        Object with the data values.\n    \"\"\"\nmap_reduce_job: MapReduceJob[NDArray, ValuationResult] = MapReduceJob(\nu.data.indices,\nmap_func=_combinatorial_montecarlo_shapley,\nreduce_func=lambda results: reduce(operator.add, results),\nmap_kwargs=dict(u=u, done=done, progress=progress),\nn_jobs=n_jobs,\nconfig=config,\n)\nreturn map_reduce_job(seed=seed)\n</code></pre>"},{"location":"api/pydvl/value/shapley/naive/","title":"Naive","text":""},{"location":"api/pydvl/value/shapley/naive/#pydvl.value.shapley.naive.permutation_exact_shapley","title":"<code>permutation_exact_shapley(u, *, progress=True)</code>","text":"<p>Computes the exact Shapley value using the formulation with permutations:</p> \\[v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} [u(\\sigma_{i-1} \\cup {i}) \u2212 u(\\sigma_{i})].\\] <p>See Data valuation for details.</p> <p>When the length of the training set is &gt; 10 this prints a warning since the computation becomes too expensive. Used mostly for internal testing and simple use cases. Please refer to the Monte Carlo approximations for practical applications.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>True</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> Source code in <code>src/pydvl/value/shapley/naive.py</code> <pre><code>def permutation_exact_shapley(u: Utility, *, progress: bool = True) -&gt; ValuationResult:\nr\"\"\"Computes the exact Shapley value using the formulation with permutations:\n    $$v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} [u(\\sigma_{i-1} \\cup {i}) \u2212 u(\\sigma_{i})].$$\n    See [Data valuation][computing-data-values] for details.\n    When the length of the training set is &gt; 10 this prints a warning since the\n    computation becomes too expensive. Used mostly for internal testing and\n    simple use cases. Please refer to the [Monte Carlo\n    approximations][pydvl.value.shapley.montecarlo] for practical applications.\n    Args:\n        u: Utility object with model, data, and scoring function\n        progress: Whether to display progress bars for each job.\n    Returns:\n        Object with the data values.\n    \"\"\"\nn = len(u.data)\n# Note that the cache in utility saves most of the refitting because we\n# use frozenset for the input.\nif n &gt; 10:\nwarnings.warn(\nf\"Large dataset! Computation requires {n}! calls to utility()\",\nRuntimeWarning,\n)\nvalues = np.zeros(n)\nfor p in maybe_progress(\npermutations(u.data.indices),\nprogress,\ndesc=\"Permutation\",\ntotal=math.factorial(n),\n):\nfor i, idx in enumerate(p):\nvalues[idx] += u(p[: i + 1]) - u(p[:i])\nvalues /= math.factorial(n)\nreturn ValuationResult(\nalgorithm=\"permutation_exact_shapley\",\nstatus=Status.Converged,\nvalues=values,\ndata_names=u.data.data_names,\n)\n</code></pre>"},{"location":"api/pydvl/value/shapley/naive/#pydvl.value.shapley.naive.combinatorial_exact_shapley","title":"<code>combinatorial_exact_shapley(u, *, n_jobs=1, config=ParallelConfig(), progress=False)</code>","text":"<p>Computes the exact Shapley value using the combinatorial definition.</p> \\[v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}} \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)].\\] <p>See Data valuation for details.</p> <p>Note</p> <p>If the length of the training set is &gt; n_jobs*20 this prints a warning because the computation is very expensive. Used mostly for internal testing and simple use cases. Please refer to the Monte Carlo approximations for practical applications.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> Source code in <code>src/pydvl/value/shapley/naive.py</code> <pre><code>def combinatorial_exact_shapley(\nu: Utility,\n*,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\n) -&gt; ValuationResult:\nr\"\"\"Computes the exact Shapley value using the combinatorial definition.\n    $$v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}} \\binom{n-1}{ | S | }^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)].$$\n    See [Data valuation][computing-data-values] for details.\n    !!! Note\n        If the length of the training set is &gt; n_jobs*20 this prints a warning\n        because the computation is very expensive. Used mostly for internal testing\n        and simple use cases. Please refer to the\n        [Monte Carlo][pydvl.value.shapley.montecarlo] approximations for practical\n        applications.\n    Args:\n        u: Utility object with model, data, and scoring function\n        n_jobs: Number of parallel jobs to use\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n    Returns:\n        Object with the data values.\n    \"\"\"\n# Arbitrary choice, will depend on time required, caching, etc.\nif len(u.data) // n_jobs &gt; 20:\nwarnings.warn(\nf\"Large dataset! Computation requires 2^{len(u.data)} calls to model.fit()\"\n)\ndef reduce_fun(results: List[NDArray]) -&gt; NDArray:\nreturn np.array(results).sum(axis=0)  # type: ignore\nmap_reduce_job: MapReduceJob[NDArray, NDArray] = MapReduceJob(\nu.data.indices,\nmap_func=_combinatorial_exact_shapley,\nmap_kwargs=dict(u=u, progress=progress),\nreduce_func=reduce_fun,\nn_jobs=n_jobs,\nconfig=config,\n)\nvalues = map_reduce_job()\nreturn ValuationResult(\nalgorithm=\"combinatorial_exact_shapley\",\nstatus=Status.Converged,\nvalues=values,\ndata_names=u.data.data_names,\n)\n</code></pre>"},{"location":"api/pydvl/value/shapley/owen/","title":"Owen","text":""},{"location":"api/pydvl/value/shapley/owen/#pydvl.value.shapley.owen--references","title":"References","text":"<ol> <li> <p>Okhrati, R., Lipani, A., 2021. A Multilinear Sampling Algorithm to Estimate Shapley Values. In: 2020 25th International Conference on Pattern Recognition (ICPR), pp. 7992\u20137999. IEEE.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/owen/#pydvl.value.shapley.owen.owen_sampling_shapley","title":"<code>owen_sampling_shapley(u, n_samples, max_q, *, method=OwenAlgorithm.Standard, n_jobs=1, config=ParallelConfig(), progress=False, seed=None)</code>","text":"<p>Owen sampling of Shapley values as described in (Okhrati and Lipani, 2021)<sup>1</sup>.</p> <p>This function computes a Monte Carlo approximation to</p> \\[v_u(i) = \\int_0^1 \\mathbb{E}_{S \\sim P_q(D_{\\backslash \\{i\\}})} [u(S \\cup \\{i\\}) - u(S)]\\] <p>using one of two methods. The first one, selected with the argument <code>mode = OwenAlgorithm.Standard</code>, approximates the integral with:</p> \\[\\hat{v}_u(i) = \\frac{1}{Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m \\cup \\{i\\}) - u(S^{(q_j)}_m)],\\] <p>where \\(q_j = \\frac{j}{Q} \\in [0,1]\\) and the sets \\(S^{(q_j)}\\) are such that a sample \\(x \\in S^{(q_j)}\\) if a draw from a \\(Ber(q_j)\\) distribution is 1.</p> <p>The second method, selected with the argument <code>mode = OwenAlgorithm.Antithetic</code>, uses correlated samples in the inner sum to reduce the variance:</p> \\[\\hat{v}_u(i) = \\frac{1}{2 Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m \\cup \\{i\\}) - u(S^{(q_j)}_m) + u((S^{(q_j)}_m)^c \\cup \\{i\\}) - u((S^{( q_j)}_m)^c)],\\] <p>where now \\(q_j = \\frac{j}{2Q} \\in [0,\\frac{1}{2}]\\), and \\(S^c\\) is the complement of \\(S\\).</p> <p>Note</p> <p>The outer integration could be done instead with a quadrature rule.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object holding data, model and scoring function.</p> <p> TYPE: <code>Utility</code> </p> <code>n_samples</code> <p>Numer of sets to sample for each value of q</p> <p> TYPE: <code>int</code> </p> <code>max_q</code> <p>Number of subdivisions for q \u2208 [0,1] (the element sampling probability) used to approximate the outer integral.</p> <p> TYPE: <code>int</code> </p> <code>method</code> <p>Selects the algorithm to use, see the description. Either OwenAlgorithm.Full for \\(q \\in [0,1]\\) or OwenAlgorithm.Halved for \\(q \\in [0,0.5]\\) and correlated samples</p> <p> TYPE: <code>OwenAlgorithm</code> DEFAULT: <code>Standard</code> </p> <code>n_jobs</code> <p>Number of parallel jobs to use. Each worker receives a chunk of the total of <code>max_q</code> values for q.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>progress</code> <p>Whether to display progress bars for each job.</p> <p> TYPE: <code>bool</code> DEFAULT: <code>False</code> </p> <code>seed</code> <p>Either an instance of a numpy random number generator or a seed for it.</p> <p> TYPE: <code>Optional[Seed]</code> DEFAULT: <code>None</code> </p> RETURNS DESCRIPTION <code>ValuationResult</code> <p>Object with the data values.</p> <p>New in version 0.3.0</p> <p>Changed in version 0.5.0</p> <p>Support for parallel computation and enable antithetic sampling.</p> Source code in <code>src/pydvl/value/shapley/owen.py</code> <pre><code>def owen_sampling_shapley(\nu: Utility,\nn_samples: int,\nmax_q: int,\n*,\nmethod: OwenAlgorithm = OwenAlgorithm.Standard,\nn_jobs: int = 1,\nconfig: ParallelConfig = ParallelConfig(),\nprogress: bool = False,\nseed: Optional[Seed] = None\n) -&gt; ValuationResult:\nr\"\"\"Owen sampling of Shapley values as described in\n    (Okhrati and Lipani, 2021)&lt;sup&gt;&lt;a href=\"#okhrati_multilinear_2021\"&gt;1&lt;/a&gt;&lt;/sup&gt;.\n    This function computes a Monte Carlo approximation to\n    $$v_u(i) = \\int_0^1 \\mathbb{E}_{S \\sim P_q(D_{\\backslash \\{i\\}})}\n    [u(S \\cup \\{i\\}) - u(S)]$$\n    using one of two methods. The first one, selected with the argument ``mode =\n    OwenAlgorithm.Standard``, approximates the integral with:\n    $$\\hat{v}_u(i) = \\frac{1}{Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m\n    \\cup \\{i\\}) - u(S^{(q_j)}_m)],$$\n    where $q_j = \\frac{j}{Q} \\in [0,1]$ and the sets $S^{(q_j)}$ are such that a\n    sample $x \\in S^{(q_j)}$ if a draw from a $Ber(q_j)$ distribution is 1.\n    The second method, selected with the argument ``mode =\n    OwenAlgorithm.Antithetic``, uses correlated samples in the inner sum to\n    reduce the variance:\n    $$\\hat{v}_u(i) = \\frac{1}{2 Q M} \\sum_{j=0}^Q \\sum_{m=1}^M [u(S^{(q_j)}_m\n    \\cup \\{i\\}) - u(S^{(q_j)}_m) + u((S^{(q_j)}_m)^c \\cup \\{i\\}) - u((S^{(\n    q_j)}_m)^c)],$$\n    where now $q_j = \\frac{j}{2Q} \\in [0,\\frac{1}{2}]$, and $S^c$ is the\n    complement of $S$.\n    !!! Note\n        The outer integration could be done instead with a quadrature rule.\n    Args:\n        u: [Utility][pydvl.utils.utility.Utility] object holding data, model\n            and scoring function.\n        n_samples: Numer of sets to sample for each value of q\n        max_q: Number of subdivisions for q \u2208 [0,1] (the element sampling\n            probability) used to approximate the outer integral.\n        method: Selects the algorithm to use, see the description. Either\n            [OwenAlgorithm.Full][pydvl.value.shapley.owen.OwenAlgorithm] for\n            $q \\in [0,1]$ or\n            [OwenAlgorithm.Halved][pydvl.value.shapley.owen.OwenAlgorithm] for\n            $q \\in [0,0.5]$ and correlated samples\n        n_jobs: Number of parallel jobs to use. Each worker receives a chunk\n            of the total of `max_q` values for q.\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        progress: Whether to display progress bars for each job.\n        seed: Either an instance of a numpy random number generator or a seed for it.\n    Returns:\n        Object with the data values.\n    !!! tip \"New in version 0.3.0\"\n    !!! tip \"Changed in version 0.5.0\"\n        Support for parallel computation and enable antithetic sampling.\n    \"\"\"\nmap_reduce_job: MapReduceJob[NDArray, ValuationResult] = MapReduceJob(\nu.data.indices,\nmap_func=_owen_sampling_shapley,\nreduce_func=lambda results: reduce(operator.add, results),\nmap_kwargs=dict(\nu=u,\nmethod=OwenAlgorithm(method),\nn_samples=n_samples,\nmax_q=max_q,\nprogress=progress,\n),\nn_jobs=n_jobs,\nconfig=config,\n)\nreturn map_reduce_job(seed=seed)\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/","title":"Truncated","text":""},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated--references","title":"References","text":"<ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning. In: Proceedings of the 36th International Conference on Machine Learning, PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> </ol>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.TruncationPolicy","title":"<code>TruncationPolicy()</code>","text":"<p>             Bases: <code>ABC</code></p> <p>A policy for deciding whether to stop computing marginals in a permutation.</p> <p>Statistics are kept on the number of calls and truncations as <code>n_calls</code> and <code>n_truncations</code> respectively.</p> ATTRIBUTE DESCRIPTION <code>n_calls</code> <p>Number of calls to the policy.</p> <p> TYPE: <code>int</code> </p> <code>n_truncations</code> <p>Number of truncations made by the policy.</p> <p> TYPE: <code>int</code> </p> <p>Todo</p> <p>Because the policy objects are copied to the workers, the statistics are not accessible from the coordinating process. We need to add methods for this.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self):\nself.n_calls: int = 0\nself.n_truncations: int = 0\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.TruncationPolicy.reset","title":"<code>reset()</code>  <code>abstractmethod</code>","text":"<p>Reset the policy to a state ready for a new permutation.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>@abc.abstractmethod\ndef reset(self):\n\"\"\"Reset the policy to a state ready for a new permutation.\"\"\"\n...\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.TruncationPolicy.__call__","title":"<code>__call__(idx, score)</code>","text":"<p>Check whether the computation should be interrupted.</p> PARAMETER  DESCRIPTION <code>idx</code> <p>Position in the permutation currently being computed.</p> <p> TYPE: <code>int</code> </p> <code>score</code> <p>Last utility computed.</p> <p> TYPE: <code>float</code> </p> RETURNS DESCRIPTION <code>bool</code> <p><code>True</code> if the computation should be interrupted.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __call__(self, idx: int, score: float) -&gt; bool:\n\"\"\"Check whether the computation should be interrupted.\n    Args:\n        idx: Position in the permutation currently being computed.\n        score: Last utility computed.\n    Returns:\n        `True` if the computation should be interrupted.\n    \"\"\"\nret = self._check(idx, score)\nself.n_calls += 1\nself.n_truncations += 1 if ret else 0\nreturn ret\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.NoTruncation","title":"<code>NoTruncation</code>","text":"<p>             Bases: <code>TruncationPolicy</code></p> <p>A policy which never interrupts the computation.</p>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.FixedTruncation","title":"<code>FixedTruncation(u, fraction)</code>","text":"<p>             Bases: <code>TruncationPolicy</code></p> <p>Break a permutation after computing a fixed number of marginals.</p> <p>The experiments in Appendix B of (Ghorbani and Zou, 2019)<sup>1</sup> show that when the training set size is large enough, one can simply truncate the iteration over permutations after a fixed number of steps. This happens because beyond a certain number of samples in a training set, the model becomes insensitive to new ones. Alas, this strongly depends on the data distribution and the model and there is no automatic way of estimating this number.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>fraction</code> <p>Fraction of marginals in a permutation to compute before stopping (e.g. 0.5 to compute half of the marginals).</p> <p> TYPE: <code>float</code> </p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self, u: Utility, fraction: float):\nsuper().__init__()\nif fraction &lt;= 0 or fraction &gt; 1:\nraise ValueError(\"fraction must be in (0, 1]\")\nself.max_marginals = len(u.data) * fraction\nself.count = 0\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.RelativeTruncation","title":"<code>RelativeTruncation(u, rtol)</code>","text":"<p>             Bases: <code>TruncationPolicy</code></p> <p>Break a permutation if the marginal utility is too low.</p> <p>This is called \"performance tolerance\" in (Ghorbani and Zou, 2019)<sup>1</sup>.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>rtol</code> <p>Relative tolerance. The permutation is broken if the last computed utility is less than <code>total_utility * rtol</code>.</p> <p> TYPE: <code>float</code> </p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self, u: Utility, rtol: float):\nsuper().__init__()\nself.rtol = rtol\nlogger.info(\"Computing total utility for permutation truncation.\")\nself.total_utility = u(u.data.indices)\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.BootstrapTruncation","title":"<code>BootstrapTruncation(u, n_samples, sigmas=1)</code>","text":"<p>             Bases: <code>TruncationPolicy</code></p> <p>Break a permutation if the last computed utility is close to the total utility, measured as a multiple of the standard deviation of the utilities.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>n_samples</code> <p>Number of bootstrap samples to use to compute the variance of the utilities.</p> <p> TYPE: <code>int</code> </p> <code>sigmas</code> <p>Number of standard deviations to use as a threshold.</p> <p> TYPE: <code>float</code> DEFAULT: <code>1</code> </p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>def __init__(self, u: Utility, n_samples: int, sigmas: float = 1):\nsuper().__init__()\nself.n_samples = n_samples\nlogger.info(\"Computing total utility for permutation truncation.\")\nself.total_utility = u(u.data.indices)\nself.count: int = 0\nself.variance: float = 0\nself.mean: float = 0\nself.sigmas: float = sigmas\n</code></pre>"},{"location":"api/pydvl/value/shapley/truncated/#pydvl.value.shapley.truncated.truncated_montecarlo_shapley","title":"<code>truncated_montecarlo_shapley(u, *, done, truncation, config=ParallelConfig(), n_jobs=1, coordinator_update_period=10, worker_update_period=5)</code>","text":"<p>Warning</p> <p>This method is deprecated and only a wrapper for permutation_montecarlo_shapley.</p> <p>Todo</p> <p>Think of how to add Robin-Gelman or some other more principled stopping criterion.</p> PARAMETER  DESCRIPTION <code>u</code> <p>Utility object with model, data, and scoring function</p> <p> TYPE: <code>Utility</code> </p> <code>done</code> <p>Check on the results which decides when to stop sampling permutations.</p> <p> TYPE: <code>StoppingCriterion</code> </p> <code>truncation</code> <p>callable that decides whether to stop computing marginals for a given permutation.</p> <p> TYPE: <code>TruncationPolicy</code> </p> <code>config</code> <p>Object configuring parallel computation, with cluster address, number of cpus, etc.</p> <p> TYPE: <code>ParallelConfig</code> DEFAULT: <code>ParallelConfig()</code> </p> <code>n_jobs</code> <p>Number of permutation monte carlo jobs to run concurrently.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> <p>Returns:     Object with the data values.</p> Source code in <code>src/pydvl/value/shapley/truncated.py</code> <pre><code>@deprecated(\ntarget=True,\ndeprecated_in=\"0.7.0\",\nremove_in=\"0.8.0\",\nargs_mapping=dict(coordinator_update_period=None, worker_update_period=None),\n)\ndef truncated_montecarlo_shapley(\nu: Utility,\n*,\ndone: StoppingCriterion,\ntruncation: TruncationPolicy,\nconfig: ParallelConfig = ParallelConfig(),\nn_jobs: int = 1,\ncoordinator_update_period: int = 10,\nworker_update_period: int = 5,\n) -&gt; ValuationResult:\n\"\"\"\n    !!! Warning\n        This method is deprecated and only a wrapper for\n        [permutation_montecarlo_shapley][pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley].\n    !!! Todo\n        Think of how to add Robin-Gelman or some other more principled stopping\n        criterion.\n    Args:\n        u: Utility object with model, data, and scoring function\n        done: Check on the results which decides when to stop sampling\n            permutations.\n        truncation: callable that decides whether to stop computing marginals\n            for a given permutation.\n        config: Object configuring parallel computation, with cluster address,\n            number of cpus, etc.\n        n_jobs: Number of permutation monte carlo jobs to run concurrently.\n    Returns:\n        Object with the data values.\n    \"\"\"\nfrom pydvl.value.shapley.montecarlo import permutation_montecarlo_shapley\nreturn cast(\nValuationResult,\npermutation_montecarlo_shapley(\nu, done=done, truncation=truncation, config=config, n_jobs=n_jobs\n),\n)\n</code></pre>"},{"location":"api/pydvl/value/shapley/types/","title":"Types","text":""},{"location":"api/pydvl/value/shapley/types/#pydvl.value.shapley.types.ShapleyMode","title":"<code>ShapleyMode</code>","text":"<p>             Bases: <code>str</code>, <code>Enum</code></p> <p>Supported algorithms for the computation of Shapley values.</p> <p>Todo</p> <p>Make algorithms register themselves here.</p>"},{"location":"examples/data_oob/","title":"Data OOB","text":"<p>This notebook introduces the Data-OOB method, an implementation based on a publication from Kwon and Zou \"Data-OOB: Out-of-bag Estimate as a Simple and Efficient Data Value\" ICML 2023 , using pyDVL.</p> <p>The objective of this paper is mainly to overcome the computational bottleneck of shapley-based data valuation methods that require to fit a significant number of models to accurately estimate marginal contributions. The algorithms computes data values from out of bag estimates using a bagging model.</p> <p>The value can be interpreted as a partition of the OOB estimate, which is originally introduced to estimate the prediction error. This OOB estimate is given as:</p> \\[ \\sum_{i=1}^n\\frac{\\sum_{b=1}^{B}\\mathbb{1}(w_{bi}=0)T(y_i, \\hat{f}_b(x_i))}{\\sum_{b=1}^{B} \\mathbb{1} (w_{bi}=0)} \\] <pre>\n<code>   age  fnlwgt  education-num  capital-gain  capital-loss  hours-per-week  \\\n0   39   77516             13          2174             0              40   \n1   50   83311             13             0             0              13   \n2   38  215646              9             0             0              40   \n3   53  234721              7             0             0              40   \n4   28  338409             13             0             0              40   \n\n  income  \n0  &lt;=50K  \n1  &lt;=50K  \n2  &lt;=50K  \n3  &lt;=50K  \n4  &lt;=50K  \n</code>\n</pre> <pre><code>oob_values = compute_data_oob(utility, n_est=1000, max_samples=0.95)\n</code></pre>"},{"location":"examples/data_oob/#bagging-for-data-valuation","title":"Bagging for data valuation","text":""},{"location":"examples/data_oob/#setup","title":"Setup","text":"<p>We begin by importing the main libraries and setting some defaults.</p>   If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/data_oob/#variance","title":"Variance","text":"<p>The variance it the weak learner variance. It is computed with Welford's online algorithm.</p>"},{"location":"examples/data_oob/#point-removal-experiments","title":"Point removal experiments","text":"<p>The standard procedure for the evaluation of data valuation schemes is the point removal experiment. The objective is to measure the evolution of performance when the best/worst points are removed from the training set. </p>"},{"location":"examples/influence_imagenet/","title":"For CNNs","text":"If you are reading this in the documentation, some boilerplate has been omitted for convenience.   <pre><code>%load_ext autoreload\n</code></pre> <pre><code>%autoreload\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nimport torch\nfrom torch import nn\nfrom notebook_support import (\nplot_sample_images,\nplot_lowest_highest_influence_images,\nplot_losses,\ncorrupt_imagenet,\nload_preprocess_imagenet,\nplot_corrupted_influences_distribution,\ncompute_mean_corrupted_influences,\nTrainingManager,\nMODEL_PATH,\nnew_resnet_model,\n)\ndefault_figsize = (7, 7)\nplt.rcParams[\"figure.figsize\"] = default_figsize\nplt.rcParams[\"font.size\"] = 12\nplt.rcParams[\"xtick.labelsize\"] = 12\nplt.rcParams[\"ytick.labelsize\"] = 10\nhessian_reg = 1e4 if os.environ.get(\"CI\") else 1e-3\nrandom_state = 42\nnp.random.seed(random_state)\nDEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n</code></pre> <pre><code>from pydvl.influence.general import compute_influences\nfrom pydvl.reporting.plots import plot_influence_distribution_by_label\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, f1_score\n</code></pre> <pre><code>label_names = {90: \"tables\", 100: \"boats\"}\ntrain_ds, val_ds, test_ds = load_preprocess_imagenet(\ntrain_size=0.8,\ntest_size=0.1,\nkeep_labels=label_names,\ndownsampling_ratio=1,\n)\nprint(\"Normalised image dtype:\", train_ds[\"normalized_images\"][0].dtype)\nprint(\"Label type:\", type(train_ds[\"labels\"][0]))\nprint(\"Image type:\", type(train_ds[\"images\"][0]))\ntrain_ds.info()\n</code></pre> <pre>\n<code>Normalised image dtype: torch.float32\nLabel type: &lt;class 'str'&gt;\nImage type: &lt;class 'PIL.JpegImagePlugin.JpegImageFile'&gt;\n&lt;class 'pandas.core.frame.DataFrame'&gt;\nRangeIndex: 707 entries, 0 to 706\nData columns (total 3 columns):\n #   Column             Non-Null Count  Dtype \n---  ------             --------------  ----- \n 0   normalized_images  707 non-null    object\n 1   labels             707 non-null    object\n 2   images             707 non-null    object\ndtypes: object(3)\nmemory usage: 16.7+ KB\n</code>\n</pre> <p>Let's take a closer look at a few image samples</p> <pre><code>plot_sample_images(train_ds, n_images_per_class=3)\n</code></pre> <p>Let's now further pre-process the data and prepare for model training. The helper function <code>process_io</code> converts the normalized images into tensors and the labels to the indices 0 and 1 to train the classifier.</p> <pre><code>from typing import Tuple\ndef process_io(df: pd.DataFrame, labels: dict) -&amp;gt; Tuple[torch.Tensor, torch.Tensor]:\nx = df[\"normalized_images\"]\ny = df[\"labels\"]\nds_label_to_model_label = {\nds_label: idx for idx, ds_label in enumerate(labels.values())\n}\nx_nn = torch.stack(x.tolist()).to(DEVICE)\ny_nn = torch.tensor([ds_label_to_model_label[yi] for yi in y], device=DEVICE)\nreturn x_nn, y_nn\ntrain_x, train_y = process_io(train_ds, label_names)\nval_x, val_y = process_io(val_ds, label_names)\ntest_x, test_y = process_io(test_ds, label_names)\n</code></pre> <pre><code>model_ft = new_resnet_model(output_size=len(label_names))\nmgr = TrainingManager(\n\"model_ft\",\nmodel_ft,\nnn.CrossEntropyLoss(),\ntrain_x,\ntrain_y,\nval_x,\nval_y,\nMODEL_PATH,\n)\n# Set use_cache=False to retrain the model\ntrain_loss, val_loss = mgr.train(n_epochs=50, use_cache=True)\n</code></pre> <pre>\n<code>Cached model found, loading...\n</code>\n</pre> <pre><code>plot_losses(train_loss, val_loss)\n</code></pre> <p>The confusion matrix and \\(F_1\\) score look good, especially considering the low resolution of the images and their complexity (they contain different objects)</p> <pre><code>pred_y_test = np.argmax(model_ft(test_x).detach(), axis=1)\nmodel_score = f1_score(test_y, pred_y_test, average=\"weighted\")\ncm = confusion_matrix(test_y, pred_y_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=label_names.values())\nprint(\"f1_score of model:\", model_score)\ndisp.plot();\n</code></pre> <pre>\n<code>f1_score of model: 0.8468272032336833\n</code>\n</pre> <pre><code>influences = compute_influences(\nmodel=mgr.model,\nloss=mgr.loss,\nx=train_x,\ny=train_y,\nx_test=test_x,\ny_test=test_y,\nhessian_regularization=hessian_reg,\ninversion_method=\"cg\",\ninfluence_type=\"up\",\nprogress=True,\n)\n</code></pre> <pre><code>test_image_idx = 42\nmodel_label_to_ds_label = {\nidx: ds_label for idx, ds_label in enumerate(label_names.values())\n}\npredicted_label = model_label_to_ds_label[\nnp.argmax(model_ft(test_x[test_image_idx].unsqueeze(0)).detach(), axis=1).item()\n]\ntrue_label = test_ds[\"labels\"][test_image_idx]\nplt.rcParams[\"figure.figsize\"] = (3, 3)\nplt.imshow(test_ds[\"images\"][test_image_idx])\nplt.axis(\"off\")\nplt.title(f\"Predicted: {predicted_label} - True: {true_label}\")\nplt.show()\n</code></pre> <p>Now we plot the histogram of the influence that all training images have on the image selected above, separated by their label.</p> <pre><code>plt.rcParams[\"figure.figsize\"] = default_figsize\nplot_influence_distribution_by_label(\ninfluences[test_image_idx],\ntrain_ds[\"labels\"].values,\ntitle_extra=f\"over index {test_image_idx}\",\n)\n</code></pre> <p>Rather unsurprisingly, the training samples that have the same label as the test image have, on average, a higher influence on the classifier's output for it. Let's then take them and visualize those with the highest and lowest influence:</p> <pre><code>images_with_same_label = train_ds[\"labels\"] == test_ds[\"labels\"][test_image_idx]\ninfluence_values_with_same_label = influences[test_image_idx][images_with_same_label]\nimages_same_label = train_ds[\"images\"][images_with_same_label].values\nplot_lowest_highest_influence_images(\ninfluence_values_with_same_label, subset_images=images_same_label, num_to_plot=3\n)\n</code></pre> <p>Looking at the images, it is difficult to explain why those on the right are more influential than those on the left. At first sight, the choice seems to be random (or at the very least noisy). Let's dig in a bit more by looking at average influences:</p> <pre><code>avg_influences = np.mean(influences, axis=0)\n</code></pre> <p>Once again, let's plot the histogram of influence values by label.</p> <pre><code>plot_influence_distribution_by_label(\navg_influences, train_ds[\"labels\"].values, \"over all test samples\"\n)\n</code></pre> <p>Next, for each class (you can change value by changing label key) we can have a look at the top and bottom images by average influence, i.e. we can show the images that have the highest and lowest average influence over all test images.</p> <pre><code>label = \"tables\"\nimg_with_selected_label = train_ds[\"labels\"] == label\nif_selected_label = avg_influences[img_with_selected_label]\nimges_same_label = train_ds[\"images\"][img_with_selected_label].values\nplot_lowest_highest_influence_images(if_selected_label, imges_same_label, num_to_plot=3)\n</code></pre> <p>Once again, it is not easy to explain why the images on the left have a lower influence than the ones on the right. One could argue that in order to predict that there is a dining table in the image it is beneficial to clearly see both the chairs and the table itself, a feature missing in some samples on the left. Also, color seems to be a discriminant: houses with a blue painting could get confused with the water around a boat. Of course, this is debatable and different people could come up with other explanations a posteriori.</p> <pre><code>corrupted_model = new_resnet_model(output_size=len(label_names))\ncorrupted_dataset, corrupted_indices = corrupt_imagenet(\ndataset=train_ds,\nfraction_to_corrupt=0.1,\navg_influences=avg_influences,\n)\ncorrupted_train_x, corrupted_train_y = process_io(corrupted_dataset, label_names)\nmgr = TrainingManager(\n\"corrupted_model\",\ncorrupted_model,\nnn.CrossEntropyLoss(),\ncorrupted_train_x,\ncorrupted_train_y,\nval_x,\nval_y,\nMODEL_PATH,\n)\ntraining_loss, validation_loss = mgr.train(n_epochs=50, use_cache=True)\n</code></pre> <pre>\n<code>Cached model found, loading...\n</code>\n</pre> <pre><code>plot_losses(training_loss, validation_loss)\n</code></pre> <pre><code>pred_y_test = np.argmax(corrupted_model(test_x).detach(), axis=1)\nmodel_score = f1_score(test_y, pred_y_test, average=\"weighted\")\nprint(\"F1 score of model with corrupted data:\", model_score)\n</code></pre> <pre>\n<code>F1 score of model with corrupted data: 0.8164795918367347\n</code>\n</pre> <p>Interestingly, despite being trained on a corrupted dataset, the model has a fairly high \\(F_1\\) score. Let's now calculate the influence of the corrupted training data points over the test data points.</p> <pre><code>influences = compute_influences(\nmodel=mgr.model,\nloss=mgr.loss,\nx=corrupted_train_x,\ny=corrupted_train_y,\nx_test=test_x,\ny_test=test_y,\nhessian_regularization=hessian_reg,\ninversion_method=\"cg\",\ninfluence_type=\"up\",\nprogress=True,\n)\n</code></pre> <pre>\n<code>Split Gradient:   0%|          | 0/98 [00:00&lt;?, ?it/s]</code>\n</pre> <pre>\n<code>Conjugate gradient:   0%|          | 0/98 [00:00&lt;?, ?it/s]</code>\n</pre> <pre>\n<code>Split Gradient:   0%|          | 0/707 [00:00&lt;?, ?it/s]</code>\n</pre> <p>As before, since we are interested in the average influence on the test dataset, we take the average of influences across rows, and then plot the highest and lowest influences for a chosen label</p> <pre><code>avg_corrupted_influences = np.mean(influences, axis=0)\n</code></pre> <pre><code>label = \"boats\"\nimg_with_selected_label = corrupted_dataset[\"labels\"] == label\nif_selected_label = avg_corrupted_influences[img_with_selected_label]\nimges_same_label = corrupted_dataset[\"images\"][img_with_selected_label].values\nplot_lowest_highest_influence_images(if_selected_label, imges_same_label, num_to_plot=3)\n</code></pre> <p>As expected, the samples with lowest (negative) influence for the label \"boats\" are those that have been corrupted: all the images on the left are tables! We can compare the average influence of corrupted data with non-corrupted ones</p> <pre><code>plot_corrupted_influences_distribution(\ncorrupted_dataset, corrupted_indices, avg_corrupted_influences\n)\n</code></pre> <pre><code>compute_mean_corrupted_influences(\ncorrupted_dataset, corrupted_indices, avg_corrupted_influences\n)\n</code></pre> label avg_non_corrupted_infl avg_corrupted_infl score_diff 0 boats 0.945390 -0.890972 1.836362 1 tables -1.092637 -2.757206 1.664569 <p>And indeed corrupted data have a more negative influence on average than clean ones!</p> <p>Despite this being a useful property, influence functions are known to be unreliable for tasks of data valuation, especially in deep learning where the fundamental assumption of the theory (convexity) is grossly violated. A lot of factors (e.g. the size of the network, the training process or the Hessian regularization term) can interfere with the computation, to the point that often the results that we obtain cannot be trusted. This has been extensively studied in the recent paper:</p> <p>Basu, S., P. Pope, and S. Feizi. Influence Functions in Deep Learning Are Fragile. International Conference on Learning Representations (ICLR). 2021.</p> <p>Nevertheless, influence functions offer a relatively quick and mathematically rigorous way to evaluate (at first order) the importance of a training point for a model's prediction.</p>"},{"location":"examples/influence_imagenet/#influence-functions-for-neural-networks","title":"Influence functions for neural networks","text":"<p>This notebook explores the use of influence functions for convolutional neural networks. In the first part we will investigate the usefulness, or lack thereof, of influence functions for the interpretation of a classifier's outputs.</p> <p>For our study we choose a pre-trained ResNet18, fine-tuned on the tiny-imagenet dataset. This dataset was created for a Stanford course on Deep Learning for Computer Vision, and is a subset of the famous ImageNet with 200 classes instead of 1000, and images down-sampled to a lower resolution of 64x64 pixels.</p> <p>After tuning the last layers of the network, we will use pyDVL to find the most and the least influential training images for the test set. This can sometimes be used to explain inference errors, or to direct efforts during data collection, although we will face inconclusive results with our model and data. This illustrates well-known issues of influence functions for neural networks.</p> <p>However, in the final part of the notebook we will see that influence functions are an effective tool for finding anomalous or corrupted data points.</p> <p>We conclude with an appendix with some basic theoretical concepts used.</p>"},{"location":"examples/influence_imagenet/#imports-and-setup","title":"Imports and setup","text":""},{"location":"examples/influence_imagenet/#loading-and-preprocessing-the-dataset","title":"Loading and preprocessing the dataset","text":"<p>We pick two classes arbitrarily to work with: 90 and 100, corresponding respectively to dining tables, and boats in Venice (you can of course select any other two classes, or more of them, although that would imply longer training times and some modifications in the notebook below). The dataset is loaded with <code>load_preprocess_imagenet()</code>, which returns three pandas <code>DataFrames</code> with training, validation and test sets respectively. Each dataframe has three columns: normalized images, labels and the original images. Note that you can load a subset of the data decreasing downsampling_ratio.</p>"},{"location":"examples/influence_imagenet/#model-definition-and-training","title":"Model definition and training","text":"<p>We use a ResNet18 from <code>torchvision</code> with final layers modified for binary classification.</p> <p>Training for influence computation is facilitated by <code>:class:~pydvl.influence.model_wrappers.torch_wrappers.TorchModel</code>, a convenience wrapper around torch models which is part of pyDVL. We wrap this with a simple class <code>TrainingManager</code> which transparently handles persistence after training. The latter is not part of the main pyDVL package but just a way to reduce clutter in this notebook.</p> <p>We train the model for 50 epochs and save the results. Then we plot the train and validation loss curves.</p>"},{"location":"examples/influence_imagenet/#influence-computation","title":"Influence computation","text":"<p>Let's now calculate influences! The main method is <code>:func:~pydvl.influence.general.compute_influences</code>, which takes a trained <code>nn.Model</code>, the training loss, some input dataset with labels (which typically is the training data, or a subset of it) and some test data.</p> <p>Other important parameters are the Hessian regularization term, which should be chosen as small as possible for the computation to converge (further details on why this is important can be found in the Appendix).</p> <p>Since Resnet18 is quite big, we pick conjugate gradient (<code>cg</code>) as the method for inversion of the Hessian. A naive computation would require a lot of memory. Finally, the influence type will be <code>up</code>. The other option, <code>perturbation</code>, is beyond the scope of this notebook, but more info can be found in the notebook using the Wine dataset or in the documentation for pyDVL.</p> <p>The output of <code>calculate_influences</code> is a matrix of size <code>test_set_length</code> x <code>training_set_length</code>. Each row represents a test data point, and each column a training data point, so that entry \\((i,j)\\) represents the influence of training point \\(j\\) on test point \\(i\\).</p>"},{"location":"examples/influence_imagenet/#analysing-influences","title":"Analysing influences","text":"<p>With the computed influences we can study single images or all of them together:</p>"},{"location":"examples/influence_imagenet/#influence-on-a-single-test-image","title":"Influence on a single test image","text":"<p>Let's take any image in the test set:</p>"},{"location":"examples/influence_imagenet/#analysing-the-average-influence-on-test-samples","title":"Analysing the average influence on test samples","text":"<p>By averaging across the rows of the influence matrix, we obtain the average influence of each training sample on the whole test set:</p>"},{"location":"examples/influence_imagenet/#detecting-corrupted-data","title":"Detecting corrupted data","text":"<p>After facing the shortcomings of influence functions for explaining decisions, we move to an application with clear-cut results. Influences can be successfully used to detect corrupted or mislabeled samples, making them an effective tool to \"debug\" training data.</p> <p>We begin by training a new model (with the same architecture as before) on a dataset with some corrupted labels. The method <code>get_corrupted_imagenet</code> will take the training dataset and corrupt a certain fraction of the labels by flipping them. We use the same number of epochs and optimizer as before.</p>"},{"location":"examples/influence_imagenet/#theory-of-influence-functions-for-neural-networks","title":"Theory of influence functions for neural networks","text":"<p>In this appendix we will briefly go through the basic ideas of influence functions adapted for neural networks as introduced in Koh, Pang Wei, and Percy Liang. \"Understanding Black-box Predictions via Influence Functions\" International conference on machine learning. PMLR, 2017.</p> <p>Note however that this paper departs from the standard and established theory and notation for influence functions. For a rigorous introduction to the topic we recommend classical texts like Hampel, Frank R., Elvezio M. Ronchetti, Peter J. Rousseeuw, and Werner A. Stahel. Robust Statistics: The Approach Based on Influence Functions. 1st edition. Wiley Series in Probability and Statistics. New York: Wiley-Interscience, 2005. https://doi.org/10.1002/9781118186435.</p>"},{"location":"examples/influence_imagenet/#upweighting-points","title":"Upweighting points","text":"<p>Let's start by considering some input space \\(\\mathcal{X}\\) to a model (e.g. images) and an output space \\(\\mathcal{Y}\\) (e.g. labels). Let's take \\(z_i = (x_i, y_i)\\) to be the \\(i\\)-th training point, and \\(\\theta\\) to be the (potentially highly) multi-dimensional parameters of the neural network (i.e. \\(\\theta\\) is a big array with very many parameters). We will indicate with \\(L(z, \\theta)\\) the loss of the model for point \\(z\\) and parameters \\(\\theta\\). When training the model we minimize the loss over all points, i.e. the optimal parameters are calculated through gradient descent on the following formula: $$ \\hat{\\theta} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_i, \\theta) $$ where \\(n\\) is the total number of training data points.</p> <p>For notational  convenience, let's define  $$ \\hat{\\theta}{-z} = \\arg \\min\\theta \\frac{1}{n}\\sum_{z_i \\ne z} L(z_i, \\theta) \\ , $$ i.e. \\(\\hat{\\theta}_{-z}\\) are the model parameters that minimize the total loss when \\(z\\) is not in the training dataset.</p> <p>In order to check the impact of each training point on the model, we would need to calculate \\(\\hat{\\theta}_{-z}\\) for each \\(z\\) in the training dataset, thus re-training the model at least ~\\(n\\) times (more if model training is noisy). This is computationally very expensive, especially for big neural networks. To circumvent this problem, we can just calculate a first order approximation of \\(\\hat{\\theta}\\). This can be done through single backpropagation and without re-training the full model.</p> <p>Let's define $$ \\hat{\\theta}{\\epsilon, z} = \\arg \\min\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_i, \\theta) + \\epsilon L(z, \\theta) \\ , $$ which is the optimal \\(\\hat{\\theta}\\) if we were to up-weigh \\(z\\) by an amount \\(\\epsilon\\).</p> <p>From a classical result (a simple derivation is available in Appendix A of Koh and Liang's paper), we know that: $$ \\frac{d \\ \\hat{\\theta}{\\epsilon, z}}{d \\epsilon} \\Big| = -H_{\\hat{\\theta}}^{-1} \\nabla_\\theta L(z, \\hat{\\theta}) $$ where \\(H_{\\hat{\\theta}} = \\frac{1}{n} \\sum_{i=1}^n \\nabla_\\theta^2 L(z_i, \\hat{\\theta})\\) is the Hessian of \\(L\\). Importantly, notice that this expression is only valid when \\(\\hat{\\theta}\\) is a minimum of \\(L\\), or otherwise \\(H_{\\hat{\\theta}}\\) cannot be inverted!</p>"},{"location":"examples/influence_imagenet/#approximating-the-influence-of-a-point","title":"Approximating the influence of a point","text":"<p>We will define the influence of training point \\(z\\) on test point \\(z_{\\text{test}}\\) as \\(\\mathcal{I}(z, z_{\\text{test}}) =  L(z_{\\text{test}}, \\hat{\\theta}_{-z}) - L(z_{\\text{test}}, \\hat{\\theta})\\) (notice that it is higher for points \\(z\\) which positively impact the model score, since if they are excluded, the loss is higher). In practice, however, we will always use the infinitesimal approximation \\(\\mathcal{I}_{up}(z, z_{\\text{test}})\\), defined as $$  \\mathcal{I}{up}(z, z}) = - \\frac{d L(z_{\\text{test}}, \\hat{\\theta}{\\epsilon, z})}{d \\epsilon} \\Big| $$</p> <p>Using the chain rule and the results calculated above, we thus have:</p> \\[  \\mathcal{I}_{up}(z, z_{\\text{test}}) = - \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ \\frac{d \\hat{\\theta}_{\\epsilon, z}}{d \\epsilon} \\Big|_{\\epsilon=0} = \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ H_{\\hat{\\theta}}^{-1} \\ \\nabla_\\theta L(z, \\hat{\\theta}) \\] <p>In order to calculate this expression we need the gradient and the Hessian of the loss wrt. the model parameters \\(\\hat{\\theta}\\). This can be easily done through a single backpropagation pass.</p>"},{"location":"examples/influence_imagenet/#regularizing-the-hessian","title":"Regularizing the Hessian","text":"<p>One very important assumption that we make when approximating influence is that \\(\\hat{\\theta}\\) is at least a local minimum of the loss. However, we clearly cannot guarantee this except for convex models, and despite good apparent convergence, \\(\\hat{\\theta}\\) might be located in a region with flat curvature or close to a saddle point. In particular, the Hessian might have vanishing eigenvalues making its direct inversion impossible.</p> <p>To circumvent this problem, instead of inverting the true Hessian \\(H_{\\hat{\\theta}}\\), one can invert a small perturbation thereof: \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\), with \\(\\mathbb{I}\\) being the identity matrix. This standard trick ensures that the eigenvalues of \\(H_{\\hat{\\theta}}\\) are bounded away from zero and therefore the matrix is invertible. In order for this regularization not to corrupt the outcome too much, the parameter \\(\\lambda\\) should be as small as possible while still allowing a reliable inversion of \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\).</p>"},{"location":"examples/influence_synthetic/","title":"For mislabeled data","text":"<pre><code>%load_ext autoreload\n</code></pre> <pre><code>%autoreload\n%matplotlib inline\nimport os\nimport random\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\nfrom pydvl.influence import compute_influences, TorchTwiceDifferentiable\nfrom support.shapley import (\nsynthetic_classification_dataset,\ndecision_boundary_fixed_variance_2d,\n)\nfrom support.common import (\nplot_gaussian_blobs,\nplot_losses,\nplot_influences,\n)\nfrom support.torch import (\nfit_torch_model,\nTorchLogisticRegression,\n)\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\nfrom torch.optim import AdamW, lr_scheduler\nfrom torch.utils.data import DataLoader\n</code></pre> <pre>\n<code>/Users/fabio/miniconda3/envs/pydvl_env/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</code>\n</pre> <pre><code>plt.rcParams[\"figure.figsize\"] = (16, 8)\nplt.rcParams[\"font.size\"] = 12\nplt.rcParams[\"xtick.labelsize\"] = 12\nplt.rcParams[\"ytick.labelsize\"] = 10\n</code></pre> <pre><code>random_state = 24\nis_CI = os.environ.get(\"CI\")\n</code></pre> <pre><code>num_samples = 10000\nnum_features = 2\nsigma = 0.2\nmeans = np.asarray([[0.0, 0.0], [1.0, 1.0]])\n</code></pre> <pre><code>random.seed(random_state)\nnp.random.seed(random_state)\n</code></pre> <p>The following code snippet generates the aforementioned dataset.</p> <pre><code>train_data, val_data, test_data = synthetic_classification_dataset(\nmeans, sigma, num_samples, train_size=0.7, test_size=0.2\n)\n# In CI we only use a subset of the training set\nif is_CI:\ntrain_data = (train_data[0][:10], train_data[1][:10])\n</code></pre> <p>Given the simplicity of the dataset, we can calculate exactly the optimal decision boundary(that which maximizes our accuracy). The following code maps a continuous line of z values to a 2-dimensional vector in feature space (More details are in the appendix to this notebook.)</p> <pre><code>decision_boundary_fn = decision_boundary_fixed_variance_2d(means[0], means[1])\ndecision_boundary = decision_boundary_fn(np.linspace(-1.5, 1.5, 100))\n</code></pre> <pre><code>plot_gaussian_blobs(\ntrain_data,\ntest_data,\nxlabel=\"$x_0$\",\nylabel=\"$x_1$\",\nlegend_title=\"$y - labels$\",\nline=decision_boundary,\ns=10,\nsuptitle=\"Plot of train-test data\",\n)\n</code></pre> <p>Note that there are samples which go across the optimal decision boundary and will be wrongly labelled. The optimal decision boundary can not discriminate these as the mislabelling is a consequence of the presence of random noise.</p> <pre><code>model = TorchLogisticRegression(num_features)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\nnum_epochs = 50\nlr = 0.05\nweight_decay = 0.05\nbatch_size = 256\ntrain_data_loader = DataLoader(\nlist(zip(train_data[0], train_data[1].astype(float))),\nbatch_size=batch_size,\nshuffle=True,\n)\nval_data_loader = DataLoader(\nlist(zip(val_data[0], val_data[1].astype(float))),\nbatch_size=batch_size,\nshuffle=True,\n)\noptimizer = AdamW(params=model.parameters(), lr=lr, weight_decay=weight_decay)\nscheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs)\nlosses = fit_torch_model(\nmodel=model,\ntraining_data=train_data_loader,\nval_data=val_data_loader,\nloss=F.binary_cross_entropy,\noptimizer=optimizer,\nscheduler=scheduler,\nnum_epochs=num_epochs,\n)\n</code></pre> <pre>\n<code>Model fitting:   0%|          | 0/50 [00:00&lt;?, ?it/s]Model fitting: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 50/50 [00:02&lt;00:00, 19.41it/s]\n</code>\n</pre> <p>And let's check that the model is not overfitting</p> <pre><code>plot_losses(losses)\n</code></pre> <p>A look at the confusion matrix also shows good results</p> <pre><code>model.eval()\npred_probabilities = model(test_data[0]).detach()\npred_y_test = [1 if prob &amp;gt; 0.5 else 0 for prob in pred_probabilities]\ncm = confusion_matrix(test_data[1], pred_y_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm)\ndisp.plot();\n</code></pre> <p>It is important that the model converges to a point near the optimum, since the influence values assume that we are at a minimum (or close) in the loss landscape. The function</p> \\[I(x_1, y_1, x_2, y_2) \\colon \\mathbb{R}^d \\times \\mathbb{R}^d \\to \\mathbb{R}\\] <p>measures the influence of the data point \\(x_1\\) onto \\(x_2\\) conditioned on the training targets \\(y_1\\) and \\(y_2\\) trough some model parameters \\(\\theta\\). If the loss function L is differentiable, we can take \\(I\\) to be</p> <p>$$ I(x_1, x_2) = \\nabla_\\theta\\; L(x_1, y_1) ^\\mathsf{T} \\; H_\\theta^{-1} \\; \\nabla_\\theta \\; L(x_2, y_2) $$ See \"Understanding Black-box Predictions via Influence Functions\" for a detailed derivation of this formula</p> <p>Let's take a subset of the training data points, which we will calculate the influence values of. </p> <pre><code>x = train_data[0][:100]\ny = train_data[1][:100]\n</code></pre> <p>In pyDVL, the influence of the training points on the test points can be calculated with the following</p> <pre><code>train_data_loader = DataLoader(list(zip(x, y.astype(float))), batch_size=batch_size)\ntest_data_loader = DataLoader(\nlist(zip(test_data[0], test_data[1].astype(float))), batch_size=batch_size\n)\ninfluence_values = compute_influences(\ndifferentiable_model=TorchTwiceDifferentiable(model, F.binary_cross_entropy),\ntraining_data=train_data_loader,\ntest_data=test_data_loader,\ninfluence_type=\"up\",\ninversion_method=\"direct\",  # use 'cg' for big models\n)\n</code></pre> <p>The above explicitly constructs the Hessian. This can often be computationally expensive and conjugate gradient approximate calculation should be used for bigger models. </p> <p>With the influence type 'up', training influences have shape [NxM] where N is the number of test samples and M is the number of training samples. They therefore associate to each training sample its influence on each test sample.  Influence type 'perturbation', instead, return an array of shape  [NxMxF], where F is the number of features in input, ie. the length of x. </p> <p>In our case, in order to have a value of the total average influence of a point we can just average across training samples.</p> <pre><code>mean_train_influences = np.mean(influence_values.numpy(), axis=0)\n</code></pre> <p>Let's plot the results (adjust colorbar_limits for better color gradient)</p> <pre><code>plot_influences(\nx,\nmean_train_influences,\nline=decision_boundary,\nxlabel=\"$x_0$\",\nylabel=\"$x_1$\",\nsuptitle=\"Influences of input points\",\nlegend_title=\"influence values\",\n# colorbar_limits=(-0.3,),\n);\n</code></pre> <p>We can see that, as we approach the separation line, the influences tend to move away from zero, i.e. the points become more decisive for model training, some in a positive way, some negative.</p> <p>As a further test, let's introduce some labelling errors into \\(y\\) and see how the distribution of the influences changes. Let's flip the first 10 labels and calculate influences</p> <pre><code>y_corrupted = np.copy(y)\ny_corrupted[:10] = [1 - yi for yi in y[:10]]\ntrain_corrupted_data_loader = DataLoader(\nlist(zip(x, y_corrupted.astype(float))), batch_size=batch_size\n)\ninfluence_values = compute_influences(\ndifferentiable_model=TorchTwiceDifferentiable(model, F.binary_cross_entropy),\ntraining_data=train_corrupted_data_loader,\ntest_data=test_data_loader,\ninfluence_type=\"up\",\ninversion_method=\"direct\",\n)\nmean_train_influences = np.mean(influence_values.numpy(), axis=0)\n</code></pre> <pre><code>print(\"Average mislabelled data influence:\", np.mean(mean_train_influences[:10]))\nprint(\"Average correct data influence:\", np.mean(mean_train_influences[10:]))\n</code></pre> <pre>\n<code>Average mislabelled data influence: -0.8225848370029777\nAverage correct data influence: 0.011277048916970962\n</code>\n</pre> <pre><code>plot_influences(\nx,\nmean_train_influences,\ncorrupted_indices=np.array(range(10)),\nline=decision_boundary,\nxlabel=\"$x_0$\",\nylabel=\"$x_1$\",\nsuptitle=\"Influences of input points with corrupted data\",\nlegend_title=\"influence values\",\n# colorbar_limits=(-0.3,),\n);\n</code></pre> <p>Red circles indicate the points which have been corrupted. We can see that the mislabelled data have a more negative average influence on the model, especially those that are farther away from the decision boundary.</p> <p>The \"direct\" method that we have used above involves the inversion of the Hessian matrix of the model. If a model has \\(n\\) training points and \\(\\theta  \\in \\mathbb{R}^p\\) parameters, this requires \\(O(n \\ p^2 + p^3)\\) operations, which for larger models, like neural networks, becomes quickly unfeasible. Conjugate gradient avoids the explicit computation of the Hessian via a technique called implicit Hessian-vector products (HVPs), which typically takes \\(O(n \\ p)\\) operations.</p> <p>In the next cell we will use conjugate gradient to compute the influence factors. Since logistic regression is a very simple model, \"cg\" actually slows computation with respect to the direct method, which in this case is a much better choice. Nevertheless, we are able to verify that the influences calculated with \"cg\" are the same (to a minor error) as those calculated directly.</p> <pre><code>influence_values = compute_influences(\ndifferentiable_model=TorchTwiceDifferentiable(model, F.binary_cross_entropy),\ntraining_data=train_corrupted_data_loader,\ntest_data=test_data_loader,\ninfluence_type=\"up\",\ninversion_method=\"cg\",\nprogress=True,\n)\nmean_train_influences = np.mean(influence_values.numpy(), axis=0)\nprint(\"Average mislabelled data influence:\", np.mean(mean_train_influences[:10]))\nprint(\"Average correct data influence:\", np.mean(mean_train_influences[10:]))\n</code></pre> <pre>\n<code>Batch Test Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 8/8 [00:00&lt;00:00, 17.89it/s]\nBatch Train Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 308.47it/s]\nConjugate gradient: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 2000/2000 [00:16&lt;00:00, 118.24it/s]\nBatch Split Input Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 44.89it/s]</code>\n</pre> <pre>\n<code>Average mislabelled data influence: -0.82248804123547\nAverage correct data influence: 0.01127580743952819\n</code>\n</pre> <pre>\n<code>\n</code>\n</pre> <p>Averages are very similar to the ones calculated through direct method. Same is true for the plot</p> <pre><code>plot_influences(\nx,\nmean_train_influences,\ncorrupted_indices=np.array(range(10)),\nline=decision_boundary,\nxlabel=\"$x_0$\",\nylabel=\"$x_1$\",\nsuptitle=\"Influences of input points with corrupted data\",\nlegend_title=\"influence values\",\n# colorbar_limits=(-0.1, 0.1),\n);\n</code></pre>"},{"location":"examples/influence_synthetic/#influence-functions-for-data-mislabeling","title":"Influence functions for data mislabeling","text":"<p>In this notebook, we will take a closer look at the theory of influence functions with the help of a synthetic dataset. Data mislabeling occurs whenever some examples from a usually big dataset are wrongly-labeled. In real-life this happens fairly often, e.g. as a consequence of human error, or noise in the data. </p> <p>Let's consider a classification problem with the following notation:</p> \\[ \\begin{align*} x_i &amp;\\in \\mathbb{R}^d \\\\ y_i &amp;\\in \\{0, 1\\} \\\\ \\forall i &amp;\\in [ N ] \\end{align*} \\] <p>In other words, we have a dataset containing \\(N\\) samples, each with label 1 or 0. As typical example you can think of y indicating whether a patient has a disease based on some feature representation \\(x\\). </p> <p>Let's now introduce a toy model that will help us delve into the theory and practical utility of influence functions. We will assume that \\(y\\) is a Bernoulli binary random variable while the input \\(x\\) is d-dimensional Gaussian distribution which depends on the label \\(y\\). More precisely:</p> \\[ y_i \\sim \\text{Ber}\\left (0.5 \\right) \\\\ x_i \\sim \\mathcal{N}\\left ((1 - y_i) \\mu_1 + y_i \\mu_2, \\sigma^2 I \\right), \\] <p>with fixed means and diagonal covariance. Implementing the sampling scheme in python is straightforward and can be achieved by first sampling \\(y\\) and afterward \\(x\\).</p>"},{"location":"examples/influence_synthetic/#imports","title":"Imports","text":""},{"location":"examples/influence_synthetic/#constants","title":"Constants","text":""},{"location":"examples/influence_synthetic/#dataset","title":"Dataset","text":""},{"location":"examples/influence_synthetic/#plotting-the-dataset","title":"Plotting the dataset","text":"<p>Let's plot the dataset is plotted with their respective labels and the optimal decision line</p>"},{"location":"examples/influence_synthetic/#training-the-model","title":"Training the model","text":"<p>We will now train a logistic regression model on the training data. This can be done with the following</p>"},{"location":"examples/influence_synthetic/#calculating-influences","title":"Calculating influences","text":""},{"location":"examples/influence_synthetic/#inversion-through-conjugate-gradient","title":"Inversion through conjugate gradient","text":""},{"location":"examples/influence_synthetic/#appendix-calculating-the-decision-boundary","title":"Appendix: Calculating the decision boundary","text":"<p>For obtaining the optimal discriminator one has to solve the equation</p> \\[p(x|y=0)=p(x|y=1)\\] <p>and determine the solution set \\(X\\). Let's take the following probabilities</p> \\[ \\begin{align*} p(x|y=0)&amp;=\\mathcal{N}\\left (\\mu_1, \\sigma^2 I \\right) \\\\ p(x|y=1)&amp;=\\mathcal{N}\\left (\\mu_2, \\sigma^2 I \\right) \\end{align*} \\] <p>For a single fixed diagonal variance parameterized by \\(\\sigma\\), the optimal discriminator lays at points which are equidistant from the means of the two distributions, i.e.</p> \\[ \\begin{align*} \\| x - \\mu_1 \\|^2 &amp;= \\| x - \\mu_2 \\|^2 \\\\ \\| \\mu_1 \\|^2 -2 x^\\mathsf{T} \\mu_1 &amp;= \\| \\mu_2 \\|^2 -2 x^\\mathsf{T} \\mu_2 \\\\ \\implies 0 &amp;= 2 (\\mu_2 - \\mu_1)^\\mathsf{T} x + \\| \\mu_1 \\|^2 - \\| \\mu_2 \\|^2 \\\\ 0 &amp;= \\mu_1^\\mathsf{T}x - \\mu_2^\\mathsf{T}x - \\frac{1}{2} \\mu_1^\\mathsf{T} \\mu_1 + \\frac{1}{2} \\mu_2^\\mathsf{T} \\mu_2 \\end{align*} \\] <p>This is just the implicit description of the line. Solving for the explicit form can be achieved by enforcing a functional form \\(f(z) = x = a z + b\\) with \\(z \\in \\mathbb{R}\\) onto \\(x\\). After the term is inserted in the previous equation</p> \\[ 0 = (\\mu_2 - \\mu_1)^\\mathsf{T} (az + b) + \\frac{1}{2} \\| \\mu_1 \\|^2 - \\| \\mu_2 \\|^2 \\] <p>We can write \\(a\\) since, by symmetry, it is expected to be explicitly orthogonal to \\(\\mu_2 - \\mu_1\\). Then, solving for \\(b\\), the solution can be found to be</p> \\[ f(z) = \\underbrace{\\begin{bmatrix} 0 &amp; 1 \\\\ -1 &amp; 0 \\end{bmatrix} (\\mu_2 - \\mu_1)}_a z + \\underbrace{\\frac{\\mu_1 + \\mu_2}{2}}_b \\]"},{"location":"examples/influence_wine/","title":"For outlier detection","text":"<p>Let's start by loading the imports, the dataset and splitting it into train, validation and test sets. We will use a large test set to have a less noisy estimate of the average influence.</p> <pre><code>%load_ext autoreload\n</code></pre> <pre><code>%autoreload\n%matplotlib inline\nimport os\nimport random\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nfrom support.common import plot_losses\nfrom support.torch import TorchMLP, fit_torch_model\nfrom pydvl.influence import compute_influences, TorchTwiceDifferentiable\nfrom support.shapley import load_wine_dataset\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, f1_score\nfrom torch.optim import Adam, lr_scheduler\nfrom torch.utils.data import DataLoader, TensorDataset\n</code></pre> <pre><code>plt.rcParams[\"figure.figsize\"] = (16, 8)\nplt.rcParams[\"font.size\"] = 12\nplt.rcParams[\"xtick.labelsize\"] = 12\nplt.rcParams[\"ytick.labelsize\"] = 10\n</code></pre> <pre><code>random_state = 24\nis_CI = os.environ.get(\"CI\")\n</code></pre> <pre><code>random.seed(random_state)\nnp.random.seed(random_state)\n</code></pre> <pre><code>training_data, val_data, test_data, feature_names = load_wine_dataset(\ntrain_size=0.3, test_size=0.6\n)\n# In CI we only use a subset of the training set\nif is_CI:\ntrain_data = (training_data[0][:10], training_data[1][:10])\n</code></pre> <p>We will corrupt some of the training points by flipping their labels</p> <pre><code>num_corrupted_idxs = 10\ntraining_data[1][:num_corrupted_idxs] = torch.tensor(\n[(val + 1) % 3 for val in training_data[1][:num_corrupted_idxs]]\n)\n</code></pre> <p>and let's wrap it in a pytorch data loader</p> <pre><code>training_data_loader = DataLoader(\nTensorDataset(*training_data), batch_size=32, shuffle=False\n)\nval_data_loader = DataLoader(TensorDataset(*val_data), batch_size=32, shuffle=False)\ntest_data_loader = DataLoader(TensorDataset(*test_data), batch_size=32, shuffle=False)\n</code></pre> <pre><code>feature_dimension = 13\nnum_classes = 3\nnetwork_size = [16, 16]\nlayers_size = [feature_dimension, *network_size, num_classes]\nnum_epochs = 300\nlr = 0.005\nweight_decay = 0.01\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nnn_model = TorchMLP(layers_size)\nnn_model.to(device)\noptimizer = Adam(params=nn_model.parameters(), lr=lr, weight_decay=weight_decay)\nscheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs)\nlosses = fit_torch_model(\nmodel=nn_model,\ntraining_data=training_data_loader,\nval_data=val_data_loader,\nloss=F.cross_entropy,\noptimizer=optimizer,\nscheduler=scheduler,\nnum_epochs=num_epochs,\n)\n</code></pre> <pre>\n<code>Model fitting: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 300/300 [00:00&lt;00:00, 307.77it/s]\n</code>\n</pre> <p>Let's check that the training has found a stable minimum by plotting the training and validation loss</p> <pre><code>plot_losses(losses)\n</code></pre> <p>Since it is a classification problem, let's also take a look at the confusion matrix on the test set</p> <pre><code>nn_model.eval()\npred_y_test = np.argmax(nn_model(test_data[0]).detach(), axis=1)\ncm = confusion_matrix(test_data[1], pred_y_test)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm)\ndisp.plot();\n</code></pre> <p>And let's compute the f1 score of the model</p> <pre><code>f1_score(test_data[1], pred_y_test, average=\"weighted\")\n</code></pre> <pre>\n<code>0.9906846833902615</code>\n</pre> <p>Let's now move to calculating influences of each point on the total score.</p> <pre><code>train_influences = compute_influences(\nTorchTwiceDifferentiable(nn_model, F.cross_entropy),\ntraining_data=training_data_loader,\ntest_data=test_data_loader,\ninfluence_type=\"up\",\ninversion_method=\"direct\",\nhessian_regularization=0.1,\nprogress=True,\n)\n</code></pre> <pre>\n<code>Batch Test Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:00&lt;00:00, 67.10it/s]\nMVP: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 547/547 [00:00&lt;00:00, 742.01it/s] \nBatch Split Input Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 2/2 [00:00&lt;00:00, 85.02it/s]\n</code>\n</pre> <p>the returned matrix, train_influences, has a quantity of columns equal to the points in the training set, and a number of rows equal to the points in the test set. At each element \\(a_{i,j}\\) it stores the influence that training point \\(j\\) has on the classification of test point \\(i\\).</p> <p>If we take the average across every column of the influences matrix, we obtain an estimate of the overall influence of a training point on the total accuracy of the network.</p> <pre><code>mean_train_influences = np.mean(train_influences.numpy(), axis=0)\n</code></pre> <p>The following histogram shows that there are big differences in score within the training set (notice the log-scale on the y axis).</p> <pre><code>_, ax = plt.subplots()\nax.hist(mean_train_influences[num_corrupted_idxs:], label=\"normal\")\nax.hist(mean_train_influences[:num_corrupted_idxs], label=\"corrupted\", bins=5)\nax.set_title(\"Influece scores distribution\")\nax.set_xlabel(\"influece score\")\nax.set_ylabel(\"number of points\")\nax.legend()\nplt.show()\n</code></pre> <p>We can see that the corrupted points tend to have a negative effect on the model, as expected</p> <pre><code>print(\n\"Average influence of corrupted points: \",\nnp.mean(mean_train_influences[:num_corrupted_idxs]),\n)\nprint(\n\"Average influence of other points: \",\nnp.mean(mean_train_influences[num_corrupted_idxs:]),\n)\n</code></pre> <pre>\n<code>Average influence of corrupted points:  -0.05317057\nAverage influence of other points:  0.034408495\n</code>\n</pre> <p>We have seen how to calculate the influence of single training points on each test point using influence_type 'up'. Using influence_type 'perturbation' we can also calculate the influence of the input features of each point. In the next cell we will calculate the average influence of each feature on training and test points, and ultimately assess which are the most relevant to model performance.</p> <pre><code>feature_influences = compute_influences(\nTorchTwiceDifferentiable(nn_model, F.cross_entropy),\ntraining_data=training_data_loader,\ntest_data=test_data_loader,\ninfluence_type=\"perturbation\",\ninversion_method=\"direct\",\nhessian_regularization=1,\nprogress=True,\n)\n</code></pre> <pre>\n<code>Batch Test Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:00&lt;00:00, 61.20it/s]\nMVP: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 547/547 [00:00&lt;00:00, 1265.72it/s]\nBatch Influence Perturbation: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 2/2 [00:03&lt;00:00,  1.66s/it]\n</code>\n</pre> <pre><code>mean_feature_influences = np.mean(feature_influences.numpy(), axis=(0, 1))\n_, ax = plt.subplots()\nax.bar(feature_names, mean_feature_influences)\nax.set_xlabel(\"training features\")\nax.set_ylabel(\"influence values\")\nax.set_title(\"Average feature influence\")\nplt.xticks(rotation=60)\nplt.show()\n</code></pre> <p>The calculation of the Hessian matrix (necessary to calculate the influences) can be quite numerically challenging, but there are some techniques to speed up its calculation. PyDVL allows to use the full method (\"direct\") or the conjugate gradient method (\"cg\"). The first one should be used only for very small networks (like our current example), while for bigger ones \"cg\" is advisable.</p> <pre><code>cg_train_influences = compute_influences(\nTorchTwiceDifferentiable(nn_model, F.cross_entropy),\ntraining_data=training_data_loader,\ntest_data=test_data_loader,\ninfluence_type=\"up\",\ninversion_method=\"cg\",\nhessian_regularization=0.1,\nprogress=True,\n)\nmean_cg_train_influences = np.mean(cg_train_influences.numpy(), axis=0)\n</code></pre> <pre>\n<code>Batch Test Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:00&lt;00:00, 81.02it/s]\nBatch Train Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 2/2 [00:00&lt;00:00, 535.33it/s]\nConjugate gradient: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 107/107 [00:04&lt;00:00, 22.66it/s]\nBatch Split Input Gradients: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 2/2 [00:00&lt;00:00, 98.91it/s]\n</code>\n</pre> <p>Let's compare the results obtained through conjugate gradient with those from the direct method</p> <pre><code>print(\nf\"Percentage error of cg over direct method:{np.mean(np.abs(mean_cg_train_influences - mean_train_influences)/np.abs(mean_train_influences))*100} %\"\n)\n</code></pre> <pre>\n<code>Percentage error of cg over direct method:1.5124550145628746e-05 %\n</code>\n</pre> <p>This was a quick introduction to the pyDVL interface for influence functions. Despite their speed and simplicity, influence functions are known to be a very noisy estimator of data quality, as pointed out in the paper \"Influence functions in deep learning are fragile\". The size of the network, the weight decay, the inversion method used for calculating influences, the size of the test set: they all add up to the total amount of noise. Experiments may therefore give quantitative and qualitatively different results if not averaged across several realisations. Shapley values, on the contrary, have shown to be a more robust, but this comes at the cost of high computational requirements. PyDVL employs several parallelization and caching techniques to optimize such calculations.</p>"},{"location":"examples/influence_wine/#influence-functions-for-outlier-detection","title":"Influence functions for outlier detection","text":"<p>This notebook shows how to calculate influences on a NN model using pyDVL for an arbitrary dataset, and how this can be used to find anomalous or corrupted data points.</p> <p>It uses the wine dataset from sklearn: given a set of 13 different input parameters regarding a particular bottle, each related to some physical property (e.g. concentration of magnesium, malic acidity, alcoholic percentage, etc.), the model will need to predict to which of 3 classes the wine belongs to. For more details, please refer to the sklearn documentation.</p>"},{"location":"examples/influence_wine/#imports","title":"Imports","text":""},{"location":"examples/influence_wine/#constants","title":"Constants","text":""},{"location":"examples/influence_wine/#dataset","title":"Dataset","text":""},{"location":"examples/influence_wine/#fit-a-neural-network-to-the-data","title":"Fit a neural network to the data","text":"<p>We will train a 2-layer neural network. PyDVL has some convenience wrappers to initialize a pytorch NN. If you already have a model loaded and trained, you can skip this section.</p>"},{"location":"examples/influence_wine/#calculating-influences-for-small-neural-networks","title":"Calculating influences for small neural networks","text":"<p>The following cell calculates the influences of each training data point on the neural network.  Neural networks have typically a very bumpy parameter space, which, during training, is explored until the configuration that minimises the loss is found. There is an important assumption in influence functions that the model lays at a (at least local) minimum of such loss, and if this is not fulfilled many issues can arise. In order to avoid this scenario, a regularisation term should be used whenever dealing with big and noisy models. </p>"},{"location":"examples/influence_wine/#influence-of-training-features","title":"Influence of training features","text":""},{"location":"examples/influence_wine/#speeding-up-influences-for-big-models","title":"Speeding up influences for big models","text":""},{"location":"examples/least_core_basic/","title":"Least Core","text":"<p>We will be using the following functions and classes from pyDVL.</p> <pre><code>%autoreload\nfrom pydvl.utils import (\nDataset,\nUtility,\n)\nfrom pydvl.value import compute_least_core_values, LeastCoreMode, ValuationResult\nfrom pydvl.reporting.plots import shaded_mean_std\nfrom pydvl.reporting.scores import compute_removal_score\n</code></pre> <pre><code>X, y = make_classification(\nn_samples=200,\nn_features=50,\nn_informative=25,\nn_classes=3,\nrandom_state=random_state,\n)\n</code></pre> <pre><code>full_dataset = Dataset.from_arrays(\nX, y, stratify_by_target=True, random_state=random_state\n)\nsmall_dataset = Dataset.from_arrays(\nX,\ny,\nstratify_by_target=True,\ntrain_size=10,\nrandom_state=random_state,\n)\n</code></pre> <pre><code>model = LogisticRegression(max_iter=500, solver=\"liblinear\")\n</code></pre> <pre><code>model.fit(full_dataset.x_train, full_dataset.y_train)\nprint(\nf\"Training accuracy: {100 * model.score(full_dataset.x_train, full_dataset.y_train):0.2f}%\"\n)\nprint(\nf\"Testing accuracy: {100 * model.score(full_dataset.x_test, full_dataset.y_test):0.2f}%\"\n)\n</code></pre> <pre>\n<code>Training accuracy: 86.25%\nTesting accuracy: 70.00%\n</code>\n</pre> <pre><code>model.fit(small_dataset.x_train, small_dataset.y_train)\nprint(\nf\"Training accuracy: {100 * model.score(small_dataset.x_train, small_dataset.y_train):0.2f}%\"\n)\nprint(\nf\"Testing accuracy: {100 * model.score(small_dataset.x_test, small_dataset.y_test):0.2f}%\"\n)\n</code></pre> <pre>\n<code>Training accuracy: 100.00%\nTesting accuracy: 47.89%\n</code>\n</pre> <pre><code>utility = Utility(model=model, data=small_dataset)\n</code></pre> <pre><code>exact_values = compute_least_core_values(\nu=utility,\nmode=LeastCoreMode.Exact,\nprogress=True,\n)\n</code></pre> <pre>\n<code>  0%|          | 0/1023 [00:00&lt;?, ?it/s]</code>\n</pre> <pre><code>exact_values_df = exact_values.to_dataframe(column=\"exact_value\").T\nexact_values_df = exact_values_df[sorted(exact_values_df.columns)]\n</code></pre> <pre><code>budget_array = np.linspace(200, 2 ** len(small_dataset), num=10, dtype=int)\nall_estimated_values_df = []\nall_errors = {budget: [] for budget in budget_array}\nfor budget in tqdm(budget_array):\ndfs = []\nerrors = []\ncolumn_name = f\"estimated_value_{budget}\"\nfor i in range(20):\nvalues = compute_least_core_values(\nu=utility,\nmode=LeastCoreMode.MonteCarlo,\nn_iterations=budget,\nn_jobs=n_jobs,\n)\ndf = (\nvalues.to_dataframe(column=column_name)\n.drop(columns=[f\"{column_name}_stderr\"])\n.T\n)\ndf = df[sorted(df.columns)]\nerror = mean_squared_error(\nexact_values_df.loc[\"exact_value\"].values, df.values.ravel()\n)\nall_errors[budget].append(error)\ndf[\"budget\"] = budget\ndfs.append(df)\nestimated_values_df = pd.concat(dfs)\nall_estimated_values_df.append(estimated_values_df)\nvalues_df = pd.concat(all_estimated_values_df)\nerrors_df = pd.DataFrame(all_errors)\n</code></pre> <pre>\n<code>  0%|          | 0/10 [00:00&lt;?, ?it/s]</code>\n</pre> <p>We can see that the approximation error decreases, on average, as the we increase the budget. </p> <p>Still, the decrease may not always necessarily happen when we increase the number of iterations because of the fact that we sample the subsets with replacement in the Monte Carlo method i.e there may be repeated subsets.</p> <pre><code>utility = Utility(model=model, data=full_dataset)\n</code></pre> <pre><code>method_names = [\"Random\", \"Least Core\"]\nremoval_percentages = np.arange(0, 0.41, 0.05)\n</code></pre> <pre><code>all_scores = []\nfor i in trange(5):\nfor method_name in method_names:\nif method_name == \"Random\":\nvalues = ValuationResult.from_random(size=len(utility.data))\nelse:\nvalues = compute_least_core_values(\nu=utility,\nmode=LeastCoreMode.MonteCarlo,\nn_iterations=25000,\nn_jobs=n_jobs,\n)\nscores = compute_removal_score(\nu=utility,\nvalues=values,\npercentages=removal_percentages,\nremove_best=True,\n)\nscores[\"method_name\"] = method_name\nall_scores.append(scores)\nscores_df = pd.DataFrame(all_scores)\n</code></pre> <pre>\n<code>  0%|          | 0/5 [00:00&lt;?, ?it/s]</code>\n</pre> <p>We can clearly see that removing the most valuable data points, as given by the Least Core method, leads to, on average, a decrease in the model's performance and that the method outperforms random removal of data points.</p> <pre><code>all_scores = []\nfor i in trange(5):\nfor method_name in method_names:\nif method_name == \"Random\":\nvalues = ValuationResult.from_random(size=len(utility.data))\nelse:\nvalues = compute_least_core_values(\nu=utility,\nmode=LeastCoreMode.MonteCarlo,\nn_iterations=25000,\nn_jobs=n_jobs,\n)\nscores = compute_removal_score(\nu=utility,\nvalues=values,\npercentages=removal_percentages,\n)\nscores[\"method_name\"] = method_name\nall_scores.append(scores)\nscores_df = pd.DataFrame(all_scores)\n</code></pre> <pre>\n<code>  0%|          | 0/5 [00:00&lt;?, ?it/s]</code>\n</pre> <p>We can clearly see that removing the least valuable data points, as given by the Least Core method, leads to, on average, an increase in the model's performance and that the method outperforms the random removal of data points.</p>"},{"location":"examples/least_core_basic/#least-core-for-data-valuation","title":"Least Core for Data Valuation","text":"<p>This notebook introduces Least Core methods for the computation of data values using pyDVL.</p> <p>Shapley values define a fair way of distributing the worth of the whole training set when every data point is part of it. But they do not consider the question of stability of subsets: Could some data points obtain a higher payoff if they formed smaller subsets? It is argued that this might be relevant if data providers are paid based on data value, since Shapley values can incentivise them not to contribute their data to the \"grand coalition\", but instead try to form smaller ones. Whether this is of actual practical relevance is debatable, but in any case, the least core is an alternative tool available for any task of Data Valuation</p> <p>The Core is another approach to compute data values originating in cooperative game theory that attempts to answer those questions. It is the set of feasible payoffs that cannot be improved upon by a coalition of the participants.</p> <p>Its use for Data Valuation was first described in the paper If You Like Shapley Then You\u2019ll Love the Core by Tom Yan and Ariel D. Procaccia.</p> <p>The Least Core value \\(v\\) of the \\(i\\)-th sample in dataset \\(D\\) wrt. utility \\(u\\) is computed by solving the following Linear Program:</p> \\[ \\begin{array}{lll} \\text{minimize} &amp; \\displaystyle{e} &amp; \\\\ \\text{subject to} &amp; \\displaystyle\\sum_{x_i\\in D} v_u(x_i) = u(D) &amp; \\\\ &amp; \\displaystyle\\sum_{x_i\\in S} v_u(x_i) + e \\geq u(S) &amp;, \\forall S \\subset D, S \\neq \\emptyset \\\\ \\end{array} \\] <p>To illustrate this method we will use a synthetic dataset. We will first use a subset of 10 data point to compute the exact values and use them to assess the Monte Carlo approximation. Afterwards, we will conduct the data removal experiments as described by Ghorbani and Zou in their paper Data Shapley: Equitable Valuation of Data for Machine Learning: We compute the data valuation given different computation budgets and incrementally remove a percentage of the best, respectively worst, data points and observe how that affects the utility.</p>"},{"location":"examples/least_core_basic/#setup","title":"Setup","text":"<p>We begin by importing the main libraries and setting some defaults.</p>   If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/least_core_basic/#dataset","title":"Dataset","text":"<p>We generate a synthetic dataset using the <code>make_classification</code> function from scikit-learn.</p> <p>We sample 200 data points from a 50-dimensional Gaussian distribution with 25 informative features and 25 non-informative features (generated as random linear combinations of the informative features).</p> <p>The 200 samples are uniformly distributed across 3 classes with a small percentage of noise added to the labels to make the task a bit more difficult.</p>"},{"location":"examples/least_core_basic/#estimating-least-core-values","title":"Estimating Least Core Values","text":"<p>In this first section we will use a smaller subset of the dataset containing 10 samples in order to be able to compute exact values in a reasonable amount of time. Afterwards, we will use the Monte Carlo method with a limited budget (maximum number of subsets) to approximate these values.</p>"},{"location":"examples/least_core_basic/#data-removal","title":"Data Removal","text":"<p>We now move on to the data removal experiments using the full dataset.</p> <p>In these experiments, we first rank the data points from most valuable  to least valuable using the values estimated by the Monte Carlo Least Core method. Then, we gradually remove from 5 to 40 percent, by increments of 5 percentage points, of the most valuable/least valuable ones, train the model on this subset and compute its accuracy.</p>"},{"location":"examples/least_core_basic/#remove-best","title":"Remove Best","text":"<p>We start by removing the best data points and seeing how the model's accuracy evolves.</p>"},{"location":"examples/least_core_basic/#remove-worst","title":"Remove Worst","text":"<p>We then proceed to removing the worst data points and seeing how the model's accuracy evolves.</p>"},{"location":"examples/shapley_basic_spotify/","title":"Shapley values","text":"<p>This notebook introduces Shapley methods for the computation of data value using pyDVL.</p> <p>In order to illustrate the practical advantages, we will predict the popularity of songs in the dataset Top Hits Spotify from 2000-2019, and highlight how data valuation can help investigate and boost the performance of the models. In doing so, we will describe the basic usage patterns of pyDVL.</p> <p>Recall that data value is a function of three things:</p> <ol> <li>The dataset.</li> <li>The model.</li> <li>The performance metric or scoring function.</li> </ol> <p>Below we will describe how to instantiate each one of these objects and how to use them for data valuation. Please also see the documentation on data valuation.</p> <p>We will be using the following functions from pyDVL. The main entry point is the function compute_shapley_values(), which provides a facade to all Shapley methods. In order to use it we need the classes Dataset, Utility and Scorer.</p> <pre><code>%autoreload\nfrom pydvl.reporting.plots import plot_shapley\nfrom pydvl.utils.dataset import GroupedDataset\nfrom support.shapley import load_spotify_dataset\nfrom pydvl.value import *\n</code></pre> <pre><code>training_data, val_data, test_data = load_spotify_dataset(\nval_size=0.3, test_size=0.3, target_column=\"popularity\", random_state=random_state\n)\n</code></pre> <pre><code>training_data[0].head()\n</code></pre> artist song duration_ms explicit year danceability energy key loudness mode speechiness acousticness instrumentalness liveness valence tempo genre 1561 Fetty Wap 679 (feat. Remy Boyz) 196693 True 2015 0.618 0.717 7 -5.738 1 0.3180 0.00256 0.000000 0.6250 0.603 190.050 8 1410 Meghan Trainor All About That Bass 187920 True 2015 0.807 0.887 9 -3.726 1 0.0503 0.05730 0.000003 0.1240 0.961 134.052 14 1772 Katy Perry Chained To The Rhythm 237733 False 2017 0.562 0.800 0 -5.404 1 0.1120 0.08140 0.000000 0.1990 0.471 95.029 14 1670 Sigala Sweet Lovin' - Radio Edit 202149 False 2015 0.683 0.910 10 -1.231 1 0.0515 0.05530 0.000005 0.3360 0.674 124.977 15 1780 Liam Payne Strip That Down 204502 False 2017 0.869 0.485 6 -5.595 1 0.0545 0.24600 0.000000 0.0765 0.527 106.028 14 <p>The dataset has many high-level features, some quite intuitive ('duration_ms' or 'tempo'), while others are a bit more cryptic ('valence'?). For information on each feature, please consult the dataset's website.</p> <p>In our analysis, we will use all the columns, except for 'artist' and 'song', to predict the 'popularity' of each song. We will nonetheless keep the information on song and artist in a separate object for future reference.</p> <pre><code>song_name = training_data[0][\"song\"]\nartist = training_data[0][\"artist\"]\ntraining_data[0] = training_data[0].drop([\"song\", \"artist\"], axis=1)\ntest_data[0] = test_data[0].drop([\"song\", \"artist\"], axis=1)\nval_data[0] = val_data[0].drop([\"song\", \"artist\"], axis=1)\n</code></pre> <p>Input and label data are then used to instantiate a Dataset object:</p> <pre><code>dataset = Dataset(*training_data, *val_data)\n</code></pre> <p>The calculation of exact Shapley values is computationally very expensive (exponentially so!) because it requires training the model on every possible subset of the training set. For this reason, PyDVL implements techniques to speed up the calculation, such as Monte Carlo approximations, surrogate models or caching of intermediate results and grouping of data to calculate group Shapley values instead of single data points.</p> <p>In our case, we will group songs by artist and calculate the Shapley value for the artists. Given the pandas Series for 'artist', to group the dataset by it, one does the following:</p> <pre><code>grouped_dataset = GroupedDataset.from_dataset(dataset=dataset, data_groups=artist)\n</code></pre> <pre><code>utility = Utility(\nmodel=GradientBoostingRegressor(n_estimators=3),\ndata=grouped_dataset,\nscorer=Scorer(\"neg_mean_absolute_error\", default=0.0),\n)\nvalues = compute_shapley_values(\nutility,\nmode=ShapleyMode.TruncatedMontecarlo,\n# Stop if the standard error is below 1% of the range of the values (which is ~2),\n# or if the number of updates exceeds 1000\ndone=AbsoluteStandardError(threshold=0.2, fraction=0.9) | MaxUpdates(1000),\ntruncation=RelativeTruncation(utility, rtol=0.01),\nn_jobs=-1,\n)\nvalues.sort(key=\"value\")\ndf = values.to_dataframe(column=\"data_value\", use_names=True)\n</code></pre> <p>The function compute_shapley_values() serves as a common access point to all Shapley methods. For most of them, we must choose a <code>StoppingCriterion</code> with the argument <code>done=</code>. In this case we choose to stop when the ratio of standard error to value is below 0.2 for at least 90% of the training points, or if the number of updates of any index exceeds 1000. The <code>mode</code> argument specifies the Shapley method to use. In this case, we use the Truncated Monte Carlo approximation, which is the fastest of the Monte Carlo methods, owing both to using the permutation definition of Shapley values and the ability to truncate the iteration over a given permutation. We configure this to happen when the contribution of the remaining elements is below 1% of the total utility with the parameter <code>truncation=</code> and the policy RelativeTruncation.</p> <p>Let's take a look at the returned dataframe:</p> <pre><code>df.head()\n</code></pre> data_value data_value_stderr Kendrick Lamar -1.279149 0.091670 BLACKPINK -1.277363 0.177476 Adele -1.241698 0.183732 5 Seconds of Summer -1.228002 0.103377 Flume -1.197065 0.102345 <p>The first thing to notice is that we sorted the results in ascending order of Shapley value. The index holds the labels for each data group: in this case, artist names. The column <code>data_value</code> is just that: the Shapley Data value, and <code>data_value_stderr</code> is its estimated standard error because we are using a Monte Carlo approximation.</p> <p>Let us plot the results. In the next cell we will take the 30 artists with the lowest score and plot their values with 95% Normal confidence intervals. Keep in mind that Monte Carlo Shapley is typically very noisy, and it can take many steps to arrive at a clean estimate.</p> <p>We can immediately see that many artists (groups of samples) have very low, even negative value, which means that they tend to decrease the total score of the model when present in the training set! What happens if we remove them?</p> <p>In the next cell we create a new training set excluding the artists with the lowest scores:</p> <pre><code>low_dvl_artists = df.iloc[:30].index.to_list()\nartist_filter = ~artist.isin(low_dvl_artists)\nX_train_good_dvl = training_data[0][artist_filter]\ny_train_good_dvl = training_data[1][artist_filter]\n</code></pre> <p>Now we will use this \"cleaned\" dataset to retrain the same model and compare its mean absolute error to the one trained on the full dataset. Notice that the score now is calculated using the test set, while in the calculation of the Shapley values we were using the validation set.</p> <pre><code>model_good_data = GradientBoostingRegressor(n_estimators=3).fit(\nX_train_good_dvl, y_train_good_dvl\n)\nerror_good_data = mean_absolute_error(\nmodel_good_data.predict(test_data[0]), test_data[1]\n)\nmodel_all_data = GradientBoostingRegressor(n_estimators=3).fit(\ntraining_data[0], training_data[1]\n)\nerror_all_data = mean_absolute_error(model_all_data.predict(test_data[0]), test_data[1])\nprint(f\"Improvement: {100*(error_all_data - error_good_data)/error_all_data:02f}%\")\n</code></pre> <pre>\n<code>Improvement: 13.940685%\n</code>\n</pre> <p>The score has improved by almost 14%! This is quite an important result, as it shows a consistent process to improve the performance of a model by excluding data points from its training set.</p>   One must however proceed with caution instead of simply throwing away data. For one, `mean_absolute_error` is an estimate of generalization error on unseen data, so the improvement we see on the test set might not be as large upon deployment. It would be advisable to cross-validate this whole process to obtain more conservative estimates. It is also advisable to manually inspect the artists with low value and to try to understand the reason why the model behaves like it does. Finally, remember that **the value depends on the model chosen**! Artists that are detrimental to the Gradient Boosting Regressor might be informative for a different model (although it is likely that the worst ones share some characteristic making them \"bad\" for other regressors).   <p>Let us take all the songs by Rihanna, set their score to 0 and re-calculate the Shapley values.</p> <pre><code>y_train_anomalous = training_data[1].copy(deep=True)\ny_train_anomalous[artist == \"Rihanna\"] = 0\nanomalous_dataset = Dataset(\nx_train=training_data[0],\ny_train=y_train_anomalous,\nx_test=val_data[0],\ny_test=val_data[1],\n)\ngrouped_anomalous_dataset = GroupedDataset.from_dataset(anomalous_dataset, artist)\nanomalous_utility = Utility(\nmodel=GradientBoostingRegressor(n_estimators=3),\ndata=grouped_anomalous_dataset,\nscorer=Scorer(\"neg_mean_absolute_error\", default=0.0),\n)\nvalues = compute_shapley_values(\nanomalous_utility,\nmode=ShapleyMode.TruncatedMontecarlo,\ndone=AbsoluteStandardError(threshold=0.2, fraction=0.9) | MaxUpdates(1000),\nn_jobs=-1,\n)\nvalues.sort(key=\"value\")\ndf = values.to_dataframe(column=\"data_value\", use_names=True)\n</code></pre> <p>Let us now consider the low-value artists (at least for predictive purposes, no claims are made about their artistic value!) and plot the results</p> <p>And Rihanna (our anomalous data group) has moved from top contributor to having negative impact on the performance of the model, as expected!</p> <p>What is going on? A popularity of 0 for Rihanna's songs is inconsistent with listening patterns for other artists. In artificially setting this, we degrade the predictive power of the model.</p> <p>By dropping low-value groups or samples, one can often increase model performance, but by inspecting them, it is possible to identify bogus data sources or acquisition methods.</p>"},{"location":"examples/shapley_basic_spotify/#shapley-for-data-valuation","title":"Shapley for data valuation","text":""},{"location":"examples/shapley_basic_spotify/#setup","title":"Setup","text":"<p>We begin by importing the main libraries and setting some defaults.</p>   If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/shapley_basic_spotify/#loading-and-grouping-the-dataset","title":"Loading and grouping the dataset","text":"<p>pyDVL provides a support function for this notebook, <code>load_spotify_dataset()</code>, which downloads data on songs published after 2014, and splits 30% of data for testing, and 30% of the remaining data for validation. The return value is a triple of training, validation and test data as lists of the form <code>[X_input, Y_label]</code>.</p>"},{"location":"examples/shapley_basic_spotify/#creating-the-utility-and-computing-values","title":"Creating the utility and computing values","text":"<p>Now we can calculate the contribution of each group to the model performance.</p> <p>As a model, we use scikit-learn's GradientBoostingRegressor, but pyDVL can work with any model from sklearn, xgboost or lightgbm. More precisely, any model that implements the protocol pydvl.utils.types.SupervisedModel, which is just the standard sklearn interface of <code>fit()</code>,<code>predict()</code> and <code>score()</code> can be used to construct the utility.</p> <p>The third and final component is the scoring function. It can be anything like accuracy or \\(R^2\\), and is set with a string from the standard sklearn scoring methods. Please refer to that documentation on information on how to define your own scoring function.</p> <p>We group dataset, model and scoring function into an instance of Utility.</p>"},{"location":"examples/shapley_basic_spotify/#evaluation-on-anomalous-data","title":"Evaluation on anomalous data","text":"<p>One interesting test is to corrupt some data and to monitor how their value changes. To do this, we will take one of the artists with the highest value and set the popularity of all their songs to 0.</p>"},{"location":"examples/shapley_knn_flowers/","title":"KNN Shapley","text":"<p>This notebook shows how to calculate Shapley values for the K-Nearest Neighbours algorithm. By making use of the local structure of KNN, it is possible to compute an exact value in almost linear time, as opposed to exponential complexity of exact, model-agnostic Shapley.</p> <p>The main idea is to exploit the fact that adding or removing points beyond the k-ball doesn't influence the score. Because the algorithm then essentially only needs to do a search it runs in \\(\\mathcal{O}(N \\log N)\\) time.</p> <p>By further using approximate nearest neighbours, it is possible to achieve \\((\\epsilon,\\delta)\\)-approximations in sublinear time. However, this is not implemented in pyDVL yet.</p> <p>We refer to the original paper that pyDVL implements for details: Jia, Ruoxi, David Dao, Boxin Wang, Frances Ann Hubis, Nezihe Merve Gurel, Bo Li, Ce Zhang, Costas Spanos, and Dawn Song. Efficient Task-Specific Data Valuation for Nearest Neighbor Algorithms. Proceedings of the VLDB Endowment 12, no. 11 (1 July 2019): 1610\u201323.</p> <p>The main entry point is the function compute_shapley_values(), which provides a facade to all Shapley methods. In order to use it we need the classes Dataset, Utility and Scorer, all of which can be imported from <code>pydvl.value</code>:</p> <pre><code>from pydvl.value import *\n</code></pre> <pre><code>sklearn_dataset = sk.datasets.load_iris()\ndata = Dataset.from_sklearn(sklearn_dataset)\nknn = sk.neighbors.KNeighborsClassifier(n_neighbors=5)\nutility = Utility(knn, data)\n</code></pre> <pre><code>shapley_values = compute_shapley_values(utility, mode=ShapleyMode.KNN, progress=True)\nshapley_values.sort(key=\"value\")\nvalues = shapley_values.values\n</code></pre> <pre>\n<code>0it [00:00, ?it/s]</code>\n</pre> <p>If we now look at  the distribution of Shapley values for each class, we see that each has samples with both high and low scores. This is expected, because an accurate model uses information of all classes.</p> <pre><code>corrupted_data = deepcopy(data)\nn_corrupted = 10\ncorrupted_data.y_train[:n_corrupted] = (corrupted_data.y_train[:n_corrupted] + 1) % 3\nknn = sk.neighbors.KNeighborsClassifier(n_neighbors=5)\ncontaminated_values = compute_shapley_values(\nUtility(knn, corrupted_data), mode=ShapleyMode.KNN\n)\n</code></pre> <p>Taking the average corrupted value and comparing it to non-corrupted ones, we notice that on average anomalous points have a much lower score, i.e. they tend to be much less valuable to the model.</p> <p>To do this, first we make sure that we access the results by data index with a call to <code>ValuationResult.sort()</code>, then we split the values into two groups: corrupted and non-corrupted. Note how we access property <code>values</code> of the <code>ValuationResult</code> object. This is a numpy array of values, sorted however the object was sorted. Finally, we compute the quantiles of the two groups and compare them. We see that the corrupted mean is in the lowest percentile of the value distribution, while the correct mean is in the 70th percentile.</p> <pre><code>contaminated_values.sort(\nkey=\"index\"\n)  # This is redundant, but illustrates sorting, which is in-place\ncorrupted_shapley_values = contaminated_values.values[:n_corrupted]\ncorrect_shapley_values = contaminated_values.values[n_corrupted:]\nmean_corrupted = np.mean(corrupted_shapley_values)\nmean_correct = np.mean(correct_shapley_values)\npercentile_corrupted = np.round(100 * np.mean(values &amp;lt; mean_corrupted), 0)\npercentile_correct = np.round(100 * np.mean(values &amp;lt; mean_correct), 0)\nprint(\nf\"The corrupted mean is at percentile {percentile_corrupted:.0f} of the value distribution.\"\n)\nprint(\nf\"The correct mean is percentile {percentile_correct:.0f} of the value distribution.\"\n)\n</code></pre> <pre>\n<code>The corrupted mean is at percentile 1 of the value distribution.\nThe correct mean is percentile 71 of the value distribution.\n</code>\n</pre> <p>This is confirmed if we plot the distribution of Shapley values and circle corrupt points in red. They all tend to have low Shapley scores, regardless of their position in space and assigned label:</p>"},{"location":"examples/shapley_knn_flowers/#knn-shapley","title":"KNN Shapley","text":""},{"location":"examples/shapley_knn_flowers/#setup","title":"Setup","text":"<p>We begin by importing the main libraries and setting some defaults.</p>   If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/shapley_knn_flowers/#building-a-dataset-and-a-utility","title":"Building a Dataset and a Utility","text":"<p>We use the sklearn iris dataset and wrap it into a pydvl.utils.dataset.Dataset calling the factory pydvl.utils.dataset.Dataset.from_sklearn(). This automatically creates a train/test split for us which will be used to compute the utility.</p> <p>We then create a model and instantiate a Utility using data and model. The model needs to implement the protocol pydvl.utils.types.SupervisedModel, which is just the standard sklearn interface of <code>fit()</code>,<code>predict()</code> and <code>score()</code>. In constructing the <code>Utility</code> one can also choose a scoring function, but we pick the default which is just the model's <code>knn.score()</code>.</p>"},{"location":"examples/shapley_knn_flowers/#computing-values","title":"Computing values","text":"<p>Calculating the Shapley values is straightforward. We just call compute_shapley_values() with the utility object we created above. The function returns a ValuationResult. This object contains the values themselves, data indices and labels.</p>"},{"location":"examples/shapley_knn_flowers/#inspecting-the-results","title":"Inspecting the results","text":"<p>Let us first look at the labels' distribution as a function of petal and sepal length:</p>"},{"location":"examples/shapley_knn_flowers/#corrupting-labels","title":"Corrupting labels","text":"<p>To test how informative values are, we can corrupt some training labels and see how their Shapley values change with respect to the non-corrupted points.</p>"},{"location":"examples/shapley_utility_learning/","title":"Data utility learning","text":"<p>This notebook introduces Data Utility Learning, a method of approximating Data Shapley values by learning to estimate the utility function.</p> <p>The idea is to employ a model to learn the performance of the learning algorithm of interest on unseen data combinations (i.e. subsets of the dataset). The method was originally described in Wang, Tianhao, Yu Yang, and Ruoxi Jia. Improving Cooperative Game Theory-Based Data Valuation via Data Utility Learning. arXiv, 2022.</p> <p>Warning: Work on Data Utility Learning is preliminary. It remains to be seen when or whether it can be put effectively into application. For this further testing and benchmarking are required.</p> <p>Recall the definition of Shapley value \\(v_u(i)\\) for data point \\(i\\):</p> \\[\\begin{equation} v_u(i) = \\frac{1}{n} \\sum_{S \\subseteq N \\setminus \\{i\\}} \\binom{n-1}{|S|}^{-1} [u(S \\cup \\{i\\}) \u2212 u(S)] , \\tag{1} \\label{eq:shapley-def} \\end{equation}\\] <p>where \\(N\\) is the set of all indices in the training set and \\(u\\) is the utility.</p> <p>In Data Utility Learning, to avoid the exponential cost of computing this sum, one learns a surrogate model for \\(u\\). We start by sampling so-called utility samples to form a training set \\(S_\\mathrm{train}\\) for our utility model. Each utility sample is a tuple consisting of a subset of indices \\(S_j\\) in the dataset and its utility \\(u(S_j)\\):</p> \\[\\mathcal{S}_\\mathrm{train} = \\{(S_j, u(S_j): j = 1 , ..., m_\\mathrm{train}\\}\\] <p>where \\(m_\\mathrm{train}\\) denotes the training budget for the learned utility function.</p> <p>The subsets are then transformed into boolean vectors \\(\\phi\\) in which a \\(1\\) at index \\(k\\) means that the \\(k\\)-th sample of the dataset is present in the subset:</p> \\[S_j \\mapsto \\phi_j \\in \\{ 0, 1 \\}^{N}\\] <p>We fit a regression model \\(\\tilde{u}\\), called data utility model, on the transformed utility samples \\(\\phi (\\mathcal{S}_\\mathrm{train}) := \\{(\\phi(S_j), u(S_j): j = 1 , ..., m_\\mathrm{train}\\}\\) and use it to predict instead of computing the utility for any \\(S_j \\notin \\mathcal{S}_\\mathrm{train}\\). We abuse notation and identify \\(\\tilde{u}\\) with the composition \\(\\tilde{u} \\circ \\phi : N \\rightarrow \\mathbb{R}\\).</p> <p>The main assumption is that it is much faster to fit and use \\(\\tilde{u}\\) than it is to compute \\(u\\) and that for most \\(i\\), \\(v_\\tilde{u}(i) \\approx v_u(i)\\) in some sense.</p> <p>As is the case with all other Shapley methods, the main entry point is the function compute_shapley_values(), which provides a facade to all algorithms in this family. We use it with the usual classes Dataset and Utility. In addition, we must import the core class for learning a utility, DataUtilityLearning.</p> <pre><code>%autoreload\nfrom pydvl.utils import DataUtilityLearning, top_k_value_accuracy\nfrom pydvl.reporting.plots import shaded_mean_std\nfrom pydvl.value import *\n</code></pre> <pre><code>dataset = Dataset.from_sklearn(\nload_iris(), train_size=0.1, random_state=random_state, stratify_by_target=True\n)\n</code></pre> <p>We verify that, as in the paper, if we fit a Support-Vector Classifier to the training data, we obtain an accuracy of around 92%:</p> <pre><code>model = LinearSVC()\nmodel.fit(dataset.x_train, dataset.y_train)\nprint(f\"Mean accuracy: {100 * model.score(dataset.x_test, dataset.y_test):0.2f}%\")\n</code></pre> <pre>\n<code>Mean accuracy: 92.59%\n</code>\n</pre> <pre><code>computation_times = {}\n</code></pre> <pre><code>utility = Utility(model=model, data=dataset)\n</code></pre> <pre><code>start_time = time.monotonic()\nresult = compute_shapley_values(\nu=utility,\nmode=ShapleyMode.CombinatorialExact,\nn_jobs=-1,\nprogress=False,  # Does not display correctly in a notebook\n)\ncomputation_time = time.monotonic() - start_time\ncomputation_times[\"exact\"] = computation_time\ndf = result.to_dataframe(column=\"exact\").drop(columns=[\"exact_stderr\"])\n</code></pre> <p>We now estimate the Data Shapley values using the DataUtilityLearning wrapper. This class wraps a Utility and delegates calls to it, up until a given budget. Every call yields a utility sample which is saved under the hood for training of the given utility model. Once the budget is exhausted, <code>DataUtilityLearning</code> fits the model to the utility samples and all subsequent calls use the learned model to predict the wrapped utility instead of delegating to it.</p> <p>For the utility model we follow the paper and use a fully connected neural network. To train it we use a total of <code>training_budget</code> utility samples. We repeat this multiple times for each training budget.</p>   Note how we use a MonteCarlo approximation instead of `combinatorial_exact` as before. This is because the exact computation samples subsets in a particular order, from the lowest size to the largest. Because the training budget for the model to learn the utility is around 1/4th of the total number of subsets, this would mean that we would never see utility samples for the larger sizes and the model would be biased (try it!)   <pre><code>mlp_kwargs = dict(\nhidden_layer_sizes=(20, 10),\nactivation=\"relu\",\nsolver=\"adam\",\nlearning_rate_init=0.001,\nbatch_size=32,\nmax_iter=800,\n)\nprint(\nf\"Doing {n_runs} runs for each of {len(training_budget_values)} different training budgets.\"\n)\npbar = tqdm(\nproduct(range(n_runs), training_budget_values),\ntotal=n_runs * len(training_budget_values),\n)\nfor idx, budget in pbar:\npbar.set_postfix_str(f\"Run {idx} for training budget: {budget}\")\ndul_utility = DataUtilityLearning(\nu=utility, training_budget=budget, model=MLPRegressor(**mlp_kwargs)\n)\nstart_time = time.monotonic()\n# DUL will kick in after training_budget calls to utility\nresult = compute_shapley_values(\nu=dul_utility,\nmode=ShapleyMode.PermutationMontecarlo,\ndone=MaxUpdates(300),\nn_jobs=-1,\n)\ncomputation_time = time.monotonic() - start_time\nif budget in computation_times:\ncomputation_times[budget].append(computation_time)\nelse:\ncomputation_times[budget] = [computation_time]\ndul_df = result.to_dataframe(column=f\"{budget}_{idx}\").drop(\ncolumns=[f\"{budget}_{idx}_stderr\"]\n)\ndf = pd.concat([df, dul_df], axis=1)\ncomputation_times_df = pd.DataFrame(computation_times)\n</code></pre> <pre>\n<code>Doing 10 runs for each of 10 different training budgets.\n</code>\n</pre> <pre>\n<code>  0%|          | 0/100 [00:00&lt;?, ?it/s]</code>\n</pre> <p>Next we compute the \\(l_1\\) error for the different training budgets across all runs and plot mean and standard deviation. We obtain results analogous to Figure 1 of the paper, verifying that the method indeed works for estimating the Data Shapley values (at least in this context).</p> <p>In the plot we also display the mean and standard deviation of the computation time taken for each training budget. </p> <pre><code>errors = np.zeros((len(training_budget_values), n_runs), dtype=float)\naccuracies = np.zeros((len(training_budget_values), n_runs), dtype=float)\ntop_k = 3\nfor i, budget in enumerate(training_budget_values):\nfor j in range(n_runs):\ny_true = df[\"exact\"].values\ny_estimated = df[f\"{budget}_{j}\"].values\nerrors[i, j] = np.linalg.norm(y_true - y_estimated, ord=2)\naccuracies[i, j] = top_k_value_accuracy(y_true, y_estimated, k=top_k)\nerror_from_mean = np.linalg.norm(df[\"exact\"].values - df[\"exact\"].values.mean(), ord=2)\n</code></pre> <p>Let us next look at how well the ranking of values resulting from using the surrogate \\(\\tilde{u}\\) matches the ranking by the exact values. For this we fix \\(k=3\\) and consider the \\(k\\) samples with the highest value according to \\(\\tilde{u}\\) and \\(u\\):</p> <p>Finally, for each sample, we look at the distance of the estimates to the exact value across runs. Boxes are centered at the 50th percentile with wiskers at the 25th and 75th. We plot relative distances, as a percentage. We observe a general tendency to underestimate the value:</p> <pre><code>highest_value_index = df.index[df[\"exact\"].argmax()]\ny_train_corrupted = dataset.y_train.copy()\ny_train_corrupted[highest_value_index] = (\ny_train_corrupted[highest_value_index] + 1\n) % 3\ncorrupted_dataset = Dataset(\nx_train=dataset.x_train,\ny_train=y_train_corrupted,\nx_test=dataset.x_test,\ny_test=dataset.y_test,\n)\n</code></pre> <p>We retrain the model on the new dataset and verify that the accuracy decreases:</p> <pre><code>model = LinearSVC()\nmodel.fit(dataset.x_train, y_train_corrupted)\nprint(f\"Mean accuracy: {100 * model.score(dataset.x_test, dataset.y_test):0.2f}%\")\n</code></pre> <pre>\n<code>Mean accuracy: 82.96%\n</code>\n</pre> <p>Finally, we recompute the values of all samples using the exact method and the best training budget previously obtained and then plot the resulting scores.</p> <pre><code>best_training_budget = training_budget_values[errors.mean(axis=1).argmin()]\nutility = Utility(\nmodel=LinearSVC(),\ndata=corrupted_dataset,\n)\nresult = compute_shapley_values(\nu=utility,\nmode=ShapleyMode.CombinatorialExact,\nn_jobs=-1,\nprogress=False,\n)\ndf_corrupted = result.to_dataframe(column=\"exact\").drop(columns=[\"exact_stderr\"])\ndul_utility = DataUtilityLearning(\nu=utility, training_budget=best_training_budget, model=MLPRegressor(**mlp_kwargs)\n)\nresult = compute_shapley_values(\nu=dul_utility,\nmode=ShapleyMode.PermutationMontecarlo,\ndone=MaxUpdates(300),\nn_jobs=-1,\n)\ndul_df = result.to_dataframe(column=\"estimated\").drop(columns=[\"estimated_stderr\"])\ndf_corrupted = pd.concat([df_corrupted, dul_df], axis=1)\n</code></pre> <p>We can see in the figure that both methods assign the lowest value to the sample with the corrupted label.</p>  As mentioned above, despite the previous results, this work is preliminary and the usefulness of Data Utility Learning remains to be tested in practice."},{"location":"examples/shapley_utility_learning/#data-utility-learning","title":"Data Utility Learning","text":""},{"location":"examples/shapley_utility_learning/#setup","title":"Setup","text":"<p>We begin by importing the main libraries and setting some defaults.</p>   If you are reading this in the documentation, some boilerplate (including most plotting code) has been omitted for convenience."},{"location":"examples/shapley_utility_learning/#dataset","title":"Dataset","text":"<p>Following the paper, we take 15 samples (10%) from the Iris dataset and compute their Data Shapley values by using all the remaining samples as test set for computing the utility, which in this case is accuracy.</p>"},{"location":"examples/shapley_utility_learning/#data-shapley","title":"Data Shapley","text":"<p>We start by defining the utility using the model and computing the exact Data Shapley values by definition \\(\\ref{eq:shapley-def}\\).</p>"},{"location":"examples/shapley_utility_learning/#evaluation-on-anomalous-data","title":"Evaluation on anomalous data","text":"<p>One interesting way to assess the Data Utility Learning approach is to corrupt some data and monitor how the value changes. To do this, we will take the sample with the highest score and change its label.</p>"},{"location":"getting-started/first-steps/","title":"Getting started","text":"<p>Warning</p> <p>Make sure you have read Installing pyDVL before using the library.  In particular read about how caching and parallelization work, since they might require additional setup.</p>"},{"location":"getting-started/first-steps/#main-concepts","title":"Main concepts","text":"<p>pyDVL aims to be a repository of production-ready, reference implementations of algorithms for data valuation and influence functions. Even though we only briefly introduce key concepts in the documentation, the following sections  should be enough to get you started.</p> <ul> <li>Basics of data valuation for key objects and usage patterns for Shapley value   computation and related methods.</li> <li>Computing Influence Values for instructions on how to compute influence functions.</li> </ul>"},{"location":"getting-started/first-steps/#running-the-examples","title":"Running the examples","text":"<p>If you are somewhat familiar with the concepts of data valuation, you can start by browsing our worked-out examples illustrating pyDVL's capabilities either:</p> <ul> <li>In the examples under Basics of data valuation and Computing Influence Values.</li> <li>Using binder notebooks, deployed from each   example's page.</li> <li>Locally, by starting a jupyter server at the root of the project. You will   have to install jupyter first manually since it's not a dependency of the   library.</li> </ul>"},{"location":"getting-started/first-steps/#advanced-usage","title":"Advanced usage","text":"<p>Besides the do's and don'ts of data valuation itself, which are the subject of the examples and the documentation of each method, there are two main things to keep in mind when using pyDVL.</p>"},{"location":"getting-started/first-steps/#caching","title":"Caching","text":"<p>pyDVL uses memcached to cache the computation of the utility function and speed up some computations (see the installation guide).</p> <p>Caching of the utility function is disabled by default. When it is enabled it takes into account the data indices passed as argument and the utility function wrapped into the Utility object. This means that care must be taken when reusing the same utility function with different data, see the documentation for the caching module for more information.</p> <p>In general, caching won't play a major role in the computation of Shapley values because the probability of sampling the same subset twice, and hence needing the same utility function computation, is very low. However, it can be very useful when comparing methods that use the same utility function, or when running multiple experiments with the same data.</p> <p>When is the cache really necessary?</p> <p>Crucially, semi-value computations with the PermutationSampler require caching to be enabled, or they will take twice as long as the direct implementation in compute_shapley_values.</p>"},{"location":"getting-started/first-steps/#parallelization","title":"Parallelization","text":"<p>pyDVL supports joblib for local parallelization (within one machine) and ray for distributed parallelization (across multiple machines).</p> <p>The former works out of the box but for the latter you will need to provide a running cluster (or run ray in local mode).</p> <p>As of v0.7.0 pyDVL does not allow requesting resources per task sent to the cluster, so you will need to make sure that each worker has enough resources to handle the tasks it receives. A data valuation task using game-theoretic methods will typically make a copy of the whole model and dataset to each worker, even if the re-training only happens on a subset of the data. This means that you should make sure that each worker has enough memory to handle the whole dataset.</p>"},{"location":"getting-started/installation/","title":"Installing pyDVL","text":"<p>To install the latest release use:</p> <pre><code>pip install pyDVL\n</code></pre> <p>To use all features of influence functions use instead:</p> <pre><code>pip install pyDVL[influence]\n</code></pre> <p>This includes a dependency on PyTorch (Version 2.0 and above) and thus is left out by default.</p> <p>In case that you have a supported version of CUDA installed (v11.2 to 11.8 as of this writing), you can enable eigenvalue computations for low-rank approximations with CuPy on the GPU by using:</p> <pre><code>pip install pyDVL[cupy]\n</code></pre> <p>If you use a different version of CUDA, please install CuPy manually.</p> <p>In order to check the installation you can use:</p> <pre><code>python -c \"import pydvl; print(pydvl.__version__)\"\n</code></pre> <p>You can also install the latest development version from TestPyPI:</p> <pre><code>pip install pyDVL --index-url https://test.pypi.org/simple/\n</code></pre>"},{"location":"getting-started/installation/#dependencies","title":"Dependencies","text":"<p>pyDVL requires Python &gt;= 3.8, Memcached for caching and Ray for parallelization in a cluster (locally it uses joblib). Additionally, the Influence functions module requires PyTorch (see Installing pyDVL).</p> <p>ray is used to distribute workloads across nodes in a cluster (it can be used locally as well, but for this we recommend joblib instead). Please follow the instructions in their documentation to set up the cluster. Once you have a running cluster, you can use it by passing the address of the head node to parallel methods via ParallelConfig.</p>"},{"location":"getting-started/installation/#setting-up-the-cache","title":"Setting up the cache","text":"<p>memcached is an in-memory key-value store accessible over the network. pyDVL uses it to cache the computation of the utility function and speed up some computations (in particular, semi-value computations with the PermutationSampler but other methods may benefit as well).</p> <p>You can either install it as a package or run it inside a docker container (the simplest). For installation instructions, refer to the Getting started section in memcached's wiki. Then you can run it with:</p> <pre><code>memcached -u user\n</code></pre> <p>To run memcached inside a container in daemon mode instead, do:</p> <pre><code>docker container run -d --rm -p 11211:11211 memcached:latest\n</code></pre> <p>Using the cache</p> <p>Continue reading about the cache in the First Steps and the documentation for the caching module.</p>"},{"location":"influence/","title":"The influence function","text":""},{"location":"influence/#the-influence-function","title":"The influence function","text":"<p>Warning</p> <p>The code in the package pydvl.influence is experimental. Package structure and basic API are bound to change before v1.0.0</p> <p>The influence function (IF) is a method to quantify the effect (influence) that each training point has on the parameters of a model, and by extension on any function thereof. In particular, it allows to estimate how much each training sample affects the error on a test point, making the IF useful for understanding and debugging models.</p> <p>Alas, the influence function relies on some assumptions that can make their application difficult. Yet another drawback is that they require the computation of the inverse of the Hessian of the model wrt. its parameters, which is intractable for large models like deep neural networks. Much of the recent research tackles this issue using approximations, like a Neuman series  (Agarwal et al., 2017)<sup>1</sup>, with the most successful solution using a low-rank approximation that iteratively finds increasing eigenspaces of the Hessian  (Schioppa et al., 2021)<sup>2</sup>.</p> <p>pyDVL implements several methods for the efficient computation of the IF for machine learning. In the examples we document some of the difficulties that can arise when using the IF.</p>"},{"location":"influence/#construction","title":"Construction","text":"<p>First introduced in the context of robust statistics in (Hampel, 1974)<sup>3</sup>, the IF was popularized in the context of machine learning in  (Koh and Liang, 2017)<sup>4</sup>.</p> <p>Following their formulation, consider an input space \\(\\mathcal{X}\\) (e.g. images) and an output space \\(\\mathcal{Y}\\) (e.g. labels). Let's take \\(z_i = (x_i, y_i)\\), for \\(i \\in  \\{1,...,n\\}\\) to be the \\(i\\)-th training point, and \\(\\theta\\) to be the (potentially highly) multi-dimensional parameters of a model (e.g. \\(\\theta\\) is a big array with all of a neural network's parameters, including biases and/or dropout rates). We will denote with \\(L(z, \\theta)\\) the loss of the model for point \\(z\\) when the parameters are \\(\\theta.\\)</p> <p>To train a model, we typically minimize the loss over all \\(z_i\\), i.e. the optimal parameters are</p> \\[\\hat{\\theta} = \\arg \\min_\\theta \\sum_{i=1}^n L(z_i, \\theta).\\] <p>In practice, lack of convexity means that one doesn't really obtain the minimizer of the loss, and the training is stopped when the validation loss stops decreasing.</p> <p>For notational convenience, let's define</p> \\[\\hat{\\theta}_{-z} = \\arg \\min_\\theta \\sum_{z_i \\ne z} L(z_i, \\theta), \\] <p>i.e. \\(\\hat{\\theta}_{-z}\\) are the model parameters that minimize the total loss when \\(z\\) is not in the training dataset.</p> <p>In order to compute the impact of each training point on the model, we would need to calculate \\(\\hat{\\theta}_{-z}\\) for each \\(z\\) in the training dataset, thus re-training the model at least ~\\(n\\) times (more if model training is stochastic). This is computationally very expensive, especially for big neural networks. To circumvent this problem, we can just calculate a first order approximation of \\(\\hat{\\theta}\\). This can be done through single backpropagation and without re-training the full model.</p> <p>pyDVL supports two ways of computing the empirical influence function, namely up-weighting of samples and perturbation influences. The choice is done by the parameter <code>influence_type</code> in the main entry point  compute_influences.</p>"},{"location":"influence/#approximating-the-influence-of-a-point","title":"Approximating the influence of a point","text":"<p>Let's define</p> \\[\\hat{\\theta}_{\\epsilon, z} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_i, \\theta) + \\epsilon L(z, \\theta), \\] <p>which is the optimal \\(\\hat{\\theta}\\) when we up-weight \\(z\\) by an amount \\(\\epsilon \\gt 0\\).</p> <p>From a classical result (a simple derivation is available in Appendix A of  (Koh and Liang, 2017)<sup>4</sup>), we know that:</p> \\[\\frac{d \\ \\hat{\\theta}_{\\epsilon, z}}{d \\epsilon} \\Big|_{\\epsilon=0} = -H_{\\hat{\\theta}}^{-1} \\nabla_\\theta L(z, \\hat{\\theta}), \\] <p>where \\(H_{\\hat{\\theta}} = \\frac{1}{n} \\sum_{i=1}^n \\nabla_\\theta^2 L(z_i, \\hat{\\theta})\\) is the Hessian of \\(L\\). These quantities are also knows as influence factors.</p> <p>Importantly, notice that this expression is only valid when \\(\\hat{\\theta}\\) is a minimum of \\(L\\), or otherwise \\(H_{\\hat{\\theta}}\\) cannot be inverted! At the same time, in machine learning full convergence is rarely achieved, so direct Hessian inversion is not possible. Approximations need to be developed that circumvent the problem of inverting the Hessian of the model in all those (frequent) cases where it is not positive definite.</p> <p>The influence of training point \\(z\\) on test point \\(z_{\\text{test}}\\) is defined as:</p> \\[\\mathcal{I}(z, z_{\\text{test}}) =  L(z_{\\text{test}}, \\hat{\\theta}_{-z}) - L(z_{\\text{test}}, \\hat{\\theta}). \\] <p>Notice that \\(\\mathcal{I}\\) is higher for points \\(z\\) which positively impact the model score, since the loss is higher when they are excluded from training. In practice, one needs to rely on the following infinitesimal approximation:</p> \\[\\mathcal{I}_{up}(z, z_{\\text{test}}) = - \\frac{d L(z_{\\text{test}}, \\hat{\\theta}_{\\epsilon, z})}{d \\epsilon} \\Big|_{\\epsilon=0} \\] <p>Using the chain rule and the results calculated above, we get:</p> \\[\\mathcal{I}_{up}(z, z_{\\text{test}}) = - \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ \\frac{d \\hat{\\theta}_{\\epsilon, z}}{d \\epsilon} \\Big|_{\\epsilon=0} = \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ H_{\\hat{\\theta}}^{-1} \\ \\nabla_\\theta L(z, \\hat{\\theta}) \\] <p>All the resulting factors are gradients of the loss wrt. the model parameters \\(\\hat{\\theta}\\). This can be easily computed through one or more backpropagation passes.</p>"},{"location":"influence/#perturbation-definition-of-the-influence-score","title":"Perturbation definition of the influence score","text":"<p>How would the loss of the model change if, instead of up-weighting an individual point \\(z\\), we were to up-weight only a single feature of that point? Given \\(z = (x, y)\\), we can define \\(z_{\\delta} = (x+\\delta, y)\\), where \\(\\delta\\) is a vector of zeros except for a 1 in the position of the feature we want to up-weight. In order to approximate the effect of modifying a single feature of a single point on the model score we can define</p> \\[\\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z} = \\arg \\min_\\theta \\frac{1}{n}\\sum_{i=1}^n L(z_{i}, \\theta) + \\epsilon L(z_{\\delta}, \\theta) - \\epsilon L(z, \\theta), \\] <p>Similarly to what was done above, we up-weight point \\(z_{\\delta}\\), but then we also remove the up-weighting for all the features that are not modified by \\(\\delta\\). From the calculations in the previous section, it is then easy to see that</p> \\[\\frac{d \\ \\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z}}{d \\epsilon} \\Big|_{\\epsilon=0} = -H_{\\hat{\\theta}}^{-1} \\nabla_\\theta \\Big( L(z_{\\delta}, \\hat{\\theta}) - L(z, \\hat{\\theta}) \\Big) \\] <p>and if the feature space is continuous and as \\(\\delta \\to 0\\) we can write</p> \\[\\frac{d \\ \\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z}}{d \\epsilon} \\Big|_{\\epsilon=0} = -H_{\\hat{\\theta}}^{-1} \\ \\nabla_x \\nabla_\\theta L(z, \\hat{\\theta}) \\delta + \\mathcal{o}(\\delta) \\] <p>The influence of each feature of \\(z\\) on the loss of the model can therefore be estimated through the following quantity:</p> \\[\\mathcal{I}_{pert}(z, z_{\\text{test}}) = - \\lim_{\\delta \\to 0} \\ \\frac{1}{\\delta} \\frac{d L(z_{\\text{test}}, \\hat{\\theta}_{\\epsilon, \\ z_{\\delta}, \\ -z})}{d \\epsilon} \\Big|_{\\epsilon=0} \\] <p>which, using the chain rule and the results calculated above, is equal to</p> \\[\\mathcal{I}_{pert}(z, z_{\\text{test}}) = - \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ \\frac{d \\hat{\\theta}_{\\epsilon, z_{\\delta} ,-z}}{d \\epsilon} \\Big|_{\\epsilon=0} = \\nabla_\\theta L(z_{\\text{test}}, \\hat{\\theta})^\\top \\ H_{\\hat{\\theta}}^{-1} \\ \\nabla_x \\nabla_\\theta L(z, \\hat{\\theta}) \\] <p>The perturbation definition of the influence score is not straightforward to understand, but it has a simple interpretation: it tells how much the loss of the model changes when a certain feature of point z is up-weighted. A positive perturbation influence score indicates that the feature might have a positive effect on the accuracy of the model.</p> <p>It is worth noting that the perturbation influence score is a very rough estimate of the impact of a point on the models loss and it is subject to large approximation errors. It can nonetheless be used to build training-set attacks, as done in (Koh and Liang, 2017)<sup>4</sup>.</p>"},{"location":"influence/#computation","title":"Computation","text":"<p>The main entry point of the library for influence calculation is compute_influences. Given a pre-trained pytorch model with a loss, first an instance of TorchTwiceDifferentiable needs to be created:</p> <pre><code>from pydvl.influence import TorchTwiceDifferentiable\nwrapped_model = TorchTwiceDifferentiable(model, loss, device)\n</code></pre> <p>The device specifies where influence calculation will be run. </p> <p>Given training and test data loaders, the influence of each training point on each test point can be calculated via:</p> <pre><code>from pydvl.influence import compute_influences\nfrom torch.utils.data import DataLoader\ntraining_data_loader = DataLoader(...)\ntest_data_loader = DataLoader(...)\ncompute_influences(\nwrapped_model,\ntraining_data_loader,\ntest_data_loader,\n)\n</code></pre> <p>The result is a tensor with one row per test point and one column per training point. Thus, each entry \\((i, j)\\) represents the influence of training point \\(j\\) on test point \\(i\\). A large positive influence indicates that training point \\(j\\) tends to improve the performance of the model on test point \\(i\\), and vice versa, a large negative influence indicates that training point \\(j\\) tends to worsen the performance of the model on test point \\(i\\).</p>"},{"location":"influence/#perturbation-influences","title":"Perturbation influences","text":"<p>The method of empirical influence computation can be selected in compute_influences with the parameter <code>influence_type</code>:</p> <pre><code>from pydvl.influence import compute_influences\ncompute_influences(\nwrapped_model,\ntraining_data_loader,\ntest_data_loader,\ninfluence_type=\"perturbation\",\n)\n</code></pre> <p>The result is a tensor with at least three dimensions. The first two dimensions are the same as in the case of <code>influence_type=up</code> case, i.e. one row per test point and one column per training point. The remaining dimensions are the same as the number of input features in the data. Therefore, each entry in the tensor represents the influence of each feature of each training point on each test point.</p>"},{"location":"influence/#approximate-matrix-inversion","title":"Approximate matrix inversion","text":"<p>In almost every practical application it is not possible to construct, even less invert the complete Hessian in memory. pyDVL offers several approximate algorithms to invert it by setting the parameter <code>inversion_method</code> of compute_influences.</p> <pre><code>from pydvl.influence import compute_influences\ncompute_influences(\nwrapped_model,\ntraining_data_loader,\ntest_data_loader,\ninversion_method=\"cg\"\n)\n</code></pre> <p>Each inversion method has its own set of parameters that can be tuned to improve the final result. These parameters can be passed directly to compute_influences as keyword arguments. For example, the following code sets the maximum number of iterations for conjugate gradient to \\(100\\) and the minimum relative error to \\(0.01\\):</p> <pre><code>from pydvl.influence import compute_influences\ncompute_influences(\nwrapped_model,\ntraining_data_loader,\ntest_data_loader,\ninversion_method=\"cg\",\nhessian_regularization=1e-4,\nmaxiter=100,\nrtol=0.01\n)\n</code></pre>"},{"location":"influence/#hessian-regularization","title":"Hessian regularization","text":"<p>Additionally, and as discussed in the introduction, in machine learning training rarely converges to a global minimum of the loss. Despite good apparent convergence, \\(\\hat{\\theta}\\) might be located in a region with flat curvature or close to a saddle point. In particular, the Hessian might have vanishing eigenvalues making its direct inversion impossible. Certain methods, such as the Arnoldi method are robust against these problems, but most are not.</p> <p>To circumvent this problem, many approximate methods can be implemented. The simplest adds a small hessian perturbation term, i.e. \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\), with \\(\\mathbb{I}\\) being the identity matrix. This standard trick ensures that the eigenvalues of \\(H_{\\hat{\\theta}}\\) are bounded away from zero and therefore the matrix is invertible. In order for this regularization not to corrupt the outcome too much, the parameter \\(\\lambda\\) should be as small as possible while still allowing a reliable inversion of \\(H_{\\hat{\\theta}} + \\lambda \\mathbb{I}\\).</p> <pre><code>from pydvl.influence import compute_influences\ncompute_influences(\nwrapped_model,\ntraining_data_loader,\ntest_data_loader,\ninversion_method=\"cg\",\nhessian_regularization=1e-4\n)\n</code></pre>"},{"location":"influence/#influence-factors","title":"Influence factors","text":"<p>The compute_influences method offers a fast way to obtain the influence scores given a model and a dataset. Nevertheless, it is often more convenient to inspect and save some of the intermediate results of influence calculation for later use.</p> <p>The influence factors(refer to the previous section for a definition) are typically the most computationally demanding part of influence calculation. They can be obtained via the compute_influence_factors function, saved, and later used for influence calculation on different subsets of the training dataset.</p> <pre><code>from pydvl.influence import compute_influence_factors\ninfluence_factors = compute_influence_factors(\nwrapped_model,\ntraining_data_loader,\ntest_data_loader,\ninversion_method=\"cg\"\n)\n</code></pre> <p>The result is an object of type  InverseHvpResult, which holds the calculated influence factors (<code>influence_factors.x</code>) and a dictionary with the info on the inversion process (<code>influence_factors.info</code>).</p>"},{"location":"influence/#methods-for-inverse-hvp-calculation","title":"Methods for inverse HVP calculation","text":"<p>In order to calculate influence values, pydvl implements several methods for the calculation of the inverse Hessian vector product (iHVP). More precisely, given a model, training data and a tensor \\(b\\), the function solve_hvp will find \\(x\\) such that \\(H x = b\\), with \\(H\\) is the hessian of model.</p> <p>Many different inversion methods can be selected via the parameter <code>inversion_method</code> of compute_influences.</p> <p>The following subsections will offer more detailed explanations for each method.</p>"},{"location":"influence/#direct-inversion","title":"Direct inversion","text":"<p>With <code>inversion_method = \"direct\"</code> pyDVL will calculate the inverse Hessian using the direct matrix inversion. This means that the Hessian will first be explicitly created and then inverted. This method is the most accurate, but also the most computationally demanding. It is therefore not recommended for large datasets or models with many parameters.</p> <pre><code>import torch\nfrom pydvl.influence.inversion import solve_hvp\nb = torch.Tensor(...)\nsolve_hvp(\n\"direct\",\nwrapped_model,\ntraining_data_loader,\nb,\n)\n</code></pre> <p>The result, an object of type  InverseHvpResult, which holds two objects: <code>influence_factors.x</code> and <code>influence_factors.info</code>. The first one is the inverse Hessian vector product, while the second one is a dictionary with the info on the inversion process. For this method, the info consists of the Hessian matrix itself.</p>"},{"location":"influence/#conjugate-gradient","title":"Conjugate Gradient","text":"<p>This classical procedure for solving linear systems of equations is an iterative method that does not require the explicit inversion of the Hessian. Instead, it only requires the calculation of Hessian-vector products, making it a good choice for large datasets or models with many parameters. It is nevertheless much slower to converge than the direct inversion method and not as accurate. More info on the theory of conjugate gradient can be found on Wikipedia.</p> <p>In pyDVL, you can select conjugate gradient with <code>inversion_method = \"cg\"</code>, like this:</p> <pre><code>from pydvl.influence.inversion import solve_hvp\nsolve_hvp(\n\"cg\",\nwrapped_model,\ntraining_data_loader,\nb,\nx0=None,\nrtol=1e-7,\natol=1e-7,\nmaxiter=None,\n)\n</code></pre> <p>The additional optional parameters <code>x0</code>, <code>rtol</code>, <code>atol</code>, and <code>maxiter</code> are passed to the solve_batch_cg function, and are respecively the initial guess for the solution, the relative tolerance, the absolute tolerance, and the maximum number of iterations.</p> <p>The resulting InverseHvpResult holds the solution of the iHVP, <code>influence_factors.x</code>, and some info on the inversion process <code>influence_factors.info</code>. More specifically, for each batch this will contain the number of iterations, a boolean indicating if the inversion converged, and the residual of the inversion.</p>"},{"location":"influence/#linear-time-stochastic-second-order-approximation-lissa","title":"Linear time Stochastic Second-Order Approximation (LiSSA)","text":"<p>The LiSSA method is a stochastic approximation of the inverse Hessian vector product. Compared to conjugate gradient it is faster but less accurate and typically suffers from instability.</p> <p>In order to find the solution of the HVP, LiSSA iteratively approximates the inverse of the Hessian matrix with the following update:</p> \\[H^{-1}_{j+1} b = b + (I - d) \\ H - \\frac{H^{-1}_j b}{s},\\] <p>where \\(d\\) and \\(s\\) are a dampening and a scaling factor, which are essential for the convergence of the method and they need to be chosen carefully, and I  is the identity matrix. More info on the theory of LiSSA can be found in the  original paper (Agarwal et al., 2017)<sup>1</sup>.</p> <p>In pyDVL, you can select LiSSA with <code>inversion_method = \"lissa\"</code>, like this:</p> <pre><code>from pydvl.influence.inversion import solve_hvp\nsolve_hvp(\n\"lissa\",\nwrapped_model,\ntraining_data_loader,\nb,\nmaxiter=1000,\ndampen=0.0,\nscale=10.0,\nh0=None,\nrtol=1e-4,\n)\n</code></pre> <p>with the additional optional parameters <code>maxiter</code>, <code>dampen</code>, <code>scale</code>, <code>h0</code>, and <code>rtol</code>, which are passed to the solve_lissa function, being the maximum number of iterations, the dampening factor, the scaling factor, the initial guess for the solution and the relative tolerance, respectively.</p> <p>The resulting InverseHvpResult holds the solution of the iHVP, <code>influence_factors.x</code>, and, within <code>influence_factors.info</code>, the maximum percentage error and the mean percentage error of the approximation.</p>"},{"location":"influence/#arnoldi-solver","title":"Arnoldi solver","text":"<p>The Arnoldi method is a Krylov subspace method for approximating dominating eigenvalues and eigenvectors. Under a low rank assumption on the Hessian at a minimizer (which is typically observed for deep neural networks), this approximation captures the essential action of the Hessian. More concretely, for \\(Hx=b\\) the solution is approximated by</p> \\[x \\approx V D^{-1} V^T b\\] <p>where \\(D\\) is a diagonal matrix with the top (in absolute value) eigenvalues of the Hessian and \\(V\\) contains the corresponding eigenvectors. See also  (Schioppa et al., 2021)<sup>2</sup>.</p> <p>In pyDVL, you can use Arnoldi with <code>inversion_method = \"arnoldi\"</code>, as follows:</p> <pre><code>from pydvl.influence.inversion import solve_hvp\nsolve_hvp(\n\"arnoldi\",\nwrapped_model,\ntraining_data_loader,\nb,\nhessian_perturbation=0.0,\nrank_estimate=10,\ntol=1e-6,\neigen_computation_on_gpu=False \n)\n</code></pre> <p>For the parameters, check solve_arnoldi. The resulting InverseHvpResult holds the solution of the iHVP, <code>influence_factors.x</code>, and, within <code>influence_factors.info</code>, the computed eigenvalues and eigenvectors.</p> <ol> <li> <p>Agarwal, N., Bullins, B., Hazan, E., 2017. Second-Order Stochastic Optimization for Machine Learning in Linear Time. JMLR 18, 1\u201340.\u00a0\u21a9\u21a9</p> </li> <li> <p>Schioppa, A., Zablotskaia, P., Vilar, D., Sokolov, A., 2021. Scaling Up Influence Functions. Presented at the AAAI-22, arXiv. https://doi.org/10.48550/arXiv.2112.03052 \u21a9\u21a9</p> </li> <li> <p>Hampel, F.R., 1974. The Influence Curve and Its Role in Robust Estimation. J. Am. Stat. Assoc. 69, 383\u2013393. https://doi.org/10.2307/2285666 \u21a9</p> </li> <li> <p>Koh, P.W., Liang, P., 2017. Understanding Black-box Predictions via Influence Functions, in: Proceedings of the 34th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 1885\u20131894.\u00a0\u21a9\u21a9\u21a9</p> </li> </ol>"},{"location":"value/","title":"Data valuation","text":"<p>Note</p> <p>If you want to jump right into the steps to compute values, skip ahead to Computing data values.</p> <p>Data valuation is the task of assigning a number to each element of a training set which reflects its contribution to the final performance of some model trained on it. Some methods attempt to be model-agnostic, but in most cases the model is an integral part of the method. In these cases, this number is not an intrinsic property of the element of interest, but typically a function of three factors:</p> <ol> <li> <p>The dataset \\(D\\), or more generally, the distribution it was sampled from: In    some cases one only cares about values wrt. a given data set, in others    value would ideally be the (expected) contribution of a data point to any    random set \\(D\\) sampled from the same distribution. pyDVL implements methods    of the first kind.</p> </li> <li> <p>The algorithm \\(\\mathcal{A}\\) mapping the data \\(D\\) to some estimator \\(f\\) in a    model class \\(\\mathcal{F}\\). E.g. MSE minimization to find the parameters of a    linear model.</p> </li> <li> <p>The performance metric of interest \\(u\\) for the problem. When value depends on    a model, it must be measured in some way which uses it. E.g. the \\(R^2\\) score    or the negative MSE over a test set. This metric will be computed over a    held-out valuation set.</p> </li> </ol> <p>pyDVL collects algorithms for the computation of data values in this sense, mostly those derived from cooperative game theory. The methods can be found in the package pydvl.value , with support from modules pydvl.utils.dataset and pydvl.utils.utility, as detailed below.</p> <p>Warning</p> <p>Be sure to read the section on the difficulties using data values.</p> <p>There are three main families of methods for data valuation: game-theoretic,  influence-based and intrinsic. As of v0.7.0 pyDVL supports the first two. Here, we focus on game-theoretic concepts and refer to the main documentation on the influence funtion for the second.</p>"},{"location":"value/#game-theoretical-methods","title":"Game theoretical methods","text":"<p>The main contenders in game-theoretic approaches are Shapley values (Ghorbani and Zou, 2019)<sup>1</sup>, (Kwon et al., 2021)<sup>2</sup>,  (Schoch et al., 2022)<sup>3</sup>, their generalization to so-called semi-values by (Kwon and Zou, 2022)<sup>4</sup> and (Wang and Jia, 2022)<sup>5</sup>, and the Core (Yan and Procaccia, 2021)<sup>6</sup>. All of these are implemented in pyDVL.</p> <p>In these methods, data points are considered players in a cooperative game  whose outcome is the performance of the model when trained on subsets  (coalitions) of the data, measured on a held-out valuation set. This  outcome, or utility, must typically be computed for every subset of  the training set, so that an exact computation is \\(\\mathcal{O} (2^n)\\) in the  number of samples \\(n\\), with each iteration requiring a full re-fitting of the  model using a coalition as training set. Consequently, most methods involve  Monte Carlo approximations, and sometimes approximate utilities which are  faster to compute, e.g. proxy models (Wang et al., 2022)<sup>7</sup> or constant-cost approximations like Neural Tangent Kernels (Wu et al., 2022)<sup>8</sup>.</p> <p>The reasoning behind using game theory is that, in order to be useful, an assignment of value, dubbed valuation function, is usually required to fulfil certain requirements of consistency and \"fairness\". For instance, in some applications value should not depend on the order in which data are considered, or it should be equal for samples that contribute equally to any subset of the data (of equal size). When considering aggregated value for (sub-)sets of data there are additional desiderata, like having a value function that does not increase with repeated samples. Game-theoretic methods are all rooted in axioms that by construction ensure different desiderata, but despite their practical usefulness, none of them are either necessary or sufficient for all applications. For instance, SV methods try to equitably distribute all value among all samples, failing to identify repeated ones as unnecessary, with e.g. a zero value.</p>"},{"location":"value/#applications-of-data-valuation","title":"Applications of data valuation","text":"<p>Many applications are touted for data valuation, but the results can be inconsistent. Values have a strong dependency on the training procedure and the performance metric used. For instance, accuracy is a poor metric for imbalanced sets and this has a stark effect on data values. Some models exhibit great variance in some regimes and this again has a detrimental effect on values.</p> <p>Nevertheless, some of the most promising applications are:</p> <ul> <li>Cleaning of corrupted data.</li> <li>Pruning unnecessary or irrelevant data.</li> <li>Repairing mislabeled data.</li> <li>Guiding data acquisition and annotation (active learning).</li> <li>Anomaly detection and model debugging and interpretation.</li> </ul> <p>Additionally, one of the motivating applications for the whole field is that of data markets: a marketplace where data owners can sell their data to interested parties. In this setting, data valuation can be key component to determine the price of data. Algorithm-agnostic methods like LAVA (Just et al., 2023)<sup>9</sup> are particularly well suited for this, as they use the Wasserstein distance between a vendor's data and the buyer's to determine the value of the former. </p> <p>However, this is a complex problem which can face practical banal problems like the fact that data owners may not wish to disclose their data for valuation.</p>"},{"location":"value/#computing-data-values","title":"Computing data values","text":"<p>Using pyDVL to compute data values is a simple process that can be broken down into three steps:</p> <ol> <li>Creating a Dataset object from your data.</li> <li>Creating a Utility which ties your model to    the dataset and a scoring function.</li> <li>Computing values with a method of your choice, e.g. via    compute_shapley_values.</li> </ol>"},{"location":"value/#creating-a-dataset","title":"Creating a Dataset","text":"<p>The first item in the tuple \\((D, \\mathcal{A}, u)\\) characterising data value is the dataset. The class Dataset is a simple convenience wrapper for the train and test splits that is used throughout pyDVL. The test set will be used to evaluate a scoring function for the model.</p> <p>It can be used as follows:</p> <pre><code>import numpy as np\nfrom pydvl.utils import Dataset\nfrom sklearn.model_selection import train_test_split\nX, y = np.arange(100).reshape((50, 2)), np.arange(50)\nX_train, X_test, y_train, y_test = train_test_split(\nX, y, test_size=0.5, random_state=16\n)\ndataset = Dataset(X_train, X_test, y_train, y_test)\n</code></pre> <p>It is also possible to construct Datasets from sklearn toy datasets for illustrative purposes using from_sklearn.</p>"},{"location":"value/#grouping-data","title":"Grouping data","text":"<p>Be it because data valuation methods are computationally very expensive, or because we are interested in the groups themselves, it can be often useful or necessary to group samples to valuate them together. GroupedDataset provides an alternative to Dataset with the same interface which allows this.</p> <p>You can see an example in action in the Spotify notebook, but here's a simple example grouping a pre-existing <code>Dataset</code>. First we construct an array mapping each index in the dataset to a group, then use from_dataset:</p> <pre><code>import numpy as np\nfrom pydvl.utils import GroupedDataset\n# Randomly assign elements to any one of num_groups:\ndata_groups = np.random.randint(0, num_groups, len(dataset))\ngrouped_dataset = GroupedDataset.from_dataset(dataset, data_groups)\n</code></pre>"},{"location":"value/#creating-a-utility","title":"Creating a Utility","text":"<p>In pyDVL we have slightly overloaded the name \"utility\" and use it to refer to an object that keeps track of all three items in \\((D, \\mathcal{A}, u)\\). This will be an instance of Utility which, as mentioned, is a convenient wrapper for the dataset, model and scoring function used for valuation methods.</p> <p>Here's a minimal example:</p> <pre><code>import sklearn as sk\nfrom pydvl.utils import Dataset, Utility\ndataset = Dataset.from_sklearn(sk.datasets.load_iris())\nmodel = sk.svm.SVC()\nutility = Utility(model, dataset)\n</code></pre> <p>The object <code>utility</code> is a callable that data valuation methods will execute with different subsets of training data. Each call will retrain the model on a subset and evaluate it on the test data using a scoring function. By default, Utility will use <code>model.score()</code>, but it is possible to use any scoring function (greater values must be better). In particular, the constructor accepts the same types as argument as sklearn.model_selection.cross_validate: a string, a scorer callable or None for the default.</p> <pre><code>utility = Utility(model, dataset, \"explained_variance\")\n</code></pre> <p><code>Utility</code> will wrap the <code>fit()</code> method of the model to cache its results. This greatly reduces computation times of Monte Carlo methods. Because of how caching is implemented, it is important not to reuse <code>Utility</code> objects for different datasets. You can read more about setting up the cache in the installation guide and the documentation of the caching module.</p>"},{"location":"value/#using-custom-scorers","title":"Using custom scorers","text":"<p>The <code>scoring</code> argument of Utility can be used to specify a custom Scorer object. This is a simple wrapper for a callable that takes a model, and test data and returns a score.</p> <p>More importantly, the object provides information about the range of the score, which is used by some methods by estimate the number of samples necessary, and about what default value to use when the model fails to train.</p> <p>Note</p> <p>The most important property of a <code>Scorer</code> is its default value. Because many models will fail to fit on small subsets of the data, it is important to provide a sensible default value for the score.</p> <p>It is possible to skip the construction of the Scorer when constructing the <code>Utility</code> object. The two following calls are equivalent:</p> <pre><code>from pydvl.utils import Utility, Scorer\nutility = Utility(\nmodel, dataset, \"explained_variance\", score_range=(-np.inf, 1), default_score=0.0\n)\nutility = Utility(\nmodel, dataset, Scorer(\"explained_variance\", range=(-np.inf, 1), default=0.0)\n)\n</code></pre>"},{"location":"value/#learning-the-utility","title":"Learning the utility","text":"<p>Because each evaluation of the utility entails a full retrain of the model with a new subset of the training set, it is natural to try to learn this mapping from subsets to scores. This is the idea behind Data Utility Learning (DUL)  (Wang et al., 2022)<sup>7</sup> and in pyDVL it's as simple as wrapping the <code>Utility</code> inside DataUtilityLearning:</p> <pre><code>from pydvl.utils import Utility, DataUtilityLearning, Dataset\nfrom sklearn.linear_model import LinearRegression, LogisticRegression\nfrom sklearn.datasets import load_iris\ndataset = Dataset.from_sklearn(load_iris())\nu = Utility(LogisticRegression(), dataset, enable_cache=False)\ntraining_budget = 3\nwrapped_u = DataUtilityLearning(u, training_budget, LinearRegression())\n# First 3 calls will be computed normally\nfor i in range(training_budget):\n_ = wrapped_u((i,))\n# Subsequent calls will be computed using the fit model for DUL\nwrapped_u((1, 2, 3))\n</code></pre> <p>As you can see, all that is required is a model to learn the utility itself and the fitting and using of the learned model happens behind the scenes.</p> <p>There is a longer example with an investigation of the results achieved by DUL in a dedicated notebook.</p>"},{"location":"value/#leave-one-out-values","title":"Leave-One-Out values","text":"<p>LOO is the simplest approach to valuation. It assigns to each sample its marginal utility as value:</p> \\[v_u(i) = u(D) \u2212 u(D_{-i}).\\] <p>For notational simplicity, we consider the valuation function as defined over the indices of the dataset \\(D\\), and \\(i \\in D\\) is the index of the sample, \\(D_{-i}\\) is the training set without the sample \\(x_i\\), and \\(u\\) is the utility function.</p> <p>For the purposes of data valuation, this is rarely useful beyond serving as a baseline for benchmarking. Although in some benchmarks it can perform astonishingly well on occasion. One particular weakness is that it does not necessarily correlate with an intrinsic value of a sample: since it is a marginal utility, it is affected by diminishing returns. Often, the training set is large enough for a single sample not to have any significant effect on training performance, despite any qualities it may possess. Whether this is indicative of low value or not depends on each one's goals and definitions, but other methods are typically preferable.</p> <pre><code>from pydvl.value.loo import compute_loo\nvalues = compute_loo(utility, n_jobs=-1)\n</code></pre> <p>The return value of all valuation functions is an object of type ValuationResult. This can be iterated over, indexed with integers, slices and Iterables, as well as converted to a pandas.DataFrame.</p>"},{"location":"value/#problems-of-data-values","title":"Problems of data values","text":"<p>There are a number of factors that affect how useful values can be for your project. In particular, regression can be especially tricky, but the particular nature of every (non-trivial) ML problem can have an effect:</p> <ul> <li> <p>Unbounded utility: Choosing a scorer for a classifier is simple: accuracy   or some F-score provides a bounded number with a clear interpretation. However,   in regression problems most scores, like \\(R^2\\), are not bounded because   regressors can be arbitrarily bad. This leads to great variability in the   utility for low sample sizes, and hence unreliable Monte Carlo approximations   to the values. Nevertheless, in practice it is only the ranking of samples   that matters, and this tends to be accurate (wrt. to the true ranking) despite   inaccurate values.</p> Squashing scores <p>pyDVL offers a dedicated function composition for scorer functions which can be used to squash a score. The following is defined in module score: <pre><code>import numpy as np\nfrom pydvl.utils import compose_score\ndef sigmoid(x: float) -&gt; float:\nreturn float(1 / (1 + np.exp(-x)))\nsquashed_r2 = compose_score(\"r2\", sigmoid, \"squashed r2\")\nsquashed_variance = compose_score(\n\"explained_variance\", sigmoid, \"squashed explained variance\"\n)\n</code></pre> These squashed scores can prove useful in regression problems, but they can also introduce issues in the low-value regime.</p> </li> <li> <p>High variance utility: Classical applications of game theoretic value   concepts operate with deterministic utilities, but in ML we use an evaluation   of the model on a validation set as a proxy for the true risk. Even if the   utility is bounded, if it has high variance then values will also have high   variance, as will their Monte Carlo estimates. One workaround in pyDVL is to   configure the caching system to allow multiple evaluations of the utility for   every index set. A moving average is computed and returned once the standard   error is small, see MemcachedConfig.   (Wang and Jia, 2022)<sup>5</sup> prove that by relaxing one of the Shapley axioms   and considering the general class of semi-values, of which Shapley is an   instance, one can prove that a choice of constant weights is the best one can   do in a utility-agnostic setting. This method, dubbed Data Banzhaf, is   available in pyDVL as   compute_banzhaf_semivalues.</p> </li> <li> <p>Data set size: Computing exact Shapley values is NP-hard, and Monte Carlo   approximations can converge slowly. Massive datasets are thus impractical, at   least with game-theoretical methods. A workaround   is to group samples and investigate their value together. You can do this using   GroupedDataset. There is a fully   worked-out example here. Some algorithms   also provide different sampling strategies to reduce the variance, but due to a   no-free-lunch-type theorem, no single strategy can be optimal for all utilities.</p> </li> <li> <p>Model size: Since every evaluation of the utility entails retraining the   whole model on a subset of the data, large models require great amounts of   computation. But also, they will effortlessly interpolate small to medium   datasets, leading to great variance in the evaluation of performance on the   dedicated validation set. One mitigation for this problem is cross-validation,   but this would incur massive computational cost. As of v.0.7.0 there are no   facilities in pyDVL for cross-validating the utility (note that this would   require cross-validating the whole value computation).</p> </li> </ul> <ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning, in: Proceedings of the 36th International Conference on Machine Learning, PMLR. Presented at the International Conference on Machine Learning (ICML 2019), PMLR, pp. 2242\u20132251.\u00a0\u21a9</p> </li> <li> <p>Kwon, Y., Rivas, M.A., Zou, J., 2021. Efficient Computation and Analysis of Distributional Shapley Values, in: Proceedings of the 24th International Conference on Artificial Intelligence and Statistics. Presented at the International Conference on Artificial Intelligence and Statistics, PMLR, pp. 793\u2013801.\u00a0\u21a9</p> </li> <li> <p>Schoch, S., Xu, H., Ji, Y., 2022. CS-Shapley: Class-wise Shapley Values for Data Valuation in Classification, in: Proc. Of the Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). Presented at the Advances in Neural Information Processing Systems (NeurIPS 2022).\u00a0\u21a9</p> </li> <li> <p>Kwon, Y., Zou, J., 2022. Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning, in: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022,. Presented at the AISTATS 2022, PMLR.\u00a0\u21a9</p> </li> <li> <p>Wang, J.T., Jia, R., 2022. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning [WWW Document]. https://doi.org/10.48550/arXiv.2205.15466 \u21a9\u21a9</p> </li> <li> <p>Yan, T., Procaccia, A.D., 2021. If You Like Shapley Then You\u2019ll Love the Core, in: Proceedings of the 35th AAAI Conference on Artificial Intelligence, 2021. Presented at the AAAI Conference on Artificial Intelligence, Association for the Advancement of Artificial Intelligence, pp. 5751\u20135759. https://doi.org/10.1609/aaai.v35i6.16721 \u21a9</p> </li> <li> <p>Wang, T., Yang, Y., Jia, R., 2022. Improving Cooperative Game Theory-based Data Valuation via Data Utility Learning. Presented at the International Conference on Learning Representations (ICLR 2022). Workshop on Socially Responsible Machine Learning, arXiv. https://doi.org/10.48550/arXiv.2107.06336 \u21a9\u21a9</p> </li> <li> <p>Wu, Z., Shu, Y., Low, B.K.H., 2022. DAVINZ: Data Valuation using Deep Neural Networks at Initialization, in: Proceedings of the 39th International Conference on Machine Learning. Presented at the International Conference on Machine Learning, PMLR, pp. 24150\u201324176.\u00a0\u21a9</p> </li> <li> <p>Just, H.A., Kang, F., Wang, T., Zeng, Y., Ko, M., Jin, M., Jia, R., 2023. LAVA: Data Valuation without Pre-Specified Learning Algorithms. Presented at the The Eleventh International Conference on Learning Representations (ICLR 2023).\u00a0\u21a9</p> </li> </ol>"},{"location":"value/notation/","title":"Notation for valuation","text":"<p>The following notation is used throughout the documentation:</p> <p>Let \\(D = \\{x_1, \\ldots, x_n\\}\\) be a training set of \\(n\\) samples.</p> <p>The utility function \\(u:\\mathcal{D} \\rightarrow \\mathbb{R}\\) maps subsets of \\(D\\) to real numbers.</p> <p>The value \\(v\\) of the \\(i\\)-th sample in dataset \\(D\\) wrt. utility \\(u\\) is denoted as \\(v_u(x_i)\\) or simply \\(v(i)\\).</p> <p>For any \\(S \\subseteq D\\), we donote by \\(S_{-i}\\) the set of samples in \\(D\\) excluding \\(x_i\\), and \\(S_{+i}\\) denotes the set \\(S\\) with \\(x_i\\) added.</p> <p>The marginal utility of adding sample \\(x_i\\) to a subset \\(S\\) is denoted as \\(\\delta(i) := u(S_{+i}) - u(S)\\).</p> <p>The set \\(D_{-i}^{(k)}\\) contains all subsets of \\(D\\) of size \\(k\\) that do not include sample \\(x_i\\).</p>"},{"location":"value/semi-values/","title":"Semi-values","text":"<p>SV is a particular case of a more general concept called semi-value, which is a generalization to different weighting schemes. A semi-value is any valuation function with the form:</p> \\[ v_\\text{semi}(i) = \\sum_{i=1}^n w(k) \\sum_{S \\subset D_{-i}^{(k)}} [u(S_{+i}) - u(S)], \\] <p>where the coefficients \\(w(k)\\) satisfy the property:</p> \\[\\sum_{k=1}^n w(k) = 1,\\] <p>the set \\(D_{-i}^{(k)}\\) contains all subsets of \\(D\\) of size \\(k\\) that do not include sample \\(x_i\\), \\(S_{+i}\\) is the set \\(S\\) with \\(x_i\\) added, and \\(u\\) is the utility function.</p> <p>Two instances of this are Banzhaf indices (Wang and Jia, 2022)<sup>1</sup>, and Beta Shapley (Kwon and Zou, 2022)<sup>2</sup>, with better numerical and rank stability in certain situations.</p> <p>Note</p> <p>Shapley values are a particular case of semi-values and can therefore also be computed with the methods described here. However, as of version 0.7.0, we recommend using compute_shapley_values instead, in particular because it implements truncation policies for TMCS.</p>"},{"location":"value/semi-values/#beta-shapley","title":"Beta Shapley","text":"<p>For some machine learning applications, where the utility is typically the performance when trained on a set \\(S \\subset D\\), diminishing returns are often observed when computing the marginal utility of adding a new data point.</p> <p>Beta Shapley is a weighting scheme that uses the Beta function to place more weight on subsets deemed to be more informative. The weights are defined as:</p> \\[ w(k) := \\frac{B(k+\\beta, n-k+1+\\alpha)}{B(\\alpha, \\beta)}, \\] <p>where \\(B\\) is the Beta function, and \\(\\alpha\\) and \\(\\beta\\) are parameters that control the weighting of the subsets. Setting both to 1 recovers Shapley values, and setting \\(\\alpha = 1\\), and \\(\\beta = 16\\) is reported in (Kwon and Zou, 2022)<sup>2</sup> to be a good choice for some applications. Beta Shapley values are available in pyDVL through compute_beta_shapley_semivalues:</p> <pre><code>from pydvl.value import *\nutility = Utility(model, data)\nvalues = compute_beta_shapley_semivalues(\nu=utility, done=AbsoluteStandardError(threshold=1e-4), alpha=1, beta=16\n)\n</code></pre> <p>See however the Banzhaf indices section  for an alternative choice of weights which is reported to work better.</p>"},{"location":"value/semi-values/#banzhaf-indices","title":"Banzhaf indices","text":"<p>As noted in the section Problems of Data Values, the Shapley value can be very sensitive to variance in the utility function. For machine learning applications, where the utility is typically the performance when trained on a set \\(S \\subset D\\), this variance is often largest for smaller subsets \\(S\\). It is therefore reasonable to try reducing the relative contribution of these subsets with adequate weights.</p> <p>One such choice of weights is the Banzhaf index, which is defined as the constant:</p> \\[w(k) := 2^{n-1},\\] <p>for all set sizes \\(k\\). The intuition for picking a constant weight is that for any choice of weight function \\(w\\), one can always construct a utility with higher variance where \\(w\\) is greater. Therefore, in a worst-case sense, the best one can do is to pick a constant weight.</p> <p>The authors of (Wang and Jia, 2022)<sup>1</sup> show that Banzhaf indices are more robust to variance in the utility function than Shapley and Beta Shapley values. They are available in pyDVL through compute_banzhaf_semivalues:</p> <pre><code>from pydvl.value import *\nutility = Utility(model, data)\nvalues = compute_banzhaf_semivalues(\nu=utility, done=AbsoluteStandardError(threshold=1e-4), alpha=1, beta=16\n)\n</code></pre>"},{"location":"value/semi-values/#general-semi-values","title":"General semi-values","text":"<p>As explained above, both Beta Shapley and Banzhaf indices are special cases of semi-values. In pyDVL we provide a general method for computing these with any combination of the three ingredients that define a semi-value:</p> <ul> <li>A utility function \\(u\\).</li> <li>A sampling method</li> <li>A weighting scheme \\(w\\).</li> </ul> <p>You can construct any combination of these three ingredients with compute_generic_semivalues. The utility function is the same as for Shapley values, and the sampling method can be any of the types defined in the samplers module. For instance, the following snippet is equivalent to the above:</p> <pre><code>from pydvl.value import *\ndata = Dataset(...)\nutility = Utility(model, data)\nvalues = compute_generic_semivalues(\nsampler=PermutationSampler(data.indices),\nu=utility,\ncoefficient=beta_coefficient(alpha=1, beta=16),\ndone=AbsoluteStandardError(threshold=1e-4),\n)\n</code></pre> <p>Allowing any coefficient can help when experimenting with models which are more sensitive to changes in training set size. However, Data Banzhaf indices are proven to be the most robust to variance in the utility function, in the sense of rank stability, across a range of models and datasets (Wang and Jia, 2022)<sup>1</sup>. </p> <p>Careful with permutation sampling</p> <p>This generic implementation of semi-values allowing for any combination of sampling and weighting schemes is very flexible and, in principle, it recovers the original Shapley value, so that  compute_shapley_values is no longer necessary. However, it loses the optimization in permutation sampling that reuses the utility computation from the last iteration when iterating over a permutation. This doubles the computation requirements (and slightly increases variance) when using permutation sampling, unless the cache is enabled. In addition, as mentioned above, truncation policies are not supported by this generic implementation (as of v0.7.0). For these reasons it is preferable to use compute_shapley_values whenever not computing other semi-values.</p> <ol> <li> <p>Wang, J.T., Jia, R., 2022. Data Banzhaf: A Robust Data Valuation Framework for Machine Learning [WWW Document]. https://doi.org/10.48550/arXiv.2205.15466 \u21a9\u21a9\u21a9</p> </li> <li> <p>Kwon, Y., Zou, J., 2022. Beta Shapley: A Unified and Noise-reduced Data Valuation Framework for Machine Learning, in: Proceedings of the 25th International Conference on Artificial Intelligence and Statistics (AISTATS) 2022,. Presented at the AISTATS 2022, PMLR.\u00a0\u21a9\u21a9</p> </li> </ol>"},{"location":"value/shapley/","title":"Shapley value","text":""},{"location":"value/shapley/#shapley-value","title":"Shapley value","text":"<p>The Shapley method is an approach to compute data values originating in cooperative game theory. Shapley values are a common way of assigning payoffs to each participant in a cooperative game (i.e. one in which players can form coalitions) in a way that ensures that certain axioms are fulfilled.</p> <p>pyDVL implements several methods for the computation and approximation of Shapley values. They can all be accessed via the facade function compute_shapley_values. The supported methods are enumerated in ShapleyMode.</p> <p>Empirically, the most useful method is the so-called Truncated Monte Carlo Shapley (Ghorbani and Zou, 2019)<sup>1</sup>, which is a Monte Carlo approximation of the permutation Shapley value.</p>"},{"location":"value/shapley/#combinatorial-shapley","title":"Combinatorial Shapley","text":"<p>The first algorithm is just a verbatim implementation of the definition. As such it returns as exact a value as the utility function allows (see what this means in Problems of Data Values).</p> <p>The value \\(v\\) of the \\(i\\)-th sample in dataset \\(D\\) wrt. utility \\(u\\) is computed as a weighted sum of its marginal utility wrt. every possible coalition of training samples within the training set:</p> \\[ v(i) = \\frac{1}{n} \\sum_{S \\subseteq D_{-i}} \\binom{n-1}{ | S | }^{-1} [u(S_{+i}) \u2212 u(S)] ,\\] <p>where \\(D_{-i}\\) denotes the set of samples in \\(D\\) excluding \\(x_i\\), and \\(S_{+i}\\) denotes the set \\(S\\) with \\(x_i\\) added.</p> <pre><code>from pydvl.value import compute_shapley_values\nvalues = compute_shapley_values(utility, mode=\"combinatorial_exact\")\ndf = values.to_dataframe(column='value')\n</code></pre> <p>We can convert the return value to a pandas.DataFrame. and name the column with the results as <code>value</code>. Please refer to the documentation in shapley and ValuationResult for more information.</p>"},{"location":"value/shapley/#monte-carlo-combinatorial-shapley","title":"Monte Carlo Combinatorial Shapley","text":"<p>Because the number of subsets \\(S \\subseteq D_{-i}\\) is \\(2^{ | D | - 1 }\\), one typically must resort to approximations. The simplest one is done via Monte Carlo sampling of the powerset \\(\\mathcal{P}(D)\\). In pyDVL this simple technique is called \"Monte Carlo Combinatorial\". The method has very poor converge rate and others are preferred, but if desired, usage follows the same pattern:</p> <pre><code>from pydvl.value import compute_shapley_values, MaxUpdates\nvalues = compute_shapley_values(\nutility, mode=\"combinatorial_montecarlo\", done=MaxUpdates(1000)\n)\ndf = values.to_dataframe(column='cmc')\n</code></pre> <p>The DataFrames returned by most Monte Carlo methods will contain approximate standard errors as an additional column, in this case named <code>cmc_stderr</code>.</p> <p>Note the usage of the object MaxUpdates as the stop condition. This is an instance of a StoppingCriterion. Other examples are MaxTime and AbsoluteStandardError.</p>"},{"location":"value/shapley/#owen-sampling","title":"Owen sampling","text":"<p>Owen Sampling (Okhrati and Lipani, 2021)<sup>2</sup> is a practical algorithm based on the combinatorial definition. It uses a continuous extension of the utility from \\(\\{0,1\\}^n\\), where a 1 in position \\(i\\) means that sample \\(x_i\\) is used to train the model, to \\([0,1]^n\\). The ensuing expression for Shapley value uses integration instead of discrete weights:</p> \\[ v_u(i) = \\int_0^1 \\mathbb{E}_{S \\sim P_q(D_{-i})} [u(S_{+i}) - u(S)]. \\] <p>Using Owen sampling follows the same pattern as every other method for Shapley values in pyDVL. First construct the dataset and utility, then call compute_shapley_values:</p> <pre><code>from pydvl.value import compute_shapley_values\nvalues = compute_shapley_values(\nu=utility, mode=\"owen\", n_iterations=4, max_q=200\n)\n</code></pre> <p>There are more details on Owen sampling, and its variant Antithetic Owen Sampling in the documentation for the function doing the work behind the scenes: owen_sampling_shapley.</p> <p>Note that in this case we do not pass a StoppingCriterion to the function, but instead the number of iterations and the maximum number of samples to use in the integration.</p>"},{"location":"value/shapley/#permutation-shapley","title":"Permutation Shapley","text":"<p>An equivalent way of computing Shapley values (<code>ApproShapley</code>) appeared in  (Castro et al., 2009)<sup>3</sup> and is the basis for the method most often used in practice. It uses permutations over indices instead of subsets:</p> \\[ v_u(x_i) = \\frac{1}{n!} \\sum_{\\sigma \\in \\Pi(n)} [u(\\sigma_{:i} \\cup \\{x_i\\}) \u2212 u(\\sigma_{:i})], \\] <p>where \\(\\sigma_{:i}\\) denotes the set of indices in permutation sigma before the position where \\(i\\) appears. To approximate this sum (which has \\(\\mathcal{O}(n!)\\) terms!) one uses Monte Carlo sampling of permutations, something which has surprisingly low sample complexity. One notable difference wrt. the combinatorial approach above is that the approximations always fulfill the efficiency axiom of Shapley, namely \\(\\sum_{i=1}^n \\hat{v}_i = u(D)\\) (see  (Castro et al., 2009)<sup>3</sup>, Proposition 3.2).</p> <p>By adding two types of early stopping, the result is the so-called Truncated Monte Carlo Shapley (Ghorbani and Zou, 2019)<sup>1</sup>, which is efficient enough to be useful in applications. The first is simply a convergence criterion, of which there are several to choose from. The second is a criterion to truncate the iteration over single permutations. RelativeTruncation chooses to stop iterating over samples in a permutation when the marginal utility becomes too small.</p> <pre><code>from pydvl.value import compute_shapley_values, MaxUpdates, RelativeTruncation\nvalues = compute_shapley_values(\nu=utility,\nmode=\"permutation_montecarlo\",\ndone=MaxUpdates(1000),\ntruncation=RelativeTruncation(utility, rtol=0.01)\n)\n</code></pre> <p>You can see this method in action in this example using the Spotify dataset.</p>"},{"location":"value/shapley/#exact-shapley-for-knn","title":"Exact Shapley for KNN","text":"<p>It is possible to exploit the local structure of K-Nearest Neighbours to reduce the amount of subsets to consider: because no sample besides the K closest affects the score, most are irrelevant and it is possible to compute a value in linear time. This method was introduced by (Jia et al., 2019)<sup>4</sup>, and can be used in pyDVL with:</p> <pre><code>from pydvl.utils import Dataset, Utility\nfrom pydvl.value import compute_shapley_values\nfrom sklearn.neighbors import KNeighborsClassifier\nmodel = KNeighborsClassifier(n_neighbors=5)\ndata = Dataset(...)\nutility = Utility(model, data)\nvalues = compute_shapley_values(u=utility, mode=\"knn\")\n</code></pre>"},{"location":"value/shapley/#group-testing","title":"Group testing","text":"<p>An alternative approach introduced in (Jia et al., 2019)<sup>4</sup> first approximates the differences of values with a Monte Carlo sum. With</p> \\[\\hat{\\Delta}_{i j} \\approx v_i - v_j,\\] <p>one then solves the following linear constraint satisfaction problem (CSP) to infer the final values:</p> \\[ \\begin{array}{lll} \\sum_{i = 1}^N v_i &amp; = &amp; U (D)\\\\ | v_i - v_j - \\hat{\\Delta}_{i j} | &amp; \\leqslant &amp; \\frac{\\varepsilon}{2 \\sqrt{N}} \\end{array} \\] <p>Warning</p> <p>We have reproduced this method in pyDVL for completeness and benchmarking, but we don't advocate its use because of the speed and memory cost. Despite our best efforts, the number of samples required in practice for convergence can be several orders of magnitude worse than with e.g. TMCS. Additionally, the CSP can sometimes turn out to be infeasible.</p> <p>Usage follows the same pattern as every other Shapley method, but with the addition of an <code>epsilon</code> parameter required for the solution of the CSP. It should be the same value used to compute the minimum number of samples required. This can be done with num_samples_eps_delta, but note that the number returned will be huge! In practice, fewer samples can be enough, but the actual number will strongly depend on the utility, in particular its variance.</p> <pre><code>from pydvl.utils import Dataset, Utility\nfrom pydvl.value import compute_shapley_values\nmodel = ...\ndata = Dataset(...)\nutility = Utility(model, data, score_range=(_min, _max))\nmin_iterations = num_samples_eps_delta(epsilon, delta, n, utility.score_range)\nvalues = compute_shapley_values(\nu=utility, mode=\"group_testing\", n_iterations=min_iterations, eps=eps\n)\n</code></pre> <ol> <li> <p>Ghorbani, A., Zou, J., 2019. Data Shapley: Equitable Valuation of Data for Machine Learning, in: Proceedings of the 36th International Conference on Machine Learning, PMLR. Presented at the International Conference on Machine Learning (ICML 2019), PMLR, pp. 2242\u20132251.\u00a0\u21a9\u21a9</p> </li> <li> <p>Okhrati, R., Lipani, A., 2021. A Multilinear Sampling Algorithm to Estimate Shapley Values, in: 2020 25th International Conference on Pattern Recognition (ICPR). Presented at the 2020 25th International Conference on Pattern Recognition (ICPR), IEEE, pp. 7992\u20137999. https://doi.org/10.1109/ICPR48806.2021.9412511 \u21a9</p> </li> <li> <p>Castro, J., G\u00f3mez, D., Tejada, J., 2009. Polynomial calculation of the Shapley value based on sampling. Computers   &amp; Operations Research, Selected papers presented at the Tenth International Symposium on Locational Decisions (ISOLDE X) 36, 1726\u20131730. https://doi.org/10.1016/j.cor.2008.04.004 \u21a9\u21a9</p> </li> <li> <p>Jia, R., Dao, D., Wang, B., Hubis, F.A., Gurel, N.M., Li, B., Zhang, C., Spanos, C., Song, D., 2019. Efficient task-specific data valuation for nearest neighbor algorithms. Proc. VLDB Endow. 12, 1610\u20131623. https://doi.org/10.14778/3342263.3342637 \u21a9\u21a9</p> </li> </ol>"},{"location":"value/the-core/","title":"Core values","text":"<p>The Shapley values define a fair way to distribute payoffs amongst all participants when they form a grand coalition. But they do not consider the question of stability: under which conditions do all participants form the grand coalition? Would the participants be willing to form the grand coalition given how the payoffs are assigned, or would some of them prefer to form smaller coalitions?</p> <p>The Core is another approach to computing data values originating in cooperative game theory that attempts to ensure this stability. It is the set of feasible payoffs that cannot be improved upon by a coalition of the participants.</p> <p>It satisfies the following 2 properties:</p> <ul> <li> <p>Efficiency:   The payoffs are distributed such that it is not possible   to make any participant better off   without making another one worse off.   \\(\\(\\sum_{i\\in D} v(i) = u(D)\\,\\)\\)</p> </li> <li> <p>Coalitional rationality:   The sum of payoffs to the agents in any coalition S is at   least as large as the amount that these agents could earn by   forming a coalition on their own.   \\(\\(\\sum_{i \\in S} v(i) \\geq u(S), \\forall S \\subset D\\,\\)\\)</p> </li> </ul> <p>The second property states that the sum of payoffs to the agents in any subcoalition \\(S\\) is at least as large as the amount that these agents could earn by forming a coalition on their own.</p>"},{"location":"value/the-core/#least-core-values","title":"Least Core values","text":"<p>Unfortunately, for many cooperative games the Core may be empty. By relaxing the coalitional rationality property by a subsidy \\(e \\gt 0\\), we are then able to find approximate payoffs:</p> \\[ \\sum_{i\\in S} v(i) + e \\geq u(S), \\forall S \\subset D, S \\neq \\emptyset \\ ,\\] <p>The least core value \\(v\\) of the \\(i\\)-th sample in dataset \\(D\\) wrt. utility \\(u\\) is computed by solving the following Linear Program:</p> \\[ \\begin{array}{lll} \\text{minimize} &amp; e &amp; \\\\ \\text{subject to} &amp; \\sum_{i\\in D} v(i) = u(D) &amp; \\\\ &amp; \\sum_{i\\in S} v(i) + e \\geq u(S) &amp;, \\forall S \\subset D, S \\neq \\emptyset  \\\\ \\end{array} \\]"},{"location":"value/the-core/#exact-least-core","title":"Exact Least Core","text":"<p>This first algorithm is just a verbatim implementation of the definition. As such it returns as exact a value as the utility function allows (see what this means in Problems of Data Values][problems-of-data-values]).</p> <pre><code>from pydvl.value import compute_least_core_values\nvalues = compute_least_core_values(utility, mode=\"exact\")\n</code></pre>"},{"location":"value/the-core/#monte-carlo-least-core","title":"Monte Carlo Least Core","text":"<p>Because the number of subsets \\(S \\subseteq D \\setminus \\{i\\}\\) is \\(2^{ | D | - 1 }\\), one typically must resort to approximations.</p> <p>The simplest approximation consists in using a fraction of all subsets for the constraints. (Yan and Procaccia, 2021)<sup>1</sup> show that a quantity of order \\(\\mathcal{O}((n - \\log \\Delta ) / \\delta^2)\\) is enough to obtain a so-called \\(\\delta\\)-approximate least core with high probability. I.e. the following property holds with probability \\(1-\\Delta\\) over the choice of subsets:</p> \\[ \\mathbb{P}_{S\\sim D}\\left[\\sum_{i\\in S} v(i) + e^{*} \\geq u(S)\\right] \\geq 1 - \\delta, \\] <p>where \\(e^{*}\\) is the optimal least core subsidy.</p> <pre><code>from pydvl.value import compute_least_core_values\nvalues = compute_least_core_values(\nutility, mode=\"montecarlo\", n_iterations=n_iterations\n)\n</code></pre> <p>Note</p> <p>Although any number is supported, it is best to choose <code>n_iterations</code> to be at least equal to the number of data points.</p> <p>Because computing the Least Core values requires the solution of a linear and a quadratic problem after computing all the utility values, we offer the possibility of splitting the latter from the former. This is useful when running multiple experiments: use mclc_prepare_problem to prepare a list of problems to solve, then solve them in parallel with lc_solve_problems.</p> <pre><code>from pydvl.value.least_core import mclc_prepare_problem, lc_solve_problems\nn_experiments = 10\nproblems = [mclc_prepare_problem(utility, n_iterations=n_iterations)\nfor _ in range(n_experiments)]\nvalues = lc_solve_problems(problems)\n</code></pre>"},{"location":"value/the-core/#method-comparison","title":"Method comparison","text":"<p>The TransferLab team reproduced the results of the original paper in a publication for the 2022 MLRC (Benmerzoug and Delgado, 2023)<sup>2</sup>.</p> <p> Best sample removal on binary image classification </p> <p>Roughly speaking, MCLC performs better in identifying high value points, as measured by best-sample removal tasks. In all other aspects, it performs worse or similarly to TMCS at comparable sample budgets. But using an equal number of subsets is more computationally expensive because of the need to solve large linear and quadratic optimization problems.</p> <p> Worst sample removal on binary image classification </p> <p>For these reasons we recommend some variation of SV like TMCS for outlier detection, data cleaning and pruning, and perhaps MCLC for the selection of interesting points to be inspected for the improvement of data collection or model design.</p> <ol> <li> <p>Yan, T., Procaccia, A.D., 2021. If You Like Shapley Then You\u2019ll Love the Core, in: Proceedings of the 35th AAAI Conference on Artificial Intelligence, 2021. Presented at the AAAI Conference on Artificial Intelligence, Association for the Advancement of Artificial Intelligence, pp. 5751\u20135759. https://doi.org/10.1609/aaai.v35i6.16721 \u21a9</p> </li> <li> <p>Benmerzoug, A., Delgado, M. de B., 2023. [Re] If you like Shapley, then you\u2019ll love the core. ReScience C 9. https://doi.org/10.5281/zenodo.8173733 \u21a9</p> </li> </ol>"}]}
\ No newline at end of file
diff --git a/devel/sitemap.xml b/devel/sitemap.xml
index 36cb7d04f..54281c5ba 100644
--- a/devel/sitemap.xml
+++ b/devel/sitemap.xml
@@ -2,372 +2,372 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/CHANGELOG/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/influence/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/influence/general/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/influence/inversion/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/influence/twice_differentiable/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/influence/torch/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/influence/torch/functional/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/influence/torch/torch_differentiable/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/influence/torch/util/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/parallel/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/parallel/backend/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/parallel/config/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/parallel/map_reduce/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/parallel/backends/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/parallel/backends/joblib/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/parallel/backends/ray/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/parallel/futures/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/parallel/futures/ray/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/reporting/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/reporting/plots/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/reporting/scores/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/utils/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/utils/caching/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/utils/config/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/utils/dataset/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/utils/functional/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/utils/numeric/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/utils/parallel/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/utils/progress/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/utils/score/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/utils/status/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/utils/types/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/utils/utility/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/result/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/sampler/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/semivalues/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/stopping/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/least_core/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/least_core/common/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/least_core/montecarlo/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/least_core/naive/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/loo/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/loo/loo/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/loo/naive/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/oob/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/oob/oob/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/shapley/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/shapley/common/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/shapley/gt/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/shapley/knn/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/shapley/montecarlo/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/shapley/naive/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/shapley/owen/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/shapley/truncated/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/api/pydvl/value/shapley/types/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/examples/data_oob/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/examples/influence_imagenet/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/examples/influence_synthetic/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/examples/influence_wine/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/examples/least_core_basic/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/examples/shapley_basic_spotify/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/examples/shapley_knn_flowers/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/examples/shapley_utility_learning/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/getting-started/first-steps/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/getting-started/installation/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/influence/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/value/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/value/notation/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/value/semi-values/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/value/shapley/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://aai-institute.github.io/pyDVL/stable/value/the-core/</loc>
-         <lastmod>2023-09-12</lastmod>
+         <lastmod>2023-09-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/devel/sitemap.xml.gz b/devel/sitemap.xml.gz
index 1d9ffc3bc..53f3efa26 100644
Binary files a/devel/sitemap.xml.gz and b/devel/sitemap.xml.gz differ
diff --git a/devel/value/index.html b/devel/value/index.html
index 0bdae9b43..43cc01e56 100644
--- a/devel/value/index.html
+++ b/devel/value/index.html
@@ -2608,24 +2608,26 @@ <h1 id="data-valuation">Data valuation<a class="headerlink" href="#data-valuatio
 training set which reflects its contribution to the final performance of some
 model trained on it. Some methods attempt to be model-agnostic, but in most
 cases the model is an integral part of the method. In these cases, this number
-not an intrinsic property of the element of interest, but typically a function
-of three factors:</p>
+is not an intrinsic property of the element of interest, but typically a
+function of three factors:</p>
 <ol>
 <li>
-<p>The dataset <span class="arithmatex">\(D\)</span>, or more generally, the distribution it was sampled
-   from (with this we mean that <em>value</em> would ideally be the (expected)
-   contribution of a data point to any random set <span class="arithmatex">\(D\)</span> sampled from the same
-   distribution).</p>
+<p>The dataset <span class="arithmatex">\(D\)</span>, or more generally, the distribution it was sampled from: In
+   some cases one only cares about values wrt. a given data set, in others
+   value would ideally be the (expected) contribution of a data point to any
+   random set <span class="arithmatex">\(D\)</span> sampled from the same distribution. pyDVL implements methods
+   of the first kind.</p>
 </li>
 <li>
-<p>The algorithm <span class="arithmatex">\(\mathcal{A}\)</span> mapping the data <span class="arithmatex">\(D\)</span> to some estimator <span class="arithmatex">\(f\)</span>
-   in a model class <span class="arithmatex">\(\mathcal{F}\)</span>. E.g. <abbr title="Mean Squared Error">MSE</abbr> minimization to find the parameters
-   of a linear model.</p>
+<p>The algorithm <span class="arithmatex">\(\mathcal{A}\)</span> mapping the data <span class="arithmatex">\(D\)</span> to some estimator <span class="arithmatex">\(f\)</span> in a
+   model class <span class="arithmatex">\(\mathcal{F}\)</span>. E.g. <abbr title="Mean Squared Error">MSE</abbr> minimization to find the parameters of a
+   linear model.</p>
 </li>
 <li>
 <p>The performance metric of interest <span class="arithmatex">\(u\)</span> for the problem. When value depends on
-   a model, it must be measured in some way which uses it. E.g. the <span class="arithmatex">\(R^2\)</span> score or
-   the negative <abbr title="Mean Squared Error">MSE</abbr> over a test set.</p>
+   a model, it must be measured in some way which uses it. E.g. the <span class="arithmatex">\(R^2\)</span> score
+   or the negative <abbr title="Mean Squared Error">MSE</abbr> over a test set. This metric will be computed over a
+   held-out valuation set.</p>
 </li>
 </ol>
 <p>pyDVL collects algorithms for the computation of data values in this sense,
@@ -2954,11 +2956,11 @@ <h2 id="problems-of-data-values">Problems of data values<a class="headerlink" hr
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/value/notation/index.html b/devel/value/notation/index.html
index e4ef3539e..972b5cfdb 100644
--- a/devel/value/notation/index.html
+++ b/devel/value/notation/index.html
@@ -2469,11 +2469,11 @@ <h1 id="notation-for-valuation">Notation for valuation<a class="headerlink" href
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/value/semi-values/index.html b/devel/value/semi-values/index.html
index 5cade4af7..a134d89f5 100644
--- a/devel/value/semi-values/index.html
+++ b/devel/value/semi-values/index.html
@@ -2667,11 +2667,11 @@ <h2 id="general-semi-values">General semi-values<a class="headerlink" href="#gen
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/value/shapley/index.html b/devel/value/shapley/index.html
index 02bf587a0..eb73660d6 100644
--- a/devel/value/shapley/index.html
+++ b/devel/value/shapley/index.html
@@ -2788,11 +2788,11 @@ <h3 id="group-testing">Group testing<a class="headerlink" href="#group-testing"
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>
diff --git a/devel/value/the-core/index.html b/devel/value/the-core/index.html
index dcc81080f..04b36c1c9 100644
--- a/devel/value/the-core/index.html
+++ b/devel/value/the-core/index.html
@@ -2674,11 +2674,11 @@ <h2 id="method-comparison">Method comparison<a class="headerlink" href="#method-
   <small>
     
       Last update:
-      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+      <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
         <br>
         Created:
-        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-12</span>
+        <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-09-18</span>
       
     
   </small>