12 13 14 15 16 def shaded_mean_std (
-data : np . ndarray ,
-abscissa : Optional [ Sequence [ Any ]] = None ,
-num_std : float = 1.0 ,
-mean_color : Optional [ str ] = "dodgerblue" ,
-shade_color : Optional [ str ] = "lightblue" ,
-title : Optional [ str ] = None ,
-xlabel : Optional [ str ] = None ,
-ylabel : Optional [ str ] = None ,
-ax : Optional [ Axes ] = None ,
-** kwargs ,
-) -> Axes :
- """The usual mean \(\pm\) std deviation plot to aggregate runs of experiments.
- Args:
- data: axis 0 is to be aggregated on (e.g. runs) and axis 1 is the
- data for each run.
- abscissa: values for the x-axis. Leave empty to use increasing integers.
- num_std: number of standard deviations to shade around the mean.
- mean_color: color for the mean
- shade_color: color for the shaded region
- title: Title text. To use mathematics, use LaTeX notation.
- xlabel: Text for the horizontal axis.
- ylabel: Text for the vertical axis
- ax: If passed, axes object into which to insert the figure. Otherwise,
- a new figure is created and returned
- kwargs: these are forwarded to the ax.plot() call for the mean.
- Returns:
- The axes used (or created)
- """
-assert len ( data . shape ) == 2
-mean = data . mean ( axis = 0 )
-std = num_std * data . std ( axis = 0 )
-if ax is None :
-fig , ax = plt . subplots ()
-if abscissa is None :
-abscissa = list ( range ( data . shape [ 1 ]))
-ax . fill_between ( abscissa , mean - std , mean + std , alpha = 0.3 , color = shade_color )
-ax . plot ( abscissa , mean , color = mean_color , ** kwargs )
-ax . set_title ( title )
-ax . set_xlabel ( xlabel )
-ax . set_ylabel ( ylabel )
-return ax
+59 60 61 62 63 64 65 66 67 68 69 @deprecated ( target = None , deprecated_in = "0.7.1" , remove_in = "0.9.0" )
+def shaded_mean_std (
+data : np . ndarray ,
+abscissa : Optional [ Sequence [ Any ]] = None ,
+num_std : float = 1.0 ,
+mean_color : Optional [ str ] = "dodgerblue" ,
+shade_color : Optional [ str ] = "lightblue" ,
+title : Optional [ str ] = None ,
+xlabel : Optional [ str ] = None ,
+ylabel : Optional [ str ] = None ,
+ax : Optional [ Axes ] = None ,
+** kwargs ,
+) -> Axes :
+ r """The usual mean \(\pm\) std deviation plot to aggregate runs of
+ experiments.
+ !!! warning "Deprecation notice"
+ This function is bogus and will be removed in the future in favour of
+ properly computed confidence intervals.
+ Args:
+ data: axis 0 is to be aggregated on (e.g. runs) and axis 1 is the
+ data for each run.
+ abscissa: values for the x-axis. Leave empty to use increasing integers.
+ num_std: number of standard deviations to shade around the mean.
+ mean_color: color for the mean
+ shade_color: color for the shaded region
+ title: Title text. To use mathematics, use LaTeX notation.
+ xlabel: Text for the horizontal axis.
+ ylabel: Text for the vertical axis
+ ax: If passed, axes object into which to insert the figure. Otherwise,
+ a new figure is created and returned
+ kwargs: these are forwarded to the ax.plot() call for the mean.
+ Returns:
+ The axes used (or created)
+ """
+assert len ( data . shape ) == 2
+mean = data . mean ( axis = 0 )
+std = num_std * data . std ( axis = 0 )
+if ax is None :
+fig , ax = plt . subplots ()
+if abscissa is None :
+abscissa = list ( range ( data . shape [ 1 ]))
+ax . fill_between ( abscissa , mean - std , mean + std , alpha = 0.3 , color = shade_color )
+ax . plot ( abscissa , mean , color = mean_color , ** kwargs )
+ax . set_title ( title )
+ax . set_xlabel ( xlabel )
+ax . set_ylabel ( ylabel )
+return ax
+
+
+
+
+
+
+ plot_ci_array ( data , level , type = 'normal' , abscissa = None , mean_color = 'dodgerblue' , shade_color = 'lightblue' , ax = None , ** kwargs )
+
+
+
+
+
Plot values and a confidence interval from a 2D array.
+
Supported intervals are based on the normal and the t distributions.
+
+
+
+
+
+
+ PARAMETER DESCRIPTION
+
+
+
+ data
+
+
A 2D array with M different values for each of the N indices.
+
+
+ TYPE:
+ NDArray
+
+
+
+
+ level
+
+
The confidence level.
+
+
+ TYPE:
+ float
+
+
+
+
+ type
+
+
The type of confidence interval to use.
+
+
+ TYPE:
+ Literal ['normal', 't', 'auto']
+
+ DEFAULT:
+ 'normal'
+
+
+
+
+
+ abscissa
+
+
The values for the x-axis. Leave empty to use increasing
+integers.
+
+
+ TYPE:
+ Optional [Sequence [str ]]
+
+ DEFAULT:
+ None
+
+
+
+
+
+ mean_color
+
+
The color of the mean line.
+
+
+ TYPE:
+ Optional [str ]
+
+ DEFAULT:
+ 'dodgerblue'
+
+
+
+
+
+ shade_color
+
+
The color of the confidence interval.
+
+
+ TYPE:
+ Optional [str ]
+
+ DEFAULT:
+ 'lightblue'
+
+
+
+
+
+ ax
+
+
If passed, axes object into which to insert the figure. Otherwise,
+a new figure is created and the axes returned.
+
+
+ TYPE:
+ Optional [Axes ]
+
+ DEFAULT:
+ None
+
+
+
+
+
+ **kwargs
+
+
Additional arguments to pass to the plot function.
+
+
+ DEFAULT:
+ {}
+
+
+
+
+
+
+
+
+
+
+
+
+ RETURNS DESCRIPTION
+
+
+
+
+
+ Axes
+
+
+
+
+
+
+
+
+
+ Source code in src/pydvl/reporting/plots.py
+ def plot_ci_array (
+data : NDArray ,
+level : float ,
+type : Literal [ "normal" , "t" , "auto" ] = "normal" ,
+abscissa : Optional [ Sequence [ str ]] = None ,
+mean_color : Optional [ str ] = "dodgerblue" ,
+shade_color : Optional [ str ] = "lightblue" ,
+ax : Optional [ plt . Axes ] = None ,
+** kwargs ,
+) -> plt . Axes :
+ """Plot values and a confidence interval from a 2D array.
+ Supported intervals are based on the normal and the t distributions.
+ Args:
+ data: A 2D array with M different values for each of the N indices.
+ level: The confidence level.
+ type: The type of confidence interval to use.
+ abscissa: The values for the x-axis. Leave empty to use increasing
+ integers.
+ mean_color: The color of the mean line.
+ shade_color: The color of the confidence interval.
+ ax: If passed, axes object into which to insert the figure. Otherwise,
+ a new figure is created and the axes returned.
+ **kwargs: Additional arguments to pass to the plot function.
+ Returns:
+ The matplotlib axes.
+ """
+m , n = data . shape
+means = np . mean ( data , axis = 0 )
+variances = np . var ( data , axis = 0 , ddof = 1 )
+dummy : ValuationResult [ np . int_ , str ] = ValuationResult (
+algorithm = "dummy" ,
+values = means ,
+variances = variances ,
+counts = np . ones_like ( means , dtype = np . int_ ) * m ,
+indices = np . arange ( n ),
+data_names = np . array ( abscissa , dtype = str )
+if abscissa is not None
+else np . arange ( n , dtype = str ),
+)
+return plot_ci_values (
+dummy ,
+level = level ,
+type = type ,
+mean_color = mean_color ,
+shade_color = shade_color ,
+ax = ax ,
+** kwargs ,
+)
+
+
+
+
+
+
+
+
+
+ plot_ci_values ( values , level , type = 'auto' , abscissa = None , mean_color = 'dodgerblue' , shade_color = 'lightblue' , ax = None , ** kwargs )
+
+
+
+
+
Plot values and a confidence interval.
+
Uses values.data_names for the x-axis.
+
Supported intervals are based on the normal and the t distributions.
+
+
+
+
+
+
+ PARAMETER DESCRIPTION
+
+
+
+ values
+
+
The valuation result.
+
+
+ TYPE:
+ ValuationResult
+
+
+
+
+ level
+
+
The confidence level.
+
+
+ TYPE:
+ float
+
+
+
+
+ type
+
+
The type of confidence interval to use. If "auto", uses "norm" if
+the minimum number of updates for all indices is greater than 30,
+otherwise uses "t".
+
+
+ TYPE:
+ Literal ['normal', 't', 'auto']
+
+ DEFAULT:
+ 'auto'
+
+
+
+
+
+ abscissa
+
+
The values for the x-axis. Leave empty to use increasing
+integers.
+
+
+ TYPE:
+ Optional [Sequence [str ]]
+
+ DEFAULT:
+ None
+
+
+
+
+
+ mean_color
+
+
The color of the mean line.
+
+
+ TYPE:
+ Optional [str ]
+
+ DEFAULT:
+ 'dodgerblue'
+
+
+
+
+
+ shade_color
+
+
The color of the confidence interval.
+
+
+ TYPE:
+ Optional [str ]
+
+ DEFAULT:
+ 'lightblue'
+
+
+
+
+
+ ax
+
+
If passed, axes object into which to insert the figure. Otherwise,
+a new figure is created and the axes returned.
+
+
+ TYPE:
+ Optional [Axes ]
+
+ DEFAULT:
+ None
+
+
+
+
+
+ **kwargs
+
+
Additional arguments to pass to the plot function.
+
+
+ DEFAULT:
+ {}
+
+
+
+
+
+
+
+
+
+
+
+
+ RETURNS DESCRIPTION
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Source code in src/pydvl/reporting/plots.py
+ def plot_ci_values (
+values : ValuationResult ,
+level : float ,
+type : Literal [ "normal" , "t" , "auto" ] = "auto" ,
+abscissa : Optional [ Sequence [ str ]] = None ,
+mean_color : Optional [ str ] = "dodgerblue" ,
+shade_color : Optional [ str ] = "lightblue" ,
+ax : Optional [ plt . Axes ] = None ,
+** kwargs ,
+):
+ """Plot values and a confidence interval.
+ Uses `values.data_names` for the x-axis.
+ Supported intervals are based on the normal and the t distributions.
+ Args:
+ values: The valuation result.
+ level: The confidence level.
+ type: The type of confidence interval to use. If "auto", uses "norm" if
+ the minimum number of updates for all indices is greater than 30,
+ otherwise uses "t".
+ abscissa: The values for the x-axis. Leave empty to use increasing
+ integers.
+ mean_color: The color of the mean line.
+ shade_color: The color of the confidence interval.
+ ax: If passed, axes object into which to insert the figure. Otherwise,
+ a new figure is created and the axes returned.
+ **kwargs: Additional arguments to pass to the plot function.
+ Returns:
+ The matplotlib axes.
+ """
+ppfs = {
+"normal" : norm . ppf ,
+"t" : partial ( t . ppf , df = values . counts - 1 ),
+"auto" : norm . ppf
+if np . min ( values . counts ) > 30
+else partial ( t . ppf , df = values . counts - 1 ),
+}
+try :
+score = ppfs [ type ]( 1 - level / 2 )
+except KeyError :
+raise ValueError (
+f "Unknown confidence interval type requested: { type } ."
+) from None
+if abscissa is None :
+abscissa = [ str ( i ) for i , _ in enumerate ( values )]
+bound = score * values . stderr
+if ax is None :
+fig , ax = plt . subplots ()
+ax . fill_between (
+abscissa ,
+values . values - bound ,
+values . values + bound ,
+alpha = 0.3 ,
+color = shade_color ,
+)
+ax . plot ( abscissa , values . values , color = mean_color , ** kwargs )
+return ax
@@ -2984,79 +3662,79 @@
Source code in src/pydvl/reporting/plots.py
- def spearman_correlation ( vv : List [ OrderedDict ], num_values : int , pvalue : float ):
- """Simple matrix plots with spearman correlation for each pair in vv.
- Args:
- vv: list of OrderedDicts with index: value. Spearman correlation
- is computed for the keys.
- num_values: Use only these many values from the data (from the start
- of the OrderedDicts)
- pvalue: correlation coefficients for which the p-value is below the
- threshold `pvalue/len(vv)` will be discarded.
- """
-r : np . ndarray = np . ndarray (( len ( vv ), len ( vv )))
-p : np . ndarray = np . ndarray (( len ( vv ), len ( vv )))
-for i , a in enumerate ( vv ):
-for j , b in enumerate ( vv ):
-from scipy.stats._stats_py import SpearmanrResult
-spearman : SpearmanrResult = sp . stats . spearmanr (
-list ( a . keys ())[: num_values ], list ( b . keys ())[: num_values ]
-)
-r [ i ][ j ] = (
-spearman . correlation if spearman . pvalue < pvalue / len ( vv ) else np . nan
-) # Bonferroni correction
-p [ i ][ j ] = spearman . pvalue
-fig , axs = plt . subplots ( 1 , 2 , figsize = ( 16 , 7 ))
-plot1 = axs [ 0 ] . matshow ( r , vmin =- 1 , vmax = 1 )
-axs [ 0 ] . set_title ( f "Spearman correlation (top { num_values } values)" )
-axs [ 0 ] . set_xlabel ( "Runs" )
-axs [ 0 ] . set_ylabel ( "Runs" )
-fig . colorbar ( plot1 , ax = axs [ 0 ])
-plot2 = axs [ 1 ] . matshow ( p , vmin = 0 , vmax = 1 )
-axs [ 1 ] . set_title ( "p-value" )
-axs [ 1 ] . set_xlabel ( "Runs" )
-axs [ 1 ] . set_ylabel ( "Runs" )
-fig . colorbar ( plot2 , ax = axs [ 1 ])
-return fig
+ def spearman_correlation ( vv : List [ OrderedDict ], num_values : int , pvalue : float ):
+ """Simple matrix plots with spearman correlation for each pair in vv.
+ Args:
+ vv: list of OrderedDicts with index: value. Spearman correlation
+ is computed for the keys.
+ num_values: Use only these many values from the data (from the start
+ of the OrderedDicts)
+ pvalue: correlation coefficients for which the p-value is below the
+ threshold `pvalue/len(vv)` will be discarded.
+ """
+r : np . ndarray = np . ndarray (( len ( vv ), len ( vv )))
+p : np . ndarray = np . ndarray (( len ( vv ), len ( vv )))
+for i , a in enumerate ( vv ):
+for j , b in enumerate ( vv ):
+from scipy.stats._stats_py import SpearmanrResult
+spearman : SpearmanrResult = sp . stats . spearmanr (
+list ( a . keys ())[: num_values ], list ( b . keys ())[: num_values ]
+)
+r [ i ][ j ] = (
+spearman . correlation if spearman . pvalue < pvalue / len ( vv ) else np . nan
+) # Bonferroni correction
+p [ i ][ j ] = spearman . pvalue
+fig , axs = plt . subplots ( 1 , 2 , figsize = ( 16 , 7 ))
+plot1 = axs [ 0 ] . matshow ( r , vmin =- 1 , vmax = 1 )
+axs [ 0 ] . set_title ( f "Spearman correlation (top { num_values } values)" )
+axs [ 0 ] . set_xlabel ( "Runs" )
+axs [ 0 ] . set_ylabel ( "Runs" )
+fig . colorbar ( plot1 , ax = axs [ 0 ])
+plot2 = axs [ 1 ] . matshow ( p , vmin = 0 , vmax = 1 )
+axs [ 1 ] . set_title ( "p-value" )
+axs [ 1 ] . set_xlabel ( "Runs" )
+axs [ 1 ] . set_ylabel ( "Runs" )
+fig . colorbar ( plot2 , ax = axs [ 1 ])
+return fig
Plots the shapley values, as returned from
-compute_shapley_values , with error bars
-corresponding to an \(\alpha\) -level confidence interval.
+
compute_shapley_values ,
+with error bars corresponding to an
\(\alpha\) -level Normal confidence
+interval.
@@ -3225,75 +3904,77 @@
Source code in src/pydvl/reporting/plots.py
- def plot_shapley (
-df : pd . DataFrame ,
-* ,
-level : float = 0.05 ,
-ax : Optional [ plt . Axes ] = None ,
-title : Optional [ str ] = None ,
-xlabel : Optional [ str ] = None ,
-ylabel : Optional [ str ] = None ,
-) -> plt . Axes :
- r """Plots the shapley values, as returned from
- [compute_shapley_values][pydvl.value.shapley.common.compute_shapley_values], with error bars
- corresponding to an $\alpha$-level confidence interval.
- Args:
- df: dataframe with the shapley values
- level: confidence level for the error bars
- ax: axes to plot on or None if a new subplots should be created
- title: string, title of the plot
- xlabel: string, x label of the plot
- ylabel: string, y label of the plot
- Returns:
- The axes created or used
- """
-if ax is None :
-_ , ax = plt . subplots ()
-yerr = norm . ppf ( 1 - level / 2 ) * df [ "data_value_stderr" ]
-ax . errorbar ( x = df . index , y = df [ "data_value" ], yerr = yerr , fmt = "o" , capsize = 6 )
-ax . set_xlabel ( xlabel )
-ax . set_ylabel ( ylabel )
-ax . set_title ( title )
-plt . xticks ( rotation = 60 )
-return ax
+ def plot_shapley (
+df : pd . DataFrame ,
+* ,
+level : float = 0.05 ,
+ax : Optional [ plt . Axes ] = None ,
+title : Optional [ str ] = None ,
+xlabel : Optional [ str ] = None ,
+ylabel : Optional [ str ] = None ,
+) -> plt . Axes :
+ r """Plots the shapley values, as returned from
+ [compute_shapley_values][pydvl.value.shapley.common.compute_shapley_values],
+ with error bars corresponding to an $\alpha$-level Normal confidence
+ interval.
+ Args:
+ df: dataframe with the shapley values
+ level: confidence level for the error bars
+ ax: axes to plot on or None if a new subplots should be created
+ title: string, title of the plot
+ xlabel: string, x label of the plot
+ ylabel: string, y label of the plot
+ Returns:
+ The axes created or used
+ """
+if ax is None :
+_ , ax = plt . subplots ()
+yerr = norm . ppf ( 1 - level / 2 ) * df [ "data_value_stderr" ]
+ax . errorbar ( x = df . index , y = df [ "data_value" ], yerr = yerr , fmt = "o" , capsize = 6 )
+ax . set_xlabel ( xlabel )
+ax . set_ylabel ( ylabel )
+ax . set_title ( title )
+plt . xticks ( rotation = 60 )
+return ax
Source code in src/pydvl/reporting/plots.py
- def plot_influence_distribution_by_label (
-influences : NDArray [ np . float_ ], labels : NDArray [ np . float_ ], title_extra : str = ""
-):
- """Plots the histogram of the influence that all samples in the training set
- have over a single sample index, separated by labels.
- Args:
- influences: array of influences (training samples x test samples)
- labels: labels for the training set.
- title_extra:
- """
-_ , ax = plt . subplots ()
-unique_labels = np . unique ( labels )
-for label in unique_labels :
-ax . hist ( influences [ labels == label ], label = label , alpha = 0.7 )
-ax . set_xlabel ( "Influence values" )
-ax . set_ylabel ( "Number of samples" )
-ax . set_title ( f "Distribution of influences " + title_extra )
-ax . legend ()
-plt . show ()
+ def plot_influence_distribution_by_label (
+influences : NDArray [ np . float_ ], labels : NDArray [ np . float_ ], title_extra : str = ""
+):
+ """Plots the histogram of the influence that all samples in the training set
+ have over a single sample index, separated by labels.
+ Args:
+ influences: array of influences (training samples x test samples)
+ labels: labels for the training set.
+ title_extra:
+ """
+_ , ax = plt . subplots ()
+unique_labels = np . unique ( labels )
+for label in unique_labels :
+ax . hist ( influences [ labels == label ], label = label , alpha = 0.7 )
+ax . set_xlabel ( "Influence values" )
+ax . set_ylabel ( "Number of samples" )
+ax . set_title ( f "Distribution of influences " + title_extra )
+ax . legend ()
+plt . show ()
+ loss
+
+
A function taking as parameters model prediction and corresponding
+data labels(preds, y) and returning an array of point-wise errors.
+
+
+ TYPE:
+ Callable
+
+ DEFAULT:
+ None
+
+
+
+
n_jobs
@@ -2712,16 +2740,16 @@
- lossseed
-
A function taking as parameters model prediction and corresponding
-data labels(preds, y) and returning an array of point-wise errors.
+
Either an instance of a numpy random number generator or a seed
+for it.
TYPE:
- Callable Optional [Seed ]
DEFAULT:
@@ -2859,88 +2887,132 @@
def compute_data_oob (
u : Utility ,
-n_est : int = 10 ,
-max_samples : float = 0.8 ,
-n_jobs : int = None ,
+* ,
+n_est : int = 10 ,
+max_samples : float = 0.8 ,
loss : Callable = None ,
-* ,
-progress : bool = False ,
-) -> ValuationResult :
- r """Computes Data out of bag values
- This implements the method described in (Kwon and Zou, 2023) <sup><a href="kwon_data_2023">1</a></sup>.
- It fits several base estimators provided through u.model through a bagging process. The point value corresponds to the average loss of estimators which were not fit on it.
- $w_{bj}\in Z$ is the number of times the j-th datum $(x_j, y_j)$ is selected in the b-th bootstrap dataset.
- $$\psi((x_i,y_i),\Theta_B):=\frac{\sum_{b=1}^{B}\mathbb{1}(w_{bi}=0)T(y_i, \hat{f}_b(x_i))}{\sum_{b=1}^{B}
- \mathbb{1}
- (w_{bi}=0)}$$
- With:
- $$
- T: Y \times Y
- \rightarrow \mathbb{R}
- $$
- T is a score function that represents the goodness of a weak learner $\hat{f}_b$ at the i-th datum $(x_i, y_i)$.
- There is a need to tune n_est and max_samples jointly to ensure all samples are at least 1 time oob, otherwise the result could include a nan value for that datum.
- Args:
- u: Utility object with model, data, and scoring function.
- n_est: Number of estimator used in the bagging procedure.
- max_samples: The fraction of samples to draw to train each base estimator.
- n_jobs: The number of jobs to run in parallel used in the bagging
- procedure for both fit and predict.
- loss: A function taking as parameters model prediction and corresponding
- data labels(preds, y) and returning an array of point-wise errors.
- progress: If True, display a progress bar.
- Returns:
- Object with the data values.
- """
-result : ValuationResult [ np . int_ , np . object_ ] = ValuationResult . empty (
-algorithm = "data_oob" , indices = u . data . indices , data_names = u . data . data_names
-)
-if is_classifier ( u . model ):
-bag = BaggingClassifier (
-u . model , n_estimators = n_est , max_samples = max_samples , n_jobs = n_jobs
-)
-if loss is None :
-loss = point_wise_accuracy
-elif is_regressor ( u . model ):
-bag = BaggingRegressor (
-u . model , n_estimators = n_est , max_samples = max_samples , n_jobs = n_jobs
-)
-if loss is None :
-loss = neg_l2_distance
-else :
-raise Exception (
-"Model has to be a classifier or a regressor in sklearn format."
-)
-bag . fit ( u . data . x_train , u . data . y_train )
-for est , samples in maybe_progress (
-zip ( bag . estimators_ , bag . estimators_samples_ ), progress , total = n_est
-): # The bottleneck is the bag fitting not this part so TQDM is not very useful here
-oob_idx = np . setxor1d ( u . data . indices , np . unique ( samples ))
-array_loss = loss (
-preds = est . predict ( u . data . x_train [ oob_idx ]), y = u . data . y_train [ oob_idx ]
-)
-result += ValuationResult (
-algorithm = "data_oob" ,
-indices = oob_idx ,
-values = array_loss ,
-counts = np . ones_like ( array_loss , dtype = u . data . indices . dtype ),
+n_jobs : int = None ,
+seed : Optional [ Seed ] = None ,
+progress : bool = False ,
+) -> ValuationResult :
+ r """Computes Data out of bag values
+ This implements the method described in
+ (Kwon and Zou, 2023)<sup><a href="kwon_data_2023">1</a></sup>.
+ It fits several base estimators provided through u.model through a bagging
+ process. The point value corresponds to the average loss of estimators which
+ were not fit on it.
+ $w_{bj}\in Z$ is the number of times the j-th datum $(x_j, y_j)$ is selected
+ in the b-th bootstrap dataset.
+ $$\psi((x_i,y_i),\Theta_B):=\frac{\sum_{b=1}^{B}\mathbb{1}(w_{bi}=0)T(y_i,
+ \hat{f}_b(x_i))}{\sum_{b=1}^{B}
+ \mathbb{1}
+ (w_{bi}=0)}$$
+ With:
+ $$
+ T: Y \times Y
+ \rightarrow \mathbb{R}
+ $$
+ T is a score function that represents the goodness of a weak learner
+ $\hat{f}_b$ at the i-th datum $(x_i, y_i)$.
+ `n_est` and `max_samples` must be tuned jointly to ensure that all samples
+ are at least 1 time out-of-bag, otherwise the result could include a NaN
+ value for that datum.
+ Args:
+ u: Utility object with model, data, and scoring function.
+ n_est: Number of estimator used in the bagging procedure.
+ max_samples: The fraction of samples to draw to train each base
+ estimator.
+ loss: A function taking as parameters model prediction and corresponding
+ data labels(preds, y) and returning an array of point-wise errors.
+ n_jobs: The number of jobs to run in parallel used in the bagging
+ procedure for both fit and predict.
+ seed: Either an instance of a numpy random number generator or a seed
+ for it.
+ progress: If True, display a progress bar.
+ Returns:
+ Object with the data values.
+ """
+rng = np . random . default_rng ( seed )
+random_state = np . random . RandomState ( rng . bit_generator )
+result : ValuationResult [ np . int_ , np . object_ ] = ValuationResult . empty (
+algorithm = "data_oob" , indices = u . data . indices , data_names = u . data . data_names
+)
+if is_classifier ( u . model ):
+bag = BaggingClassifier (
+u . model ,
+n_estimators = n_est ,
+max_samples = max_samples ,
+n_jobs = n_jobs ,
+random_state = random_state ,
+)
+if loss is None :
+loss = point_wise_accuracy
+elif is_regressor ( u . model ):
+bag = BaggingRegressor (
+u . model ,
+n_estimators = n_est ,
+max_samples = max_samples ,
+n_jobs = n_jobs ,
+random_state = random_state ,
)
-return result
+if loss is None :
+loss = neg_l2_distance
+else :
+raise Exception (
+"Model has to be a classifier or a regressor in sklearn format."
+)
+bag . fit ( u . data . x_train , u . data . y_train )
+for est , samples in maybe_progress (
+zip ( bag . estimators_ , bag . estimators_samples_ ), progress , total = n_est
+): # The bottleneck is the bag fitting not this part so TQDM is not very useful here
+oob_idx = np . setxor1d ( u . data . indices , np . unique ( samples ))
+array_loss = loss (
+y_true = u . data . y_train [ oob_idx ], y_pred = est . predict ( u . data . x_train [ oob_idx ])
+)
+result += ValuationResult (
+algorithm = "data_oob" ,
+indices = oob_idx ,
+values = array_loss ,
+counts = np . ones_like ( array_loss , dtype = u . data . indices . dtype ),
+)
+return result
- point_wise_accuracy ( preds , y ) point_wise_accuracy ( y_true , y_pred )
-
Computes point wise accuracy
+
Point-wise 0-1 loss between two arrays
@@ -2974,29 +3046,29 @@
- predsy_true
-
Model prediction on
+
Array of true values (e.g. labels)
TYPE:
- NDArray NDArray [T ]
- yy_pred
-
data labels corresponding to the model predictions
+
Array of estimated values (e.g. model predictions)
TYPE:
- NDArray NDArray [T ]
@@ -3017,12 +3089,12 @@
- NDArray NDArray [T ]
-
Array of point wise accuracy
+
Array with point-wise 0-1 losses between labels and model predictions
@@ -3031,27 +3103,27 @@
Source code in src/pydvl/value/oob/oob.py
- def point_wise_accuracy ( preds : NDArray , y : NDArray ) -> NDArray :
- r """Computes point wise accuracy
- Args:
- preds: Model prediction on
- y: data labels corresponding to the model predictions
- Returns:
- Array of point wise accuracy
- """
-return np . array ( preds == y , dtype = np . int_ )
+ def point_wise_accuracy ( y_true : NDArray [ T ], y_pred : NDArray [ T ]) -> NDArray [ T ]:
+ r """Point-wise 0-1 loss between two arrays
+ Args:
+ y_true: Array of true values (e.g. labels)
+ y_pred: Array of estimated values (e.g. model predictions)
+ Returns:
+ Array with point-wise 0-1 losses between labels and model predictions
+ """
+return np . array ( y_pred == y_true , dtype = y_pred . dtype )
- neg_l2_distance ( preds , y ) neg_l2_distance ( y_true , y_pred )
-
Computes negative l2 distance between label and model prediction
+
Point-wise negative \(l_2\) distance between two arrays
@@ -3085,10 +3157,10 @@
- predsy_true
-
Model prediction on
+
Array of true values (e.g. labels)
@@ -3099,10 +3171,10 @@
- yy_pred
-
data labels corresponding to the model predictions
+
Array of estimated values (e.g. model predictions)
@@ -3133,7 +3205,19 @@
-
Array with point wise negative l2 distance between label and model prediction
+
Array with point-wise negative \(l_2\) distances between labels and model
+
+
+
+
+
+ NDArray [T ]
+
+
+
@@ -3142,37 +3226,29 @@
Source code in src/pydvl/value/oob/oob.py
- def neg_l2_distance ( preds : NDArray [ T ], y : NDArray [ T ]) -> NDArray [ T ]:
- r """Computes negative l2 distance between label and model prediction
- Args:
- preds: Model prediction on
- y: data labels corresponding to the model predictions
- Returns:
- Array with point wise negative l2 distance between label and model prediction
- """
-return - np . square (
-np . array (
-preds - y ,
-dtype = np . float64 ,
-)
-)
+ def neg_l2_distance ( y_true : NDArray [ T ], y_pred : NDArray [ T ]) -> NDArray [ T ]:
+ r """Point-wise negative $l_2$ distance between two arrays
+ Args:
+ y_true: Array of true values (e.g. labels)
+ y_pred: Array of estimated values (e.g. model predictions)
+ Returns:
+ Array with point-wise negative $l_2$ distances between labels and model
+ predictions
+ """
+return - np . square ( np . array ( y_pred - y_true ), dtype = y_pred . dtype )
Last update:
- 2023-09-18
+ 2023-09-20
2023-09-18
+ 2023-09-20
diff --git a/devel/api/pydvl/value/result/index.html b/devel/api/pydvl/value/result/index.html
index 68f94256a..61323bd81 100644
--- a/devel/api/pydvl/value/result/index.html
+++ b/devel/api/pydvl/value/result/index.html
@@ -18,7 +18,7 @@
Last update:
- 2023-09-18
+ 2023-09-20
2023-09-18
+ 2023-09-20
diff --git a/devel/api/pydvl/value/sampler/index.html b/devel/api/pydvl/value/sampler/index.html
index 79b936c36..69e7ef875 100644
--- a/devel/api/pydvl/value/sampler/index.html
+++ b/devel/api/pydvl/value/sampler/index.html
@@ -18,7 +18,7 @@
Last update:
- 2023-09-18
+ 2023-09-20
2023-09-18
+ 2023-09-20
diff --git a/devel/api/pydvl/value/semivalues/index.html b/devel/api/pydvl/value/semivalues/index.html
index b4f8f4434..b32eb99c9 100644
--- a/devel/api/pydvl/value/semivalues/index.html
+++ b/devel/api/pydvl/value/semivalues/index.html
@@ -18,7 +18,7 @@
"""
from concurrent.futures import FIRST_COMPLETED , Future , wait
from pydvl.utils import effective_n_jobs , init_executor , init_parallel_backend
+from pydvl.parallel import effective_n_jobs , init_executor , init_parallel_backend
if isinstance ( sampler , PermutationSampler ) and not u . enable_cache :
log . warning (
@@ -4791,11 +4791,11 @@
Last update:
- 2023-09-18
+ 2023-09-20
2023-09-18
+ 2023-09-20
diff --git a/devel/api/pydvl/value/shapley/common/index.html b/devel/api/pydvl/value/shapley/common/index.html
index 05389805b..5357fbdaf 100644
--- a/devel/api/pydvl/value/shapley/common/index.html
+++ b/devel/api/pydvl/value/shapley/common/index.html
@@ -18,7 +18,7 @@
2023-09-18 2023-09-20
2023-09-18
+ 2023-09-20
diff --git a/devel/api/pydvl/value/shapley/gt/index.html b/devel/api/pydvl/value/shapley/gt/index.html
index 67513714d..8d2f98148 100644
--- a/devel/api/pydvl/value/shapley/gt/index.html
+++ b/devel/api/pydvl/value/shapley/gt/index.html
@@ -18,7 +18,7 @@
Last update:
- 2023-09-18
+ 2023-09-20
2023-09-18
+ 2023-09-20
diff --git a/devel/api/pydvl/value/shapley/index.html b/devel/api/pydvl/value/shapley/index.html
index 284d0e2c6..14db5c65f 100644
--- a/devel/api/pydvl/value/shapley/index.html
+++ b/devel/api/pydvl/value/shapley/index.html
@@ -18,7 +18,7 @@
Shapley
Last update:
- 2023-09-18
+ 2023-09-20
2023-09-18
+ 2023-09-20
diff --git a/devel/api/pydvl/value/shapley/knn/index.html b/devel/api/pydvl/value/shapley/knn/index.html
index d16a152f5..299b61aa7 100644
--- a/devel/api/pydvl/value/shapley/knn/index.html
+++ b/devel/api/pydvl/value/shapley/knn/index.html
@@ -18,7 +18,7 @@
Last update:
- 2023-09-18
+ 2023-09-20
2023-09-18
+ 2023-09-20
diff --git a/devel/api/pydvl/value/shapley/montecarlo/index.html b/devel/api/pydvl/value/shapley/montecarlo/index.html
index d32b4a68d..40732e2da 100644
--- a/devel/api/pydvl/value/shapley/montecarlo/index.html
+++ b/devel/api/pydvl/value/shapley/montecarlo/index.html
@@ -18,7 +18,7 @@
2023-09-18
+ 2023-09-20
2023-09-18
+ 2023-09-20
diff --git a/devel/api/pydvl/value/shapley/naive/index.html b/devel/api/pydvl/value/shapley/naive/index.html
index c093aa14b..0aa094f54 100644
--- a/devel/api/pydvl/value/shapley/naive/index.html
+++ b/devel/api/pydvl/value/shapley/naive/index.html
@@ -18,7 +18,7 @@
2023-09-18
+ 2023-09-20
2023-09-18
+ 2023-09-20
diff --git a/devel/api/pydvl/value/shapley/owen/index.html b/devel/api/pydvl/value/shapley/owen/index.html
index 982b8b4b5..893fb806a 100644
--- a/devel/api/pydvl/value/shapley/owen/index.html
+++ b/devel/api/pydvl/value/shapley/owen/index.html
@@ -18,7 +18,7 @@
Last update:
- 2023-09-18
+ 2023-09-20
2023-09-18
+ 2023-09-20
diff --git a/devel/api/pydvl/value/shapley/truncated/index.html b/devel/api/pydvl/value/shapley/truncated/index.html
index b58758a6c..8c226610f 100644
--- a/devel/api/pydvl/value/shapley/truncated/index.html
+++ b/devel/api/pydvl/value/shapley/truncated/index.html
@@ -18,7 +18,7 @@
2023-09-18
+ 2023-09-20
2023-09-18
+ 2023-09-20
diff --git a/devel/api/pydvl/value/shapley/types/index.html b/devel/api/pydvl/value/shapley/types/index.html
index aec9c2bda..832374107 100644
--- a/devel/api/pydvl/value/shapley/types/index.html
+++ b/devel/api/pydvl/value/shapley/types/index.html
@@ -16,7 +16,7 @@
Last update:
- 2023-09-18
+ 2023-09-20
2023-09-18
+ 2023-09-20
diff --git a/devel/api/pydvl/value/stopping/index.html b/devel/api/pydvl/value/stopping/index.html
index 3225d6e2b..f79a5ee63 100644
--- a/devel/api/pydvl/value/stopping/index.html
+++ b/devel/api/pydvl/value/stopping/index.html
@@ -18,7 +18,7 @@
Last update:
- 2023-09-18
+ 2023-09-20
2023-09-18
+ 2023-09-20
diff --git a/devel/assets/stylesheets/main.046329b4.min.css b/devel/assets/stylesheets/main.046329b4.min.css
deleted file mode 100644
index 77ac5aae8..000000000
--- a/devel/assets/stylesheets/main.046329b4.min.css
+++ /dev/null
@@ -1 +0,0 @@
-@charset "UTF-8";html{-webkit-text-size-adjust:none;-moz-text-size-adjust:none;text-size-adjust:none;box-sizing:border-box}*,:after,:before{box-sizing:inherit}@media (prefers-reduced-motion){*,:after,:before{transition:none!important}}body{margin:0}a,button,input,label{-webkit-tap-highlight-color:transparent}a{color:inherit;text-decoration:none}hr{border:0;box-sizing:initial;display:block;height:.05rem;overflow:visible;padding:0}small{font-size:80%}sub,sup{line-height:1em}img{border-style:none}table{border-collapse:initial;border-spacing:0}td,th{font-weight:400;vertical-align:top}button{background:#0000;border:0;font-family:inherit;font-size:inherit;margin:0;padding:0}input{border:0;outline:none}:root{--md-primary-fg-color:#4051b5;--md-primary-fg-color--light:#5d6cc0;--md-primary-fg-color--dark:#303fa1;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3;--md-accent-fg-color:#526cfe;--md-accent-fg-color--transparent:#526cfe1a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-scheme=default]{color-scheme:light}[data-md-color-scheme=default] img[src$="#gh-dark-mode-only"],[data-md-color-scheme=default] img[src$="#only-dark"]{display:none}:root,[data-md-color-scheme=default]{--md-default-fg-color:#000000de;--md-default-fg-color--light:#0000008a;--md-default-fg-color--lighter:#00000052;--md-default-fg-color--lightest:#00000012;--md-default-bg-color:#fff;--md-default-bg-color--light:#ffffffb3;--md-default-bg-color--lighter:#ffffff4d;--md-default-bg-color--lightest:#ffffff1f;--md-code-fg-color:#36464e;--md-code-bg-color:#f5f5f5;--md-code-hl-color:#ffff0080;--md-code-hl-number-color:#d52a2a;--md-code-hl-special-color:#db1457;--md-code-hl-function-color:#a846b9;--md-code-hl-constant-color:#6e59d9;--md-code-hl-keyword-color:#3f6ec6;--md-code-hl-string-color:#1c7d4d;--md-code-hl-name-color:var(--md-code-fg-color);--md-code-hl-operator-color:var(--md-default-fg-color--light);--md-code-hl-punctuation-color:var(--md-default-fg-color--light);--md-code-hl-comment-color:var(--md-default-fg-color--light);--md-code-hl-generic-color:var(--md-default-fg-color--light);--md-code-hl-variable-color:var(--md-default-fg-color--light);--md-typeset-color:var(--md-default-fg-color);--md-typeset-a-color:var(--md-primary-fg-color);--md-typeset-mark-color:#ffff0080;--md-typeset-del-color:#f5503d26;--md-typeset-ins-color:#0bd57026;--md-typeset-kbd-color:#fafafa;--md-typeset-kbd-accent-color:#fff;--md-typeset-kbd-border-color:#b8b8b8;--md-typeset-table-color:#0000001f;--md-typeset-table-color--light:rgba(0,0,0,.035);--md-admonition-fg-color:var(--md-default-fg-color);--md-admonition-bg-color:var(--md-default-bg-color);--md-warning-fg-color:#000000de;--md-warning-bg-color:#ff9;--md-footer-fg-color:#fff;--md-footer-fg-color--light:#ffffffb3;--md-footer-fg-color--lighter:#ffffff73;--md-footer-bg-color:#000000de;--md-footer-bg-color--dark:#00000052;--md-shadow-z1:0 0.2rem 0.5rem #0000000d,0 0 0.05rem #0000001a;--md-shadow-z2:0 0.2rem 0.5rem #0000001a,0 0 0.05rem #00000040;--md-shadow-z3:0 0.2rem 0.5rem #0003,0 0 0.05rem #00000059}.md-icon svg{fill:currentcolor;display:block;height:1.2rem;width:1.2rem}body{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;--md-text-font-family:var(--md-text-font,_),-apple-system,BlinkMacSystemFont,Helvetica,Arial,sans-serif;--md-code-font-family:var(--md-code-font,_),SFMono-Regular,Consolas,Menlo,monospace}aside,body,input{font-feature-settings:"kern","liga";color:var(--md-typeset-color);font-family:var(--md-text-font-family)}code,kbd,pre{font-feature-settings:"kern";font-family:var(--md-code-font-family)}:root{--md-typeset-table-sort-icon:url('data:image/svg+xml;charset=utf-8,
-
- Variance
+
+ Computing the OOB values
+
+
+
+
+
+
+ Point removal experiments
@@ -2494,8 +2501,15 @@
-
- Variance
+
+ Computing the OOB values
+
+
+
+
+
+
+ Point removal experiments
@@ -2586,32 +2600,6 @@ Setup
+
+
+
+