From 3e69df0bf07f6c4f0b584063317f2da0403b7092 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Sat, 12 Aug 2023 23:21:28 +0200 Subject: [PATCH 01/43] Implement algorithm from paper `CS-Shapley: Class-wise Shapley Values for Data Valuation in Classification` (https://arxiv.org/abs/2211.06800) --- CHANGELOG.md | 2 + docs/30-data-valuation.rst | 46 ++ src/pydvl/utils/config.py | 1 + src/pydvl/utils/dataset.py | 9 +- src/pydvl/utils/numeric.py | 84 ++- src/pydvl/utils/score.py | 151 ++++- src/pydvl/utils/util.py | 14 + src/pydvl/value/result.py | 26 +- src/pydvl/value/shapley/__init__.py | 1 + src/pydvl/value/shapley/classwise.py | 251 ++++++++ src/pydvl/value/shapley/montecarlo.py | 186 +++++- src/pydvl/value/shapley/truncated.py | 22 +- src/pydvl/value/stopping.py | 10 +- tests/conftest.py | 24 +- tests/misc.py | 36 ++ tests/utils/conftest.py | 22 + tests/utils/test_numeric.py | 27 + tests/utils/test_score.py | 126 ++++- tests/value/shapley/test_classwise.py | 786 ++++++++++++++++++++++++++ 19 files changed, 1779 insertions(+), 45 deletions(-) create mode 100644 src/pydvl/utils/util.py create mode 100644 src/pydvl/value/shapley/classwise.py create mode 100644 tests/misc.py create mode 100644 tests/value/shapley/test_classwise.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 14aca878a..f625f407c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ [PR #382](https://github.com/appliedAI-Initiative/pyDVL/pull/382) - Decouple ray.init from ParallelConfig [PR #373](https://github.com/appliedAI-Initiative/pyDVL/pull/383) +- **New Method**: Add classwise Shapley algorithm. + [PR #338](https://github.com/appliedAI-Initiative/pyDVL/pull/338) ## 0.6.1 - 🏗 Bug fixes and small improvement diff --git a/docs/30-data-valuation.rst b/docs/30-data-valuation.rst index b3fd5018e..565f17906 100644 --- a/docs/30-data-valuation.rst +++ b/docs/30-data-valuation.rst @@ -359,6 +359,52 @@ useful in applications. u=utility, mode="truncated_montecarlo", done=MaxUpdates(1000) ) +Classwise Shapley +^^^^^^^^^^^^^^^^^^ + +A different schema applicable for classification problems first appeared in +:footcite:t:`schoch_csshapley_2022`. The key insight is that samples can be beneficial +for overall performance, while being detrimental for their own class. This could be an +indication of some problem with the data. CS-Shapley changes the utility to account for +this effect by decomposing it into a product of two functions: one gives +priority to in-class accuracy, while the other adds a slight discount which +increases as the out-of-class accuracy increases. + +The value is computed as: + +$$ +v_u(x_i) \approx \frac{1}{K \cdot L} +\sum_{S^{(k)}_{-y_i} \subseteq T_{-y_i} \setminus \{i\}} +\sum_{\sigma^{(l)} \in \Pi(T_{y_i} \setminus \{i\})} +[u( \sigma_{\colon i} \cup \{i\} | S_{-y_i} ) +− u( \sigma_{\colon i} | S_{-y_i})] +$$ + +where $K$ is the number of subsets $S^{(k)}_{-y_i}$ sampled from the class complement +set $T_{-y_i}$ of class c and $L$ is the number of permutations sampled from the class +indices set $T_{y_i}$. The scoring function used has the form + +$$u(S_{y_i}|S_{-y_i}) = a_S(D_{y_i}))) \exp\{a_S(D_{-y_i}))\}.$$ + +This can be further customised, but that form is shown by the authors to have certain +desirable properties. + +.. code-block:: python + + from pydvl.utils import Dataset, Utility + from pydvl.value import compute_shapley_values + + model = ... + scoring = ClassWiseScorer("accuracy") + data = Dataset(...) + utility = Utility(model, data, scoring) + values = classwise_shapley( + utility, + done=HistoryDeviation(n_steps=500, rtol=1e-3), + n_resample_complement_sets=10, + normalize_values=True + ) + Exact Shapley for KNN ^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/pydvl/utils/config.py b/src/pydvl/utils/config.py index 36b9ab647..675c1df02 100644 --- a/src/pydvl/utils/config.py +++ b/src/pydvl/utils/config.py @@ -25,6 +25,7 @@ class ParallelConfig: address: Optional[Union[str, Tuple[str, int]]] = None n_cpus_local: Optional[int] = None logging_level: int = logging.WARNING + _temp_dir: Optional[str] = None def __post_init__(self) -> None: if self.address is not None and self.n_cpus_local is not None: diff --git a/src/pydvl/utils/dataset.py b/src/pydvl/utils/dataset.py index 980957cbc..cab59416b 100644 --- a/src/pydvl/utils/dataset.py +++ b/src/pydvl/utils/dataset.py @@ -222,6 +222,10 @@ def indices(self): """ return self._indices + @indices.setter + def indices(self, indices: np.ndarray): + self._indices = indices + @property def data_names(self): """Names of each individual datapoint. @@ -410,11 +414,6 @@ def __init__( def __len__(self): return len(self.groups) - @property - def indices(self): - """Indices of the groups.""" - return self._indices - # FIXME this is a misnomer, should be `names` in `Dataset` so that here it # makes sense @property diff --git a/src/pydvl/utils/numeric.py b/src/pydvl/utils/numeric.py index c639da82b..5e5904c56 100644 --- a/src/pydvl/utils/numeric.py +++ b/src/pydvl/utils/numeric.py @@ -4,8 +4,22 @@ """ from __future__ import annotations +import logging +import os +import random +import time from itertools import chain, combinations -from typing import Collection, Generator, Iterator, Optional, Tuple, TypeVar, overload +from typing import ( + Collection, + Generator, + Iterator, + List, + Optional, + Tuple, + TypeVar, + cast, + overload, +) import numpy as np from numpy.typing import NDArray @@ -17,10 +31,15 @@ "random_matrix_with_condition_number", "random_subset", "random_powerset", + "random_powerset_group_conditional", "random_subset_of_size", "top_k_value_accuracy", ] + +logger = logging.getLogger(__name__) + + T = TypeVar("T", bound=np.generic) @@ -110,6 +129,69 @@ def random_powerset( total += 1 +def random_powerset_group_conditional( + s: NDArray[T], + groups: NDArray[np.int_], + min_elements_per_group: int = 1, +) -> Generator[NDArray[T], None, None]: + """ + Draw infinite random group-conditional subsets from the passed set s. It is ensured + that in each sampled set, each unique group is represented at least ``min_elements`` + times. The groups are specified as integers for all elements of the set separately. + + :param s: Vector of size N representing the set to sample elements from. + :param groups: Vector of size N containing the group as an integer for each element. + :param min_elements_per_group: The minimum number of elements for each group. + + :return: Generated draw from the power set of s with ``min_elements`` of each group. + :raises: TypeError: If the data ``s`` or ``groups`` is not a NumPy array. + :raises: ValueError: If the length of ``s``and ``groups`` different or + ``min_elements`` is smaller than 0. + """ + if not isinstance(s, np.ndarray): + raise TypeError("Set must be an NDArray") + + if not isinstance(groups, np.ndarray): + raise TypeError("Labels must be an NDArray") + + if len(groups) != len(s): + raise ValueError("Set and labels have to be of same size.") + + if min_elements_per_group < 0: + raise ValueError( + f"Parameter min_elements={min_elements_per_group} needs to be bigger or equal to 0." + ) + + if min_elements_per_group == 0: + logger.warning( + "It is recommended to ensure at least one element of each group is" + " contained in the sampled and yielded set." + ) + + rng = np.random.default_rng() + unique_labels = np.unique(groups) + + while True: + subsets: List[NDArray[T]] = [] + for label in unique_labels: + label_indices = np.asarray(np.where(groups == label)[0]) + subset_length = int( + rng.integers( + min(min_elements_per_group, len(label_indices)), + len(label_indices) + 1, + ) + ) + if subset_length > 0: + subsets.append(random_subset_of_size(s[label_indices], subset_length)) + + if len(subsets) > 0: + subset = np.concatenate(tuple(subsets)) + rng.shuffle(subset) + yield subset + else: + yield np.array([]) + + def random_subset_of_size(s: NDArray[T], size: int) -> NDArray[T]: """Samples a random subset of given size uniformly from the powerset of ``s``. diff --git a/src/pydvl/utils/score.py b/src/pydvl/utils/score.py index 933706d98..ca02b1ccb 100644 --- a/src/pydvl/utils/score.py +++ b/src/pydvl/utils/score.py @@ -2,7 +2,7 @@ This module provides a :class:`Scorer` class that wraps scoring functions with additional information. -Scorers can be constructed in the same way as in scikit-learn: either from +Scorers can be constructed in the same way as in scikit-learn: either from known strings or from a callable. Greater values must be better. If they are not, a negated version can be used, see scikit-learn's `make_scorer() `_. @@ -17,11 +17,17 @@ import numpy as np from numpy.typing import NDArray from scipy.special import expit -from sklearn.metrics import get_scorer +from sklearn.metrics import accuracy_score, get_scorer, make_scorer from pydvl.utils.types import SupervisedModel -__all__ = ["Scorer", "compose_score", "squashed_r2", "squashed_variance"] +__all__ = [ + "Scorer", + "ClasswiseScorer", + "compose_score", + "squashed_r2", + "squashed_variance", +] class ScorerCallable(Protocol): @@ -58,7 +64,7 @@ class Scorer: def __init__( self, scoring: Union[str, ScorerCallable], - default: float = np.nan, + default: float = 0.0, range: Tuple = (-np.inf, np.inf), name: Optional[str] = None, ): @@ -81,6 +87,143 @@ def __repr__(self): return f"{capitalized_name} (scorer={self._scorer})" +class ClasswiseScorer(Scorer): + """A Scorer which is applicable for valuation in classification problems. Its value + is based on in-cls and out-of-cls score :footcite:t:`schoch_csshapley_2022`. For + each class ``label`` it separates the elements into two groups, namely in-cls + instances and out-of-cls instances. The value function itself than estimates the + in-cls metric discounted by the out-of-cls metric. In other words the value function + for each element of one class is conditioned on the out-of-cls instances (or a + subset of it). The form of the value function can be written as + + .. math:: + v_{y_i}(D) = f(a_S(D_{y_i}))) * g(a_S(D_{-y_i}))) + + where f and g are continuous, monotonic functions and D is the test set. + + in order to produce meaningful results. For further reference see also section four + of :footcite:t:`schoch_csshapley_2022`. + + :param default: Score used when a model cannot be fit, e.g. when too little data is + passed, or errors arise. + :param range: Numerical range of the score function. Some Monte Carlo methods can + use this to estimate the number of samples required for a certain quality of + approximation. If not provided, it can be read from the ``scoring`` object if it + provides it, for instance if it was constructed with + :func:`~pydvl.utils.types.compose_score`. + :param in_class_discount_fn: Continuous, monotonic increasing function used to + discount the in-class score. + :param out_of_class_discount_fn: Continuous, monotonic increasing function used to + discount the out-of-class score. + :param initial_label: Set initial label (Doesn't require to set parameter ``label`` + on ``ClassWiseDiscountedScorer`` in first iteration) + :param name: Name of the scorer. If not provided, the name of the passed + function will be prefixed by 'classwise '. + + .. versionadded:: 0.7.0 + """ + + def __init__( + self, + scoring: str = "accuracy", + default: float = 0.0, + range: Tuple[float, float] = (-np.inf, np.inf), + in_class_discount_fn: Callable[[float], float] = lambda x: x, + out_of_class_discount_fn: Callable[[float], float] = np.exp, + initial_label: Optional[int] = None, + name: Optional[str] = None, + ): + disc_score_in_cls = in_class_discount_fn(range[1]) + disc_score_out_of_cls = out_of_class_discount_fn(range[1]) + transformed_range = (0, disc_score_in_cls * disc_score_out_of_cls) + super().__init__( + "accuracy", + range=transformed_range, + default=default, + name=name or f"classwise {scoring}", + ) + self._in_cls_discount_fn = in_class_discount_fn + self._out_of_cls_discount_fn = out_of_class_discount_fn + self.label = initial_label + + def __str__(self): + return self._name + + def __call__( + self: "ClasswiseScorer", + model: SupervisedModel, + x_test: NDArray[np.float_], + y_test: NDArray[np.int_], + ) -> float: + """ + :param model: Model used for computing the score on the validation set. + :param x_test: Array containing the features of the classification problem. + :param y_test: Array containing the labels of the classification problem. + :return: Calculated score. + """ + in_cls_score, out_of_cls_score = self.estimate_in_cls_and_out_of_cls_score( + model, x_test, y_test + ) + disc_score_in_cls = self._in_cls_discount_fn(in_cls_score) + disc_score_out_of_cls = self._out_of_cls_discount_fn(out_of_cls_score) + return disc_score_in_cls * disc_score_out_of_cls + + def estimate_in_cls_and_out_of_cls_score( + self, + model: SupervisedModel, + x_test: NDArray[np.float_], + y_test: NDArray[np.int_], + rescale_scores: bool = True, + ) -> Tuple[float, float]: + r""" + Computes in-class and out-of-class scores using the provided scoring function, + which can be expressed as: + + .. math:: + a_S(D=\{(\hat{x}_1, \hat{y}_1), \dots, (\hat{x}_K, \hat{y}_K)\}) &= + \frac{1}{N} \sum_k s(y(\hat{x}_k), \hat{y}_k) + + In this context, the computation is performed twice: once on D_i and once on D_o + to calculate the in-class and out-of-class scores. Here, D_i contains only + samples with the specified 'label' from the validation set, while D_o contains + all other samples. By default, the scores are scaled to have the same order of + magnitude. In such cases, the raw scores are multiplied by: + + .. math:: + N_{y_i} = \frac{a_S(D_{y_i})}{a_S(D_{y_i})+a_S(D_{-y_i})} \quad \text{and} + \quad N_{-y_i} = \frac{a_S(D_{-y_i})}{a_S(D_{y_i})+a_S(D_{-y_i})} + + :param model: Model used for computing the score on the validation set. + :param x_test: Array containing the features of the classification problem. + :param y_test: Array containing the labels of the classification problem. + :param rescale_scores: If set to True, the scores will be denormalized. This is + particularly useful when the inner score is calculated by an estimator of + the form 1/N sum_i x_i. + :return: Tuple containing the in-class and out-of-class scores. + """ + scorer = self._scorer + label_set_match = y_test == self.label + label_set = np.where(label_set_match)[0] + num_classes = len(np.unique(y_test)) + + if len(label_set) == 0: + return 0, 1 / (num_classes - 1) + + complement_label_set = np.where(~label_set_match)[0] + in_cls_score = scorer(model, x_test[label_set], y_test[label_set]) + out_of_cls_score = scorer( + model, x_test[complement_label_set], y_test[complement_label_set] + ) + + if rescale_scores: + n_in_cls = np.count_nonzero(y_test == self.label) + n_out_of_cls = len(y_test) - n_in_cls + in_cls_score *= n_in_cls / (n_in_cls + n_out_of_cls) + out_of_cls_score *= n_out_of_cls / (n_in_cls + n_out_of_cls) + + return in_cls_score, out_of_cls_score + + def compose_score( scorer: Scorer, transformation: Callable[[float], float], diff --git a/src/pydvl/utils/util.py b/src/pydvl/utils/util.py new file mode 100644 index 000000000..d556b4d28 --- /dev/null +++ b/src/pydvl/utils/util.py @@ -0,0 +1,14 @@ +import numpy as np +from numpy.typing import NDArray + + +def arr_or_writeable_copy(arr: NDArray) -> NDArray: + """Return a copy of ``arr`` if it's not writeable, otherwise return ``arr``. + + :param arr: Array to copy if it's not writeable. + :return: Copy of ``arr`` if it's not writeable, otherwise ``arr``. + """ + if not arr.flags.writeable: + return np.copy(arr) + + return arr diff --git a/src/pydvl/value/result.py b/src/pydvl/value/result.py index 219b8ea90..f66a514e7 100644 --- a/src/pydvl/value/result.py +++ b/src/pydvl/value/result.py @@ -66,6 +66,7 @@ from pydvl.utils.dataset import Dataset from pydvl.utils.numeric import running_moments from pydvl.utils.status import Status +from pydvl.utils.util import arr_or_writeable_copy try: import pandas # Try to import here for the benefit of mypy @@ -234,8 +235,12 @@ def __init__( self._algorithm = algorithm self._status = Status(status) # Just in case we are given a string - self._values = values - self._variances = np.zeros_like(values) if variances is None else variances + self._values = arr_or_writeable_copy(values) + self._variances = ( + np.zeros_like(values) + if variances is None + else arr_or_writeable_copy(variances) + ) self._counts = np.ones_like(values) if counts is None else counts self._sort_order = None self._extra_values = extra_values or {} @@ -526,10 +531,14 @@ def __add__(self, other: "ValuationResult") -> "ValuationResult": xm[other_pos] = other._values vm[other_pos] = other._variances + # np.maximum(1, n + m) covers case n = m = 0 with + n_m_sum = np.maximum(1, n + m) + # Sample mean of n+m samples from two means of n and m samples - xnm = (n * xn + m * xm) / (n + m) + xnm = (n * xn + m * xm) / n_m_sum + # Sample variance of n+m samples from two sample variances of n and m samples - vnm = (n * (vn + xn**2) + m * (vm + xm**2)) / (n + m) - xnm**2 + vnm = (n * (vn + xn**2) + m * (vm + xm**2)) / n_m_sum - xnm**2 if np.any(vnm < 0): if np.any(vnm < -1e-6): @@ -610,6 +619,15 @@ def update(self, idx: int, new_value: float) -> "ValuationResult": ) return self + def scale(self, coefficient: float, indices: Optional[NDArray[IndexT]] = None): + """ + Scales the values and variances of the result by a coefficient. + :param coefficient: Coefficient to scale by. + :param indices: Indices to scale. If None, all values are scaled. + """ + self._values[self._sort_positions[indices]] *= coefficient + self._variances[self._sort_positions[indices]] *= coefficient**2 + def get(self, idx: Integral) -> ValueItem: """Retrieves a ValueItem by data index, as opposed to sort index, like the indexing operator. diff --git a/src/pydvl/value/shapley/__init__.py b/src/pydvl/value/shapley/__init__.py index 6f93cd60e..db5802f25 100644 --- a/src/pydvl/value/shapley/__init__.py +++ b/src/pydvl/value/shapley/__init__.py @@ -8,6 +8,7 @@ from ..result import * from ..stopping import * +from .classwise import * from .common import * from .gt import * from .knn import * diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py new file mode 100644 index 000000000..9dd9f5ad6 --- /dev/null +++ b/src/pydvl/value/shapley/classwise.py @@ -0,0 +1,251 @@ +""" +Implementation of the algorithm footcite:t:`schoch_csshapley_2022`. +""" +import logging +import numbers +from concurrent.futures import FIRST_COMPLETED, wait +from copy import copy +from typing import cast + +import numpy as np + +from pydvl.utils import ( + ParallelConfig, + Utility, + effective_n_jobs, + init_executor, + init_parallel_backend, +) + +__all__ = [ + "compute_classwise_shapley_values", +] + +from tqdm import tqdm + +from pydvl.utils.score import ClasswiseScorer +from pydvl.value.result import ValuationResult +from pydvl.value.shapley.montecarlo import permutation_montecarlo_classwise_shapley +from pydvl.value.shapley.truncated import TruncationPolicy +from pydvl.value.stopping import MaxChecks, StoppingCriterion + +logger = logging.getLogger(__name__) + + +def compute_classwise_shapley_values( + u: Utility, + *, + done: StoppingCriterion, + truncation: TruncationPolicy, + normalize_values: bool = True, + n_resample_complement_sets: int = 1, + use_default_scorer_value: bool = True, + min_elements_per_label: int = 1, + n_jobs: int = 1, + config: ParallelConfig = ParallelConfig(), + progress: bool = False, +) -> ValuationResult: + """ + Computes the classwise Shapley value by parallel processing. Independent workers + are spawned to process the data in parallel. Once the data is aggregated, the values + can be optionally normalized, depending on ``normalize_values``. + + :param u: Utility object containing model, data, and scoring function. The scoring + function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + :param done: Function that checks whether the computation needs to stop. + :param truncation: Callable function that decides whether to interrupt processing a + permutation and set subsequent marginals to zero. + :param normalize_values: Indicates whether to normalize the values by the variation + in each class times their in-class accuracy. + :param n_resample_complement_sets: Number of times to resample the complement set + for each permutation. + :param use_default_scorer_value: Use default scorer value even if additional_indices + is not None. + :param min_elements_per_label: The minimum number of elements for each opposite + label. + :param n_jobs: Number of parallel jobs to run. + :param config: Parallel configuration. + :param progress: Whether to display progress bars for each job. + :return: ValuationResult object containing computed data values. + """ + + _check_classwise_shapley_utility(u) + + parallel_backend = init_parallel_backend(config) + u_ref = parallel_backend.put(u) + # This represents the number of jobs that are running + n_jobs = effective_n_jobs(n_jobs, config) + # This determines the total number of submitted jobs + # including the ones that are running + n_submitted_jobs = 2 * n_jobs + + pbar = tqdm(disable=not progress, position=0, total=100, unit="%") + accumulated_result = ValuationResult.zeros( + algorithm="classwise_shapley", + indices=u.data.indices, + data_names=u.data.data_names, + ) + terminate_exec = False + with init_executor(max_workers=n_jobs, config=config) as executor: + futures = set() + # Initial batch of computations + for _ in range(n_submitted_jobs): + future = executor.submit( + _classwise_shapley_one_step, + u_ref, + truncation=truncation, + n_resample_complement_sets=n_resample_complement_sets, + use_default_scorer_value=use_default_scorer_value, + min_elements_per_label=min_elements_per_label, + ) + futures.add(future) + while futures: + # Wait for the next futures to complete. + completed_futures, futures = wait( + futures, timeout=60, return_when=FIRST_COMPLETED + ) + for future in completed_futures: + accumulated_result += future.result() + if done(accumulated_result): + terminate_exec = True + break + + pbar.n = 100 * done.completion() + pbar.refresh() + if terminate_exec: + break + + # Submit more computations + # The goal is to always have `n_jobs` + # computations running + for _ in range(n_submitted_jobs - len(futures)): + future = executor.submit( + _classwise_shapley_one_step, + u_ref, + truncation=truncation, + n_resample_complement_sets=n_resample_complement_sets, + use_default_scorer_value=use_default_scorer_value, + min_elements_per_label=min_elements_per_label, + ) + futures.add(future) + + result = accumulated_result + if normalize_values: + result = _normalize_classwise_shapley_values(result, u) + + return result + + +def _classwise_shapley_one_step( + u: Utility, + *, + truncation: TruncationPolicy, + n_resample_complement_sets: int = 1, + use_default_scorer_value: bool = True, + min_elements_per_label: int = 1, +) -> ValuationResult: + """Computes classwise Shapley value using truncated Monte Carlo permutation + sampling for the subsets. + + :param u: Utility object containing model, data, and scoring function. The scoring + function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + :param truncation: Callable function that decides whether to interrupt processing a + permutation and set subsequent marginals to zero. + :param n_resample_complement_sets: Number of times to resample the complement set + for each permutation. + :param use_default_scorer_value: Use default scorer value even if additional_indices + is not None. + :param min_elements_per_label: The minimum number of elements for each opposite + label. + :return: ValuationResult object containing computed data values. + """ + result = ValuationResult.zeros( + algorithm="classwise_shapley", + indices=u.data.indices, + data_names=u.data.data_names, + ) + x_train, y_train = u.data.get_training_data(u.data.indices) + unique_labels = np.unique(y_train) + scorer = cast(ClasswiseScorer, copy(u.scorer)) + u.scorer = scorer + + for label in unique_labels: + u.scorer.label = label + result += permutation_montecarlo_classwise_shapley( + u, + label, + done=MaxChecks(n_resample_complement_sets - 1), + truncation=truncation, + use_default_scorer_value=use_default_scorer_value, + min_elements_per_label=min_elements_per_label, + ) + + return result + + +def _check_classwise_shapley_utility(u: Utility): + """ + Verifies if the provided utility object supports classwise Shapley values. + + :param u: Utility object containing model, data, and scoring function. The scoring + function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + :raises: ValueError: If ``u.data`` is not a classification problem. + :raises: ValueError: If ``u.scorer`` is not an instance of + :class:`~pydvl.utils.score.ClassWiseScorer` + """ + + dim_correct = u.data.y_train.ndim == 1 and u.data.y_test.ndim == 1 + is_integral = all( + map( + lambda v: isinstance(v, numbers.Integral), (*u.data.y_train, *u.data.y_test) + ) + ) + if not dim_correct or not is_integral: + raise ValueError( + "The supplied dataset has to be a 1-dimensional classification dataset." + ) + + if not isinstance(u.scorer, ClasswiseScorer): + raise ValueError( + "Please set a subclass of ClassWiseScorer object as scorer object of the" + " utility. See scoring argument of Utility." + ) + + +def _normalize_classwise_shapley_values( + result: ValuationResult, + u: Utility, +) -> ValuationResult: + """ + Normalize a valuation result specific to classwise Shapley. + + Each value corresponds to a class c and gets normalized by multiplying + `in-class-score / sigma`. In this context `sigma` is the magnitude of all values + belonging to the currently viewed class. See footcite:t:`schoch_csshapley_2022` for + more details. + + :param result: ValuationResult object to be normalized. + :param u: Utility object containing model, data, and scoring function. The scoring + function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + """ + y_train = u.data.y_train + unique_labels = np.unique(np.concatenate((y_train, u.data.y_test))) + scorer = cast(ClasswiseScorer, u.scorer) + + for idx_label, label in enumerate(unique_labels): + scorer.label = label + active_elements = y_train == label + indices_label_set = np.where(active_elements)[0] + indices_label_set = u.data.indices[indices_label_set] + + u.model.fit(u.data.x_train, u.data.y_train) + scorer.label = label + in_cls_acc, _ = scorer.estimate_in_cls_and_out_of_cls_score( + u.model, u.data.x_test, u.data.y_test + ) + + sigma = np.sum(result.values[indices_label_set]) + if sigma != 0: + result.scale(in_cls_acc / sigma, indices=indices_label_set) + + return result diff --git a/src/pydvl/value/shapley/montecarlo.py b/src/pydvl/value/shapley/montecarlo.py index ad43edad1..4ed258f15 100644 --- a/src/pydvl/value/shapley/montecarlo.py +++ b/src/pydvl/value/shapley/montecarlo.py @@ -35,12 +35,13 @@ import operator from functools import reduce from itertools import cycle, takewhile -from typing import Sequence +from typing import Optional, Sequence, Tuple import numpy as np from numpy.typing import NDArray from tqdm import tqdm +from pydvl.utils import Utility, random_powerset_group_conditional from pydvl.utils.config import ParallelConfig from pydvl.utils.numeric import random_powerset from pydvl.utils.parallel import MapReduceJob @@ -51,7 +52,11 @@ logger = logging.getLogger(__name__) -__all__ = ["permutation_montecarlo_shapley", "combinatorial_montecarlo_shapley"] +__all__ = [ + "permutation_montecarlo_shapley", + "permutation_montecarlo_classwise_shapley", + "combinatorial_montecarlo_shapley", +] def _permutation_montecarlo_shapley( @@ -87,20 +92,11 @@ def _permutation_montecarlo_shapley( while not done(result): pbar.n = 100 * done.completion() pbar.refresh() - prev_score = 0.0 permutation = np.random.permutation(u.data.indices) - permutation_done = False - truncation.reset() - for i, idx in enumerate(permutation): - if permutation_done: - score = prev_score - else: - score = u(permutation[: i + 1]) - marginal = score - prev_score - result.update(idx, marginal) - prev_score = score - if not permutation_done and truncation(i, score): - permutation_done = True + result += _permutation_montecarlo_shapley_rollout( + u, permutation, truncation=truncation, algorithm_name=algorithm_name + ) + return result @@ -152,6 +148,146 @@ def permutation_montecarlo_shapley( return map_reduce_job() +def permutation_montecarlo_classwise_shapley( + u: Utility, + label: int, + *, + done: StoppingCriterion, + truncation: TruncationPolicy, + use_default_scorer_value: bool = True, + min_elements_per_label: int = 1, +) -> ValuationResult: + """ + Samples a random subset of the complement set and computes the truncated Monte Carlo + estimator. + + :param u: Utility object containing model, data, and scoring function. The scoring + function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + :param done: Function checking whether computation needs to stop. + :param label: The label for which to sample the complement (e.g. all other labels) + :param truncation: Callable which decides whether to interrupt processing a + permutation and set all subsequent marginals to zero. + :param use_default_scorer_value: Use default scorer value even if additional_indices + is not None. + :param min_elements_per_label: The minimum number of elements for each opposite + label. + :return: ValuationResult object containing computed data values. + """ + + algorithm_name = "classwise_shapley" + result = ValuationResult.zeros( + algorithm="classwise_shapley", + indices=u.data.indices, + data_names=u.data.data_names, + ) + + _, y_train = u.data.get_training_data(u.data.indices) + class_indices_set, class_complement_indices_set = split_indices_by_label( + u.data.indices, + y_train, + label, + ) + _, complement_y_train = u.data.get_training_data(class_complement_indices_set) + indices_permutation = np.random.permutation(class_indices_set) + + for subset_idx, subset_complement in enumerate( + random_powerset_group_conditional( + class_complement_indices_set, + complement_y_train, + min_elements_per_group=min_elements_per_label, + ) + ): + result += _permutation_montecarlo_shapley_rollout( + u, + indices_permutation, + additional_indices=subset_complement, + truncation=truncation, + algorithm_name=algorithm_name, + use_default_scorer_value=use_default_scorer_value, + ) + if done(result): + break + + return result + + +def _permutation_montecarlo_shapley_rollout( + u: Utility, + permutation: NDArray[np.int_], + *, + truncation: TruncationPolicy, + algorithm_name: str, + additional_indices: Optional[NDArray[np.int_]] = None, + use_default_scorer_value: bool = True, +) -> ValuationResult: + """ + A truncated version of a permutation-based MC estimator for classwise Shapley + values. It generates a permutation p[i] of the class label indices and iterates over + all subsets starting from the empty set to the full set of indices. + + :param u: Utility object containing model, data, and scoring function. The scoring + function should to be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + :param permutation: Permutation of indices to be considered. + :param truncation: Callable which decides whether to interrupt processing a + permutation and set all subsequent marginals to zero. + :param additional_indices: Set of additional indices for data points which should be + always considered. + :param use_default_scorer_value: Use default scorer value even if additional_indices + is not None. + :return: ValuationResult object containing computed data values. + """ + if ( + additional_indices is not None + and len(np.intersect1d(permutation, additional_indices)) > 0 + ): + raise ValueError( + "The class label set and the complement set have to be disjoint." + ) + + result = ValuationResult.zeros( + algorithm=algorithm_name, + indices=u.data.indices, + data_names=u.data.data_names, + ) + + prev_score = ( + u.default_score + if ( + use_default_scorer_value + or additional_indices is None + or additional_indices is not None + and len(additional_indices) == 0 + ) + else u(additional_indices) + ) + + # hack to calculate the correct value in reset. + if additional_indices is not None: + old_indices = u.data.indices + u.data.indices = np.sort(np.concatenate((permutation, additional_indices))) + truncation.reset(u) + u.data.indices = old_indices + else: + truncation.reset(u) + + is_terminated = False + for i, idx in enumerate(permutation): + if is_terminated or (is_terminated := truncation(i, prev_score)): + score = prev_score + else: + score = u( + np.concatenate((permutation[: i + 1], additional_indices)) + if additional_indices is not None and len(additional_indices) > 0 + else permutation[: i + 1] + ) + + marginal = score - prev_score + result.update(idx, marginal) + prev_score = score + + return result + + def _combinatorial_montecarlo_shapley( indices: Sequence[int], u: Utility, @@ -246,3 +382,23 @@ def combinatorial_montecarlo_shapley( config=config, ) return map_reduce_job() + + +def split_indices_by_label( + indices: NDArray[np.int_], labels: NDArray[np.int_], label: int +) -> Tuple[NDArray[np.int_], NDArray[np.int_]]: + """ + Splits the indices into two sets based on the value of ``label``: those samples + with and without that label. + + :param indices: The indices to be used for referring to the data. + :param labels: Corresponding labels for the indices. + :param label: Label to be used for splitting. + :return: Tuple with two sets of indices. + """ + active_elements = labels == label + class_indices_set = np.where(active_elements)[0] + class_complement_indices_set = np.where(~active_elements)[0] + class_indices_set = indices[class_indices_set] + class_complement_indices_set = indices[class_complement_indices_set] + return class_indices_set, class_complement_indices_set diff --git a/src/pydvl/value/shapley/truncated.py b/src/pydvl/value/shapley/truncated.py index 23b871699..2945c95bf 100644 --- a/src/pydvl/value/shapley/truncated.py +++ b/src/pydvl/value/shapley/truncated.py @@ -1,6 +1,7 @@ import abc import logging from concurrent.futures import FIRST_COMPLETED, wait +from typing import Optional import numpy as np from deprecate import deprecated @@ -48,7 +49,7 @@ def _check(self, idx: int, score: float) -> bool: ... @abc.abstractmethod - def reset(self): + def reset(self, u: Optional[Utility] = None): """Reset the policy to a state ready for a new permutation.""" ... @@ -71,7 +72,7 @@ class NoTruncation(TruncationPolicy): def _check(self, idx: int, score: float) -> bool: return False - def reset(self): + def reset(self, u: Optional[Utility] = None): pass @@ -94,7 +95,7 @@ def _check(self, idx: int, score: float) -> bool: self.count += 1 return self.count >= self.max_marginals - def reset(self): + def reset(self, u: Optional[Utility] = None): self.count = 0 @@ -111,14 +112,18 @@ class RelativeTruncation(TruncationPolicy): def __init__(self, u: Utility, rtol: float): super().__init__() self.rtol = rtol - logger.info("Computing total utility for permutation truncation.") - self.total_utility = u(u.data.indices) + self.total_utility = self.reset(u) + self._u = u def _check(self, idx: int, score: float) -> bool: return np.allclose(score, self.total_utility, rtol=self.rtol) - def reset(self): - pass + def reset(self, u: Optional[Utility] = None) -> float: + if u is None: + u = self._u + + self.total_utility = u(u.data.indices) + return self.total_utility class BootstrapTruncation(TruncationPolicy): @@ -134,7 +139,6 @@ class BootstrapTruncation(TruncationPolicy): def __init__(self, u: Utility, n_samples: int, sigmas: float = 1): super().__init__() self.n_samples = n_samples - logger.info("Computing total utility for permutation truncation.") self.total_utility = u(u.data.indices) self.count: int = 0 self.variance: float = 0 @@ -155,7 +159,7 @@ def _check(self, idx: int, score: float) -> bool: self.sigmas * np.sqrt(self.variance) ) - def reset(self): + def reset(self, u: Optional[Utility] = None): self.count = 0 self.variance = self.mean = 0 diff --git a/src/pydvl/value/stopping.py b/src/pydvl/value/stopping.py index 09ba84475..b235d2067 100644 --- a/src/pydvl/value/stopping.py +++ b/src/pydvl/value/stopping.py @@ -279,13 +279,13 @@ class MaxChecks(StoppingCriterion): def __init__(self, n_checks: Optional[int], modify_result: bool = True): super().__init__(modify_result=modify_result) - if n_checks is not None and n_checks < 1: - raise ValueError("n_iterations must be at least 1 or None") + if n_checks is not None and n_checks < 0: + raise ValueError("n_iterations must be at least 0 or None") self.n_checks = n_checks self._count = 0 def _check(self, result: ValuationResult) -> Status: - if self.n_checks: + if self.n_checks is not None: self._count += 1 if self._count > self.n_checks: self._converged = np.ones_like(result.values, dtype=bool) @@ -293,7 +293,7 @@ def _check(self, result: ValuationResult) -> Status: return Status.Pending def completion(self) -> float: - if self.n_checks: + if self.n_checks is not None: return min(1.0, self._count / self.n_checks) return 0.0 @@ -476,7 +476,7 @@ def _check(self, r: ValuationResult) -> Status: quots = np.divide(diffs, curr[ii], out=diffs, where=curr[ii] != 0) # quots holds the quotients when the denominator is non-zero, and # the absolute difference, which is just the memory, otherwise. - if np.mean(quots) < self.rtol: + if len(quots) > 0 and np.mean(quots) < self.rtol: self._converged = self.update_op( self._converged, r.counts > self.n_steps ) # type: ignore diff --git a/tests/conftest.py b/tests/conftest.py index 41244d275..d03779214 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,8 +11,9 @@ from sklearn import datasets from sklearn.utils import Bunch -from pydvl.utils import Dataset, MemcachedClientConfig +from pydvl.utils import ClasswiseScorer, Dataset, MemcachedClientConfig, Utility from pydvl.utils.parallel.backend import available_cpus +from tests.misc import ClosedFormLinearClassifier if TYPE_CHECKING: from _pytest.config import Config @@ -411,3 +412,24 @@ def pytest_terminal_summary( ): tolerate_session = terminalreporter.config._tolerate_session tolerate_session.display(terminalreporter) + + +@pytest.fixture(scope="function") +def dataset_alt_seq_full() -> Dataset: + x_train = np.arange(1, 5).reshape([-1, 1]) + y_train = np.array([0, 0, 1, 1]) + x_test = x_train + y_test = np.array([0, 0, 0, 1]) + return Dataset(x_train, y_train, x_test, y_test) + + +@pytest.fixture(scope="function") +def linear_classifier_cs_scorer( + dataset_alt_seq_full: Dataset, +) -> Utility: + return Utility( + ClosedFormLinearClassifier(), + dataset_alt_seq_full, + ClasswiseScorer("accuracy"), + catch_errors=False, + ) diff --git a/tests/misc.py b/tests/misc.py new file mode 100644 index 000000000..2d12fb673 --- /dev/null +++ b/tests/misc.py @@ -0,0 +1,36 @@ +import numpy as np +from numpy._typing import NDArray + + +class ThresholdClassifier: + def fit(self, x: NDArray, y: NDArray) -> float: + raise NotImplementedError("Mock model") + + def predict(self, x: NDArray) -> NDArray: + y = 0.5 < x + return y[:, 0].astype(int) + + def score(self, x: NDArray, y: NDArray) -> float: + raise NotImplementedError("Mock model") + + +class ClosedFormLinearClassifier: + def __init__(self): + self._beta = None + + def fit(self, x: NDArray, y: NDArray) -> float: + v = x[:, 0] + self._beta = np.dot(v, y) / np.dot(v, v) + return -1 + + def predict(self, x: NDArray) -> NDArray: + if self._beta is None: + raise AttributeError("Model not fitted") + + x = x[:, 0] + probs = self._beta * x + return np.clip(np.round(probs + 1e-10), 0, 1).astype(int) + + def score(self, x: NDArray, y: NDArray) -> float: + pred_y = self.predict(x) + return np.sum(pred_y == y) / 4 diff --git a/tests/utils/conftest.py b/tests/utils/conftest.py index f64330777..5783aba8a 100644 --- a/tests/utils/conftest.py +++ b/tests/utils/conftest.py @@ -1,5 +1,9 @@ +from typing import Dict, Tuple + +import numpy as np import pytest import ray +from numpy.typing import NDArray from ray.cluster_utils import Cluster from pydvl.utils.config import ParallelConfig @@ -23,3 +27,21 @@ def parallel_config(request, num_workers): yield ParallelConfig(backend="ray", address=cluster.address) ray.shutdown() cluster.shutdown() + + +@pytest.fixture(scope="function") +def dataset_alt_seq_simple( + request, +) -> Tuple[NDArray[np.float_], NDArray[np.int_], Dict[str, float]]: + """ + The label set is represented as 0000011100011111, with adjustable left and right + margins. The left margin denotes the percentage of zeros at the beginning, while the + right margin denotes the percentage of ones at the end. Accuracy can be efficiently + calculated using a closed-form solution. + """ + n_element, left_margin, right_margin = request.param + x = np.linspace(0, 1, n_element) + y = ((left_margin <= x) & (x < 0.5)) | ((1 - right_margin) <= x) + y = y.astype(int) + x = np.expand_dims(x, -1) + return x, y, {"left_margin": left_margin, "right_margin": right_margin} diff --git a/tests/utils/test_numeric.py b/tests/utils/test_numeric.py index e6101defb..5b3d2582e 100644 --- a/tests/utils/test_numeric.py +++ b/tests/utils/test_numeric.py @@ -5,6 +5,7 @@ powerset, random_matrix_with_condition_number, random_powerset, + random_powerset_group_conditional, random_subset_of_size, running_moments, ) @@ -138,3 +139,29 @@ def test_running_moments(): true_variances = [np.var(vv) for vv in values] assert np.allclose(means, true_means) assert np.allclose(variances, true_variances) + + +@pytest.mark.parametrize("min_elements", [1, 2]) +@pytest.mark.parametrize("elements_per_group", [10]) +@pytest.mark.parametrize("num_groups", [3]) +@pytest.mark.parametrize("check_num_samples", [10]) +def test_random_powerset_group_conditional( + min_elements: int, + elements_per_group: int, + num_groups: int, + check_num_samples: int, +): + s = np.arange(num_groups * elements_per_group) + groups = np.arange(num_groups).repeat(elements_per_group) + + for idx, subset in enumerate( + random_powerset_group_conditional(s, groups, min_elements) + ): + assert np.all(np.isin(subset, s)) + assert np.all(np.unique(groups[subset]) == np.unique(groups)) + + for group in np.unique(groups): + assert np.sum(group == groups[subset]) >= min_elements + + if idx == check_num_samples: + break diff --git a/tests/utils/test_score.py b/tests/utils/test_score.py index 078775240..15bd91d1e 100644 --- a/tests/utils/test_score.py +++ b/tests/utils/test_score.py @@ -1,7 +1,19 @@ +from typing import Dict, Tuple, cast + import numpy as np +import pandas as pd +import pytest from numpy.typing import NDArray -from pydvl.utils.score import Scorer, compose_score, squashed_r2, squashed_variance +from pydvl.utils import Utility, powerset +from pydvl.utils.score import ( + ClasswiseScorer, + Scorer, + compose_score, + squashed_r2, + squashed_variance, +) +from tests.misc import ThresholdClassifier sigmoid = lambda x: 1 / (1 + np.exp(-x)) @@ -69,3 +81,115 @@ def test_squashed_variance(): X = np.array([[1, 2], [3, 4]]) model = FittedLinearModel(coef) assert sigmoid(1.0) == squashed_variance(model, X, X @ coef) + + +@pytest.mark.parametrize( + "dataset_alt_seq_simple", + [((101, 0.3, 0.4))], + indirect=True, +) +def test_cs_scorer_on_dataset_alt_seq_simple(dataset_alt_seq_simple): + """ + Tests the class wise scorer. + """ + + scorer = ClasswiseScorer("accuracy", initial_label=0) + assert str(scorer) == "classwise accuracy" + assert repr(scorer) == "ClasswiseAccuracy (scorer=make_scorer(accuracy_score))" + + x, y, info = dataset_alt_seq_simple + n_element = len(x) + target_in_cls_acc_0 = (info["left_margin"] * 100 + 1) / n_element + target_out_of_cls_acc_0 = (info["right_margin"] * 100 + 1) / n_element + + model = ThresholdClassifier() + in_cls_acc_0, out_of_cls_acc_0 = scorer.estimate_in_cls_and_out_of_cls_score( + model, x, y + ) + assert np.isclose(in_cls_acc_0, target_in_cls_acc_0) + assert np.isclose(out_of_cls_acc_0, target_out_of_cls_acc_0) + + scorer.label = 1 + in_cls_acc_1, out_of_cls_acc_1 = scorer.estimate_in_cls_and_out_of_cls_score( + model, x, y + ) + assert in_cls_acc_1 == out_of_cls_acc_0 + assert in_cls_acc_0 == out_of_cls_acc_1 + + scorer.label = 0 + value = scorer(model, x, y) + assert np.isclose(value, in_cls_acc_0 * np.exp(out_of_cls_acc_0)) + + scorer.label = 1 + value = scorer(model, x, y) + assert np.isclose(value, in_cls_acc_1 * np.exp(out_of_cls_acc_1)) + + +def test_cs_scorer_on_alt_seq_cf_linear_classifier_cs_score( + linear_classifier_cs_scorer: Utility, +): + subsets_zero = list(powerset(np.array((0, 1)))) + subsets_one = list(powerset(np.array((2, 3)))) + subsets_zero = [tuple(s) for s in subsets_zero] + subsets_one = [tuple(s) for s in subsets_one] + target_betas = pd.DataFrame( + [ + [np.nan, 1 / 3, 1 / 4, 7 / 25], + [0, 3 / 10, 4 / 17, 7 / 26], + [0, 3 / 13, 1 / 5, 7 / 29], + [0, 3 / 14, 4 / 21, 7 / 30], + ], + index=subsets_zero, + columns=subsets_one, + ) + target_accuracies_zero = pd.DataFrame( + [ + [0, 1 / 4, 1 / 4, 1 / 4], + [3 / 4, 1 / 4, 1 / 2, 1 / 4], + [3 / 4, 1 / 2, 1 / 2, 1 / 2], + [3 / 4, 1 / 2, 1 / 2, 1 / 2], + ], + index=subsets_zero, + columns=subsets_one, + ) + target_accuracies_one = pd.DataFrame( + [ + [0, 1 / 4, 1 / 4, 1 / 4], + [0, 1 / 4, 1 / 4, 1 / 4], + [0, 1 / 4, 1 / 4, 1 / 4], + [0, 1 / 4, 1 / 4, 1 / 4], + ], + index=subsets_zero, + columns=subsets_one, + ) + model = linear_classifier_cs_scorer.model + scorer = cast(ClasswiseScorer, linear_classifier_cs_scorer.scorer) + scorer.label = 0 + + for set_zero_idx in range(len(subsets_zero)): + for set_one_idx in range(len(subsets_one)): + indices = list(subsets_zero[set_zero_idx] + subsets_one[set_one_idx]) + ( + x_train, + y_train, + ) = linear_classifier_cs_scorer.data.get_training_data(indices) + linear_classifier_cs_scorer.model.fit(x_train, y_train) + fitted_beta = linear_classifier_cs_scorer.model._beta # noqa + target_beta = target_betas.iloc[set_zero_idx, set_one_idx] + assert ( + np.isnan(fitted_beta) + if np.isnan(target_beta) + else fitted_beta == target_beta + ) + + ( + x_test, + y_test, + ) = linear_classifier_cs_scorer.data.get_test_data() + in_cls_acc_0, in_cls_acc_1 = scorer.estimate_in_cls_and_out_of_cls_score( + model, x_test, y_test + ) + assert ( + in_cls_acc_0 == target_accuracies_zero.iloc[set_zero_idx, set_one_idx] + ) + assert in_cls_acc_1 == target_accuracies_one.iloc[set_zero_idx, set_one_idx] diff --git a/tests/value/shapley/test_classwise.py b/tests/value/shapley/test_classwise.py new file mode 100644 index 000000000..1d263a7d3 --- /dev/null +++ b/tests/value/shapley/test_classwise.py @@ -0,0 +1,786 @@ +""" +Test cases for the class wise shapley value. +""" +import random +from random import seed +from typing import Dict, Tuple + +import numpy as np +import pytest + +from pydvl.utils import Utility +from pydvl.value import MaxChecks, ValuationResult +from pydvl.value.shapley.classwise import compute_classwise_shapley_values +from pydvl.value.shapley.truncated import NoTruncation +from tests.value import check_values + + +@pytest.fixture(scope="function") +def linear_classifier_cs_scorer_args_exact_solution_use_default_score() -> Tuple[ + Dict, ValuationResult, Dict +]: + r""" + Returns the exact solution for the class wise shapley value of the training and + validation set of the `utility_alt_seq_cf_linear_classifier_cs_scorer` fixture. + + =========================== + CS-Shapley Manual Derivation + =========================== + + :Author: Markus Semmler + :Date: August 2023 + + Dataset description + =================== + + We have a training and a test dataset. We want to model a simple XOR dataset. The + development set :math:`D` is given by + + .. math:: + \begin{aligned} + \hat{x}_0 &= 1 \quad &\hat{y}_0 = 0 \\ + \hat{x}_1 &= 2 \quad &\hat{y}_1 = 0 \\ + \hat{x}_2 &= 3 \quad &\hat{y}_2 = 0 \\ + \hat{x}_3 &= 4 \quad &\hat{y}_3 = 1 \\ + \end{aligned} + + and the training set :math:`T` is given by + + .. math:: + \begin{aligned} + x_0 &= 1 \quad &y_0 = 0 \\ + x_1 &= 2 \quad &y_1 = 0 \\ + x_2 &= 3 \quad &y_2 = 1 \\ + x_3 &= 4 \quad &y_3 = 1 \\ + \end{aligned} + + Note that the training set and the development set contain the same + inputs x, but differ in the label :math:`\hat{y}_2 \neq y_2` + + Model + ===== + + We use an adapted version of linear regression + + .. math:: y = \max(0, \min(1, \text{round}(\beta^T x))) + + for classification, with the closed form solution + + .. math:: \beta = \frac{\text{dot}(x, y)}{\text{dot}(x, x)} + + Fitted model + ============ + + The hyperparameters for all combinations are + + .. container:: tabular + + | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & + :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & + :math:`\{x_2, x_3\}` + | :math:`\emptyset` & nan & :math:`\frac{1}{3}` & :math:`\frac{1}{4}` + & :math:`\frac{7}{25}` + | :math:`\{x_0\}` & :math:`0` & :math:`\frac{3}{10}` & + :math:`\frac{4}{17}` & :math:`\frac{7}{26}` + | :math:`\{x_1\}` & :math:`0` & :math:`\frac{3}{13}` & + :math:`\frac{1}{5}` &\ :math:`\frac{7}{29}` + | :math:`\{x_0, x_1 \}` & :math:`0` & :math:`\frac{3}{14}` & + :math:`\frac{4}{21}` & :math:`\frac{7}{30}` + + Accuracy tables on development set :math:`D` + ============================================ + + (*) Note that the algorithm described in the paper overwrites these + values with 0. + + .. container:: tabular + + | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & + :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & + :math:`\{x_2, x_3\}` + | :math:`\emptyset` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` & :math:`\frac{1}{4}` + | :math:`\{x_0\}` & :math:`\frac{3}{4}` & :math:`\frac{1}{4}` & + :math:`\frac{1}{2}` & :math:`\frac{1}{4}` + | :math:`\{x_1\}` & :math:`\frac{3}{4}` & :math:`\frac{1}{2}` & + :math:`\frac{1}{2}` &\ :math:`\frac{1}{2}` + | :math:`\{x_0, x_1 \}` & :math:`\frac{3}{4}` & :math:`\frac{1}{2}` & + :math:`\frac{1}{2}` & :math:`\frac{1}{2}` + + .. container:: tabular + + | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & + :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & + :math:`\{x_2, x_3\}` + | :math:`\emptyset` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` & :math:`\frac{1}{4}` + | :math:`\{x_0\}` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` & :math:`\frac{1}{4}` + | :math:`\{x_1\}` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` &\ :math:`\frac{1}{4}` + | :math:`\{x_0, x_1 \}` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` & :math:`\frac{1}{4}` + + CS-Shapley + ========== + + The formulas of the algorithm are given by + + .. math:: + + \begin{aligned} + \delta(\pi, S_{-y_i}, i) &= v_{y_i}(\pi_{:i} \cup \{ i \} | S_{-y_i}) + - v_{y_i}(\pi_{:i} | S_{-y_i}) \\ + \left [ \phi_i | S_{-y_i} \right ] &= \frac{1}{|T_{y_i}|!} + \sum_{\pi \in \Pi(T_{y_i})} \delta(\pi, S_{-y_i}, i) \\ + \phi_i &= \frac{1}{2^{|T_{-y_i}|}-1} \left [\sum_{\emptyset \subset S_{-y_i} + \subseteq T_{-y_i}} \left [ \phi_i | S_{-y_i} \right ] \right ] + \end{aligned} + + Valuation of :math:`x_0` + ======================== + + .. math:: + \begin{aligned} + \delta((x_0, x_1), \{ x_2 \}, 0) &= \frac{1}{4} e^\frac{1}{4} &\quad + \delta((x_1, x_0), \{ x_2 \}, 0) &= 0 \\ + \delta((x_0, x_1), \{ x_3 \}, 0) &= \frac{1}{2} e^\frac{1}{4} &\quad + \delta((x_1, x_0), \{ x_3 \}, 0) &= 0 \\ + \delta((x_0, x_1), \{ x_2, x_3 \}, 0) &= \frac{1}{4} e^\frac{1}{4} &\quad + \delta((x_1, x_0), \{ x_2, x_3 \}, 0) &= 0 + \end{aligned} + + .. math:: + \begin{aligned} + \left [ \phi_0 | \{ x_2 \} \right] &= \frac{1}{8} e^\frac{1}{4} \\ + \left [ \phi_0 | \{ x_3 \} \right] &= \frac{1}{4} e^\frac{1}{4} \\ + \left [ \phi_0 | \{ x_2, x_3 \} \right] &= \frac{1}{8} e^\frac{1}{4} + \end{aligned} + + .. math:: \phi_0 = \frac{1}{6} e^\frac{1}{4} \approx 0.214 + + Valuation of :math:`x_1` + ======================== + + .. math:: + \begin{aligned} + \delta((x_0, x_1), \{ x_2 \}, 1) &= \frac{1}{4} e^\frac{1}{4} &\quad + \delta((x_1, x_0), \{ x_2 \}, 1) &= \frac{1}{2} e^\frac{1}{4} \\ + \delta((x_0, x_1), \{ x_3 \}, 1) &= 0 &\quad + \delta((x_1, x_0), \{ x_3 \}, 1) &= \frac{1}{2} e^\frac{1}{4} \\ + \delta((x_0, x_1), \{ x_2, x_3 \}, 1) &= \frac{1}{4} e^\frac{1}{4} &\quad + \delta((x_1, x_0), \{ x_2, x_3 \}, 1) &= \frac{1}{2} e^\frac{1}{4} + \end{aligned} + + .. math:: + \begin{aligned} + \left [ \phi_1 | \{ x_2 \} \right] &= \frac{3}{8} e^\frac{1}{4} \\ + \left [ \phi_1 | \{ x_3 \} \right] &= \frac{1}{4} e^\frac{1}{4} \\ + \left [ \phi_1 | \{ x_2, x_3 \} \right] &= \frac{3}{8} e^\frac{1}{4} + \end{aligned} + + .. math:: \phi_0 = \frac{1}{3} e^\frac{1}{4} \approx 0.428 + + Valuation of :math:`x_2` + ======================== + + .. math:: + \begin{aligned} + \delta((x_2, x_3), \{ x_0 \}, 2) &= \frac{1}{4} e^\frac{1}{4} &\quad + \delta((x_3, x_2), \{ x_0 \}, 2) + &= \frac{1}{4} e^\frac{1}{4} - \frac{1}{4} e^\frac{1}{2} \\ + \delta((x_2, x_3), \{ x_1 \}, 2) &= \frac{1}{4} e^\frac{1}{2} &\quad + \delta((x_3, x_2), \{ x_1 \}, 2) &= 0 \\ + \delta((x_2, x_3), \{ x_0, x_1 \}, 2) &= \frac{1}{4} e^\frac{1}{2} &\quad + \delta((x_3, x_2), \{ x_0, x_1 \}, 2) &= 0 + \end{aligned} + + .. math:: + \begin{aligned} + \left [ \phi_2 | \{ x_0 \} \right] + &= \frac{1}{4} e^\frac{1}{4} - \frac{1}{8} e^\frac{1}{2} \\ + \left [ \phi_2 | \{ x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ + \left [ \phi_2 | \{ x_0, x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} + \end{aligned} + + .. math:: \phi_2 = \frac{1}{12} e^\frac{1}{4} + \frac{1}{24} e^\frac{1}{2} \approx 0.1757 + + Valuation of :math:`x_3` + ======================== + + .. math:: + \begin{aligned} + \delta((x_2, x_3), \{ x_0 \}, 3) &= 0 &\quad + \delta((x_3, x_2), \{ x_0 \}, 3) &= \frac{1}{4} e^\frac{1}{2} \\ + \delta((x_2, x_3), \{ x_1 \}, 3) &= 0 &\quad + \delta((x_3, x_2), \{ x_1 \}, 3) &= \frac{1}{4} e^\frac{1}{2} \\ + \delta((x_2, x_3), \{ x_0, x_1 \}, 3) &= 0 &\quad + \delta((x_3, x_2), \{ x_0, x_1 \}, 3) &= \frac{1}{4} e^\frac{1}{2} + \end{aligned} + + .. math:: + \begin{aligned} + \left [ \phi_3 | \{ x_0 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ + \left [ \phi_3 | \{ x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ + \left [ \phi_3 | \{ x_0, x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} + \end{aligned} + + .. math:: \phi_3 = \frac{1}{8} e^\frac{1}{2} \approx 0.2061 + """ + return ( + { + "normalize_values": False, + }, + ValuationResult( + values=np.array( + [ + 1 / 6 * np.exp(1 / 4), + 1 / 3 * np.exp(1 / 4), + 1 / 12 * np.exp(1 / 4) + 1 / 24 * np.exp(1 / 2), + 1 / 8 * np.exp(1 / 2), + ] + ) + ), + {"atol": 0.05}, + ) + + +@pytest.fixture(scope="function") +def linear_classifier_cs_scorer_args_exact_solution_use_default_score_norm( + linear_classifier_cs_scorer_args_exact_solution_use_default_score: Tuple[ + Dict, ValuationResult, Dict + ] +) -> Tuple[Dict, ValuationResult, Dict]: + """ + Same as :func:`linear_classifier_cs_scorer_args_exact_solution_use_default_score` + but with normalization. The values of label c are normalized by the in-class score + of label c divided by the sum of values of that specific label. + """ + values = linear_classifier_cs_scorer_args_exact_solution_use_default_score[1].values + label_zero_coefficient = 1 / np.exp(1 / 4) + label_one_coefficient = 1 / (1 / 3 * np.exp(1 / 4) + 2 / 3 * np.exp(1 / 2)) + + return ( + { + "normalize_values": True, + }, + ValuationResult( + values=np.array( + [ + values[0] * label_zero_coefficient, + values[1] * label_zero_coefficient, + values[2] * label_one_coefficient, + values[3] * label_one_coefficient, + ] + ) + ), + {"atol": 0.05}, + ) + + +@pytest.fixture(scope="function") +def linear_classifier_cs_scorer_args_exact_solution_use_add_idx() -> Tuple[ + Dict, ValuationResult, Dict +]: + r""" + Returns the exact solution for the class wise shapley value of the training and + validation set of the `utility_alt_seq_cf_linear_classifier_cs_scorer` fixture. + + =========================== + CS-Shapley Manual Derivation + =========================== + + :Author: Markus Semmler + :Date: August 2023 + + Dataset description + =================== + + We have a training and a test dataset. We want to model a simple XOR dataset. The + development set :math:`D` is given by + + .. math:: + \begin{aligned} + \hat{x}_0 &= 1 \quad &\hat{y}_0 = 0 \\ + \hat{x}_1 &= 2 \quad &\hat{y}_1 = 0 \\ + \hat{x}_2 &= 3 \quad &\hat{y}_2 = 0 \\ + \hat{x}_3 &= 4 \quad &\hat{y}_3 = 1 \\ + \end{aligned} + + and the training set :math:`T` is given by + + .. math:: + \begin{aligned} + x_0 &= 1 \quad &y_0 = 0 \\ + x_1 &= 2 \quad &y_1 = 0 \\ + x_2 &= 3 \quad &y_2 = 1 \\ + x_3 &= 4 \quad &y_3 = 1 \\ + \end{aligned} + + Note that the training set and the development set contain the same + inputs x, but differ in the label :math:`\hat{y}_2 \neq y_2` + + Model + ===== + + We use an adapted version of linear regression + + .. math:: y = \max(0, \min(1, \text{round}(\beta^T x))) + + for classification, with the closed form solution + + .. math:: \beta = \frac{\text{dot}(x, y)}{\text{dot}(x, x)} + + Fitted model + ============ + + The hyperparameters for all combinations are + + .. container:: tabular + + | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & + :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & + :math:`\{x_2, x_3\}` + | :math:`\emptyset` & nan & :math:`\frac{1}{3}` & :math:`\frac{1}{4}` + & :math:`\frac{7}{25}` + | :math:`\{x_0\}` & :math:`0` & :math:`\frac{3}{10}` & + :math:`\frac{4}{17}` & :math:`\frac{7}{26}` + | :math:`\{x_1\}` & :math:`0` & :math:`\frac{3}{13}` & + :math:`\frac{1}{5}` &\ :math:`\frac{7}{29}` + | :math:`\{x_0, x_1 \}` & :math:`0` & :math:`\frac{3}{14}` & + :math:`\frac{4}{21}` & :math:`\frac{7}{30}` + + Accuracy tables on development set :math:`D` + ============================================ + + .. container:: tabular + + | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & + :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & + :math:`\{x_2, x_3\}` + | :math:`\emptyset` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` & :math:`\frac{1}{4}` + | :math:`\{x_0\}` & :math:`\frac{3}{4}` & :math:`\frac{1}{4}` & + :math:`\frac{1}{2}` & :math:`\frac{1}{4}` + | :math:`\{x_1\}` & :math:`\frac{3}{4}` & :math:`\frac{1}{2}` & + :math:`\frac{1}{2}` &\ :math:`\frac{1}{2}` + | :math:`\{x_0, x_1 \}` & :math:`\frac{3}{4}` & :math:`\frac{1}{2}` & + :math:`\frac{1}{2}` & :math:`\frac{1}{2}` + + .. container:: tabular + + | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & + :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & + :math:`\{x_2, x_3\}` + | :math:`\emptyset` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` & :math:`\frac{1}{4}` + | :math:`\{x_0\}` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` & :math:`\frac{1}{4}` + | :math:`\{x_1\}` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` &\ :math:`\frac{1}{4}` + | :math:`\{x_0, x_1 \}` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` & :math:`\frac{1}{4}` + + CS-Shapley + ========== + + The formulas of the algorithm are given by + + .. math:: + + \begin{aligned} + \delta(\pi, S_{-y_i}, i) &= v_{y_i}(\pi_{:i} \cup \{ i \} | S_{-y_i}) + - v_{y_i}(\pi_{:i} | S_{-y_i}) \\ + \left [ \phi_i | S_{-y_i} \right ] &= \frac{1}{|T_{y_i}|!} + \sum_{\pi \in \Pi(T_{y_i})} \delta(\pi, S_{-y_i}, i) \\ + \phi_i &= \frac{1}{2^{|T_{-y_i}|}-1} \left [\sum_{\emptyset \subset S_{-y_i} + \subseteq T_{-y_i}} \left [ \phi_i | S_{-y_i} \right ] \right ] + \end{aligned} + + Valuation of :math:`x_0` + ======================== + + .. math:: + \begin{aligned} + \delta((x_0, x_1), \{ x_2 \}, 0) &= 0 &\quad + \delta((x_1, x_0), \{ x_2 \}, 0) &= 0 \\ + \delta((x_0, x_1), \{ x_3 \}, 0) &= \frac{1}{4} e^\frac{1}{4} &\quad + \delta((x_1, x_0), \{ x_3 \}, 0) &= 0 \\ + \delta((x_0, x_1), \{ x_2, x_3 \}, 0) &= 0 &\quad + \delta((x_1, x_0), \{ x_2, x_3 \}, 0) &= 0 + \end{aligned} + + .. math:: + \begin{aligned} + \left [ \phi_0 | \{ x_2 \} \right] &= 0 \\ + \left [ \phi_0 | \{ x_3 \} \right] &= \frac{1}{8} e^\frac{1}{4} \\ + \left [ \phi_0 | \{ x_2, x_3 \} \right] &= 0 + \end{aligned} + + .. math:: \phi_0 = \frac{1}{24} e^\frac{1}{4} \approx 0.0535 + + Valuation of :math:`x_1` + ======================== + + .. math:: + \begin{aligned} + \delta((x_0, x_1), \{ x_2 \}, 1) &= \frac{1}{4} e^\frac{1}{4} &\quad + \delta((x_1, x_0), \{ x_2 \}, 1) &= \frac{1}{4} e^\frac{1}{4} \\ + \delta((x_0, x_1), \{ x_3 \}, 1) &= 0 &\quad + \delta((x_1, x_0), \{ x_3 \}, 1) &= \frac{1}{4} e^\frac{1}{4} \\ + \delta((x_0, x_1), \{ x_2, x_3 \}, 1) &= \frac{1}{4} e^\frac{1}{4} &\quad + \delta((x_1, x_0), \{ x_2, x_3 \}, 1) &= \frac{1}{4} e^\frac{1}{4} + \end{aligned} + + .. math:: + \begin{aligned} + \left [ \phi_1 | \{ x_2 \} \right] &= \frac{1}{4} e^\frac{1}{4} \\ + \left [ \phi_1 | \{ x_3 \} \right] &= \frac{1}{8} e^\frac{1}{4} \\ + \left [ \phi_1 | \{ x_2, x_3 \} \right] &= \frac{1}{4} e^\frac{1}{4} + \end{aligned} + + .. math:: \phi_0 = \frac{5}{24} e^\frac{1}{4} \approx 0.2675 + + Valuation of :math:`x_2` + ======================== + + .. math:: + \begin{aligned} + \delta((x_2, x_3), \{ x_0 \}, 2) &= \frac{1}{4} e^\frac{1}{4} &\quad + \delta((x_3, x_2), \{ x_0 \}, 2) + &= \frac{1}{4} e^\frac{1}{4} - \frac{1}{4} e^\frac{1}{2} \\ + \delta((x_2, x_3), \{ x_1 \}, 2) &= \frac{1}{4} e^\frac{1}{2} &\quad + \delta((x_3, x_2), \{ x_1 \}, 2) &= 0 \\ + \delta((x_2, x_3), \{ x_0, x_1 \}, 2) &= \frac{1}{4} e^\frac{1}{2} &\quad + \delta((x_3, x_2), \{ x_0, x_1 \}, 2) &= 0 + \end{aligned} + + .. math:: + \begin{aligned} + \left [ \phi_2 | \{ x_0 \} \right] + &= \frac{1}{4} e^\frac{1}{4} - \frac{1}{8} e^\frac{1}{2} \\ + \left [ \phi_2 | \{ x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ + \left [ \phi_2 | \{ x_0, x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} + \end{aligned} + + .. math:: \phi_2 = \frac{1}{12} e^\frac{1}{4} + \frac{1}{24} e^\frac{1}{2} \approx 0.1757 + + Valuation of :math:`x_3` + ======================== + + .. math:: + \begin{aligned} + \delta((x_2, x_3), \{ x_0 \}, 3) &= 0 &\quad + \delta((x_3, x_2), \{ x_0 \}, 3) &= \frac{1}{4} e^\frac{1}{2} \\ + \delta((x_2, x_3), \{ x_1 \}, 3) &= 0 &\quad + \delta((x_3, x_2), \{ x_1 \}, 3) &= \frac{1}{4} e^\frac{1}{2} \\ + \delta((x_2, x_3), \{ x_0, x_1 \}, 3) &= 0 &\quad + \delta((x_3, x_2), \{ x_0, x_1 \}, 3) &= \frac{1}{4} e^\frac{1}{2} + \end{aligned} + + .. math:: + \begin{aligned} + \left [ \phi_3 | \{ x_0 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ + \left [ \phi_3 | \{ x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ + \left [ \phi_3 | \{ x_0, x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} + \end{aligned} + + .. math:: \phi_3 = \frac{1}{8} e^\frac{1}{2} \approx 0.2061 + """ + return ( + { + "use_default_scorer_value": False, + "normalize_values": False, + }, + ValuationResult( + values=np.array( + [ + 1 / 24 * np.exp(1 / 4), + 5 / 24 * np.exp(1 / 4), + 1 / 12 * np.exp(1 / 4) + 1 / 24 * np.exp(1 / 2), + 1 / 8 * np.exp(1 / 2), + ] + ) + ), + {"atol": 0.05}, + ) + + +@pytest.fixture(scope="function") +def linear_classifier_cs_scorer_args_exact_solution_use_add_idx_empty_set() -> Tuple[ + Dict, ValuationResult, Dict +]: + r""" + Returns the exact solution for the class wise shapley value of the training and + validation set of the `utility_alt_seq_cf_linear_classifier_cs_scorer` fixture. + + =========================== + CS-Shapley Manual Derivation + =========================== + + :Author: Markus Semmler + :Date: August 2023 + + Dataset description + =================== + + We have a training and a test dataset. We want to model a simple XOR dataset. The + development set :math:`D` is given by + + .. math:: + \begin{aligned} + \hat{x}_0 &= 1 \quad &\hat{y}_0 = 0 \\ + \hat{x}_1 &= 2 \quad &\hat{y}_1 = 0 \\ + \hat{x}_2 &= 3 \quad &\hat{y}_2 = 0 \\ + \hat{x}_3 &= 4 \quad &\hat{y}_3 = 1 \\ + \end{aligned} + + and the training set :math:`T` is given by + + .. math:: + \begin{aligned} + x_0 &= 1 \quad &y_0 = 0 \\ + x_1 &= 2 \quad &y_1 = 0 \\ + x_2 &= 3 \quad &y_2 = 1 \\ + x_3 &= 4 \quad &y_3 = 1 \\ + \end{aligned} + + Note that the training set and the development set contain the same + inputs x, but differ in the label :math:`\hat{y}_2 \neq y_2` + + Model + ===== + + We use an adapted version of linear regression + + .. math:: y = \max(0, \min(1, \text{round}(\beta^T x))) + + for classification, with the closed form solution + + .. math:: \beta = \frac{\text{dot}(x, y)}{\text{dot}(x, x)} + + Fitted model + ============ + + The hyperparameters for all combinations are + + .. container:: tabular + + | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & + :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & + :math:`\{x_2, x_3\}` + | :math:`\emptyset` & nan & :math:`\frac{1}{3}` & :math:`\frac{1}{4}` + & :math:`\frac{7}{25}` + | :math:`\{x_0\}` & :math:`0` & :math:`\frac{3}{10}` & + :math:`\frac{4}{17}` & :math:`\frac{7}{26}` + | :math:`\{x_1\}` & :math:`0` & :math:`\frac{3}{13}` & + :math:`\frac{1}{5}` &\ :math:`\frac{7}{29}` + | :math:`\{x_0, x_1 \}` & :math:`0` & :math:`\frac{3}{14}` & + :math:`\frac{4}{21}` & :math:`\frac{7}{30}` + + Accuracy tables on development set :math:`D` + ============================================ + + .. container:: tabular + + | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & + :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & + :math:`\{x_2, x_3\}` + | :math:`\emptyset` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` & :math:`\frac{1}{4}` + | :math:`\{x_0\}` & :math:`\frac{3}{4}` & :math:`\frac{1}{4}` & + :math:`\frac{1}{2}` & :math:`\frac{1}{4}` + | :math:`\{x_1\}` & :math:`\frac{3}{4}` & :math:`\frac{1}{2}` & + :math:`\frac{1}{2}` &\ :math:`\frac{1}{2}` + | :math:`\{x_0, x_1 \}` & :math:`\frac{3}{4}` & :math:`\frac{1}{2}` & + :math:`\frac{1}{2}` & :math:`\frac{1}{2}` + + .. container:: tabular + + | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & + :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & + :math:`\{x_2, x_3\}` + | :math:`\emptyset` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` & :math:`\frac{1}{4}` + | :math:`\{x_0\}` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` & :math:`\frac{1}{4}` + | :math:`\{x_1\}` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` &\ :math:`\frac{1}{4}` + | :math:`\{x_0, x_1 \}` & :math:`0` & :math:`\frac{1}{4}` & + :math:`\frac{1}{4}` & :math:`\frac{1}{4}` + + CS-Shapley + ========== + + The formulas of the algorithm are given by + + .. math:: + + \begin{aligned} + \delta(\pi, S_{-y_i}, i) &= v_{y_i}(\pi_{:i} \cup \{ i \} | S_{-y_i}) + - v_{y_i}(\pi_{:i} | S_{-y_i}) \\ + \left [ \phi_i | S_{-y_i} \right ] &= \frac{1}{|T_{y_i}|!} + \sum_{\pi \in \Pi(T_{y_i})} \delta(\pi, S_{-y_i}, i) \\ + \phi_i &= \frac{1}{2^{|T_{-y_i}|}} \left [\sum_{S_{-y_i} + \subseteq T_{-y_i}} \left [ \phi_i | S_{-y_i} \right ] \right ] + \end{aligned} + + Valuation of :math:`x_0` + ======================== + + .. math:: + \begin{aligned} + \delta((x_0, x_1), \emptyset, 0) &= \frac{3}{4} &\quad + \delta((x_1, x_0), \emptyset, 0) &= 0 \\ + \delta((x_0, x_1), \{ x_2 \}, 0) &= 0 &\quad + \delta((x_1, x_0), \{ x_2 \}, 0) &= 0 \\ + \delta((x_0, x_1), \{ x_3 \}, 0) &= \frac{1}{4} e^\frac{1}{4} &\quad + \delta((x_1, x_0), \{ x_3 \}, 0) &= 0 \\ + \delta((x_0, x_1), \{ x_2, x_3 \}, 0) &= 0 &\quad + \delta((x_1, x_0), \{ x_2, x_3 \}, 0) &= 0 + \end{aligned} + + .. math:: + \begin{aligned} + \left [ \phi_0 | \emptyset \right] &= \frac{3}{8} \\ + \left [ \phi_0 | \{ x_2 \} \right] &= 0 \\ + \left [ \phi_0 | \{ x_3 \} \right] &= \frac{1}{8} e^\frac{1}{4} \\ + \left [ \phi_0 | \{ x_2, x_3 \} \right] &= 0 + \end{aligned} + + .. math:: \phi_0 = \frac{3}{32} + \frac{1}{32} e^\frac{1}{4} \approx 0.1339 + + Valuation of :math:`x_1` + ======================== + + .. math:: + \begin{aligned} + \delta((x_0, x_1), \emptyset, 1) &= 0 &\quad + \delta((x_1, x_0), \emptyset, 1) &= \frac{3}{4} \\ + \delta((x_0, x_1), \{ x_2 \}, 1) &= \frac{1}{4} e^\frac{1}{4} &\quad + \delta((x_1, x_0), \{ x_2 \}, 1) &= \frac{1}{4} e^\frac{1}{4} \\ + \delta((x_0, x_1), \{ x_3 \}, 1) &= 0 &\quad + \delta((x_1, x_0), \{ x_3 \}, 1) &= \frac{1}{4} e^\frac{1}{4} \\ + \delta((x_0, x_1), \{ x_2, x_3 \}, 1) &= \frac{1}{4} e^\frac{1}{4} &\quad + \delta((x_1, x_0), \{ x_2, x_3 \}, 1) &= \frac{1}{4} e^\frac{1}{4} + \end{aligned} + + .. math:: + \begin{aligned} + \left [ \phi_1 | \emptyset \right] &= \frac{3}{8} \\ + \left [ \phi_1 | \{ x_2 \} \right] &= \frac{1}{4} e^\frac{1}{4} \\ + \left [ \phi_1 | \{ x_3 \} \right] &= \frac{1}{8} e^\frac{1}{4} \\ + \left [ \phi_1 | \{ x_2, x_3 \} \right] &= \frac{1}{4} e^\frac{1}{4} + \end{aligned} + + .. math:: \phi_0 = \frac{3}{32} + \frac{5}{32} e^\frac{1}{4} \approx 0.2944 + + Valuation of :math:`x_2` + ======================== + + .. math:: + \begin{aligned} + \delta((x_2, x_3), \emptyset, 2) &= \frac{1}{4} e^\frac{1}{4} &\quad + \delta((x_3, x_2), \emptyset, 2) &= 0 \\ + \delta((x_2, x_3), \{ x_0 \}, 2) &= \frac{1}{4} e^\frac{1}{4} &\quad + \delta((x_3, x_2), \{ x_0 \}, 2) + &= \frac{1}{4} e^\frac{1}{4} - \frac{1}{4} e^\frac{1}{2} \\ + \delta((x_2, x_3), \{ x_1 \}, 2) &= \frac{1}{4} e^\frac{1}{2} &\quad + \delta((x_3, x_2), \{ x_1 \}, 2) &= 0 \\ + \delta((x_2, x_3), \{ x_0, x_1 \}, 2) &= \frac{1}{4} e^\frac{1}{2} &\quad + \delta((x_3, x_2), \{ x_0, x_1 \}, 2) &= 0 + \end{aligned} + + .. math:: + \begin{aligned} + \left [ \phi_2 | \emptyset \right] &= \frac{1}{8} e^\frac{1}{4} \\ + \left [ \phi_2 | \{ x_0 \} \right] + &= \frac{1}{4} e^\frac{1}{4} - \frac{1}{8} e^\frac{1}{2} \\ + \left [ \phi_2 | \{ x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ + \left [ \phi_2 | \{ x_0, x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} + \end{aligned} + + .. math:: + \phi_2 = \frac{5}{32} e^\frac{1}{4} + \frac{1}{32} e^\frac{1}{2} \approx 0.2522 + + Valuation of :math:`x_3` + ======================== + + .. math:: + \begin{aligned} + \delta((x_2, x_3), \emptyset, 3) &= 0 &\quad + \delta((x_3, x_2), \emptyset, 3) &= \frac{1}{4} e^\frac{1}{4} \\ + \delta((x_2, x_3), \{ x_0 \}, 3) &= 0 &\quad + \delta((x_3, x_2), \{ x_0 \}, 3) &= \frac{1}{4} e^\frac{1}{2} \\ + \delta((x_2, x_3), \{ x_1 \}, 3) &= 0 &\quad + \delta((x_3, x_2), \{ x_1 \}, 3) &= \frac{1}{4} e^\frac{1}{2} \\ + \delta((x_2, x_3), \{ x_0, x_1 \}, 3) &= 0 &\quad + \delta((x_3, x_2), \{ x_0, x_1 \}, 3) &= \frac{1}{4} e^\frac{1}{2} + \end{aligned} + + .. math:: + \begin{aligned} + \left [ \phi_3 | \emptyset \right] &= \frac{1}{8} e^\frac{1}{4} \\ + \left [ \phi_3 | \{ x_0 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ + \left [ \phi_3 | \{ x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ + \left [ \phi_3 | \{ x_0, x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} + \end{aligned} + + .. math:: + \phi_3 = \frac{1}{32} e^\frac{1}{4} + \frac{3}{32} e^\frac{1}{2} \approx 0.1947 + """ + return ( + { + "use_default_scorer_value": False, + "min_elements_per_label": 0, + "normalize_values": False, + }, + ValuationResult( + values=np.array( + [ + 3 / 32 + 1 / 32 * np.exp(1 / 4), + 3 / 32 + 5 / 32 * np.exp(1 / 4), + 5 / 32 * np.exp(1 / 4) + 1 / 32 * np.exp(1 / 2), + 1 / 32 * np.exp(1 / 4) + 3 / 32 * np.exp(1 / 2), + ] + ) + ), + {"atol": 0.05}, + ) + + +@pytest.mark.parametrize("n_samples", [500], ids=lambda x: "n_samples={}".format(x)) +@pytest.mark.parametrize( + "n_resample_complement_sets", + [1], + ids=lambda x: "n_resample_complement_sets={}".format(x), +) +@pytest.mark.parametrize( + "linear_classifier_cs_scorer_args_exact_solution", + [ + "linear_classifier_cs_scorer_args_exact_solution_use_default_score", + "linear_classifier_cs_scorer_args_exact_solution_use_default_score_norm", + "linear_classifier_cs_scorer_args_exact_solution_use_add_idx", + "linear_classifier_cs_scorer_args_exact_solution_use_add_idx_empty_set", + ], +) +def test_classwise_shapley( + linear_classifier_cs_scorer: Utility, + linear_classifier_cs_scorer_args_exact_solution: Tuple[Dict, ValuationResult], + n_samples: int, + n_resample_complement_sets: int, + request, +): + args, exact_solution, check_args = request.getfixturevalue( + linear_classifier_cs_scorer_args_exact_solution + ) + values = compute_classwise_shapley_values( + linear_classifier_cs_scorer, + done=MaxChecks(n_samples - 1), + truncation=NoTruncation(), + n_resample_complement_sets=n_resample_complement_sets, + **args, + progress=True, + ) + check_values(values, exact_solution, **check_args) + assert np.all(values.counts == n_samples * n_resample_complement_sets) From 06769be5ed76a8bab4b6040828e385aa02f71584 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Mon, 28 Aug 2023 02:34:02 +0200 Subject: [PATCH 02/43] Remove setter method from dataset to fix https://github.com/appliedAI-Initiative/pyDVL/pull/338#discussion_r1306497638 --- src/pydvl/utils/dataset.py | 4 ---- src/pydvl/utils/numeric.py | 2 +- src/pydvl/value/shapley/montecarlo.py | 23 +++++++++++++++-------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/pydvl/utils/dataset.py b/src/pydvl/utils/dataset.py index cab59416b..eaeb05b77 100644 --- a/src/pydvl/utils/dataset.py +++ b/src/pydvl/utils/dataset.py @@ -222,10 +222,6 @@ def indices(self): """ return self._indices - @indices.setter - def indices(self, indices: np.ndarray): - self._indices = indices - @property def data_names(self): """Names of each individual datapoint. diff --git a/src/pydvl/utils/numeric.py b/src/pydvl/utils/numeric.py index 5e5904c56..74f6fdcfe 100644 --- a/src/pydvl/utils/numeric.py +++ b/src/pydvl/utils/numeric.py @@ -189,7 +189,7 @@ def random_powerset_group_conditional( rng.shuffle(subset) yield subset else: - yield np.array([]) + yield np.array([], dtype=int) def random_subset_of_size(s: NDArray[T], size: int) -> NDArray[T]: diff --git a/src/pydvl/value/shapley/montecarlo.py b/src/pydvl/value/shapley/montecarlo.py index 4ed258f15..d44bfff26 100644 --- a/src/pydvl/value/shapley/montecarlo.py +++ b/src/pydvl/value/shapley/montecarlo.py @@ -41,7 +41,7 @@ from numpy.typing import NDArray from tqdm import tqdm -from pydvl.utils import Utility, random_powerset_group_conditional +from pydvl.utils import Dataset, Utility, random_powerset_group_conditional from pydvl.utils.config import ParallelConfig from pydvl.utils.numeric import random_powerset from pydvl.utils.parallel import MapReduceJob @@ -261,14 +261,21 @@ def _permutation_montecarlo_shapley_rollout( else u(additional_indices) ) - # hack to calculate the correct value in reset. + truncation_u = u if additional_indices is not None: - old_indices = u.data.indices - u.data.indices = np.sort(np.concatenate((permutation, additional_indices))) - truncation.reset(u) - u.data.indices = old_indices - else: - truncation.reset(u) + # hack to calculate the correct value in reset. + truncation_indices = np.sort(np.concatenate((permutation, additional_indices))) + truncation_u = Utility( + u.model, + Dataset( + u.data.x_train[truncation_indices], + u.data.y_train[truncation_indices], + u.data.x_test, + u.data.y_test, + ), + u.scorer, + ) + truncation.reset(truncation_u) is_terminated = False for i, idx in enumerate(permutation): From d77fbc0b2e2d777ebcd3dd20b30aa5380070e8a2 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Mon, 28 Aug 2023 02:46:02 +0200 Subject: [PATCH 03/43] Moved files to `*classwise.py` file as in https://github.com/appliedAI-Initiative/pyDVL/pull/338#discussion_r1306497793 --- src/pydvl/utils/score.py | 140 +-------------- src/pydvl/value/shapley/classwise.py | 241 +++++++++++++++++++++++-- src/pydvl/value/shapley/montecarlo.py | 245 +++++++++----------------- tests/conftest.py | 3 +- tests/utils/conftest.py | 22 --- tests/utils/test_score.py | 126 +------------ tests/value/conftest.py | 21 +++ tests/value/shapley/test_classwise.py | 123 ++++++++++++- 8 files changed, 456 insertions(+), 465 deletions(-) diff --git a/src/pydvl/utils/score.py b/src/pydvl/utils/score.py index ca02b1ccb..344f97266 100644 --- a/src/pydvl/utils/score.py +++ b/src/pydvl/utils/score.py @@ -17,13 +17,12 @@ import numpy as np from numpy.typing import NDArray from scipy.special import expit -from sklearn.metrics import accuracy_score, get_scorer, make_scorer +from sklearn.metrics import get_scorer from pydvl.utils.types import SupervisedModel __all__ = [ "Scorer", - "ClasswiseScorer", "compose_score", "squashed_r2", "squashed_variance", @@ -87,143 +86,6 @@ def __repr__(self): return f"{capitalized_name} (scorer={self._scorer})" -class ClasswiseScorer(Scorer): - """A Scorer which is applicable for valuation in classification problems. Its value - is based on in-cls and out-of-cls score :footcite:t:`schoch_csshapley_2022`. For - each class ``label`` it separates the elements into two groups, namely in-cls - instances and out-of-cls instances. The value function itself than estimates the - in-cls metric discounted by the out-of-cls metric. In other words the value function - for each element of one class is conditioned on the out-of-cls instances (or a - subset of it). The form of the value function can be written as - - .. math:: - v_{y_i}(D) = f(a_S(D_{y_i}))) * g(a_S(D_{-y_i}))) - - where f and g are continuous, monotonic functions and D is the test set. - - in order to produce meaningful results. For further reference see also section four - of :footcite:t:`schoch_csshapley_2022`. - - :param default: Score used when a model cannot be fit, e.g. when too little data is - passed, or errors arise. - :param range: Numerical range of the score function. Some Monte Carlo methods can - use this to estimate the number of samples required for a certain quality of - approximation. If not provided, it can be read from the ``scoring`` object if it - provides it, for instance if it was constructed with - :func:`~pydvl.utils.types.compose_score`. - :param in_class_discount_fn: Continuous, monotonic increasing function used to - discount the in-class score. - :param out_of_class_discount_fn: Continuous, monotonic increasing function used to - discount the out-of-class score. - :param initial_label: Set initial label (Doesn't require to set parameter ``label`` - on ``ClassWiseDiscountedScorer`` in first iteration) - :param name: Name of the scorer. If not provided, the name of the passed - function will be prefixed by 'classwise '. - - .. versionadded:: 0.7.0 - """ - - def __init__( - self, - scoring: str = "accuracy", - default: float = 0.0, - range: Tuple[float, float] = (-np.inf, np.inf), - in_class_discount_fn: Callable[[float], float] = lambda x: x, - out_of_class_discount_fn: Callable[[float], float] = np.exp, - initial_label: Optional[int] = None, - name: Optional[str] = None, - ): - disc_score_in_cls = in_class_discount_fn(range[1]) - disc_score_out_of_cls = out_of_class_discount_fn(range[1]) - transformed_range = (0, disc_score_in_cls * disc_score_out_of_cls) - super().__init__( - "accuracy", - range=transformed_range, - default=default, - name=name or f"classwise {scoring}", - ) - self._in_cls_discount_fn = in_class_discount_fn - self._out_of_cls_discount_fn = out_of_class_discount_fn - self.label = initial_label - - def __str__(self): - return self._name - - def __call__( - self: "ClasswiseScorer", - model: SupervisedModel, - x_test: NDArray[np.float_], - y_test: NDArray[np.int_], - ) -> float: - """ - :param model: Model used for computing the score on the validation set. - :param x_test: Array containing the features of the classification problem. - :param y_test: Array containing the labels of the classification problem. - :return: Calculated score. - """ - in_cls_score, out_of_cls_score = self.estimate_in_cls_and_out_of_cls_score( - model, x_test, y_test - ) - disc_score_in_cls = self._in_cls_discount_fn(in_cls_score) - disc_score_out_of_cls = self._out_of_cls_discount_fn(out_of_cls_score) - return disc_score_in_cls * disc_score_out_of_cls - - def estimate_in_cls_and_out_of_cls_score( - self, - model: SupervisedModel, - x_test: NDArray[np.float_], - y_test: NDArray[np.int_], - rescale_scores: bool = True, - ) -> Tuple[float, float]: - r""" - Computes in-class and out-of-class scores using the provided scoring function, - which can be expressed as: - - .. math:: - a_S(D=\{(\hat{x}_1, \hat{y}_1), \dots, (\hat{x}_K, \hat{y}_K)\}) &= - \frac{1}{N} \sum_k s(y(\hat{x}_k), \hat{y}_k) - - In this context, the computation is performed twice: once on D_i and once on D_o - to calculate the in-class and out-of-class scores. Here, D_i contains only - samples with the specified 'label' from the validation set, while D_o contains - all other samples. By default, the scores are scaled to have the same order of - magnitude. In such cases, the raw scores are multiplied by: - - .. math:: - N_{y_i} = \frac{a_S(D_{y_i})}{a_S(D_{y_i})+a_S(D_{-y_i})} \quad \text{and} - \quad N_{-y_i} = \frac{a_S(D_{-y_i})}{a_S(D_{y_i})+a_S(D_{-y_i})} - - :param model: Model used for computing the score on the validation set. - :param x_test: Array containing the features of the classification problem. - :param y_test: Array containing the labels of the classification problem. - :param rescale_scores: If set to True, the scores will be denormalized. This is - particularly useful when the inner score is calculated by an estimator of - the form 1/N sum_i x_i. - :return: Tuple containing the in-class and out-of-class scores. - """ - scorer = self._scorer - label_set_match = y_test == self.label - label_set = np.where(label_set_match)[0] - num_classes = len(np.unique(y_test)) - - if len(label_set) == 0: - return 0, 1 / (num_classes - 1) - - complement_label_set = np.where(~label_set_match)[0] - in_cls_score = scorer(model, x_test[label_set], y_test[label_set]) - out_of_cls_score = scorer( - model, x_test[complement_label_set], y_test[complement_label_set] - ) - - if rescale_scores: - n_in_cls = np.count_nonzero(y_test == self.label) - n_out_of_cls = len(y_test) - n_in_cls - in_cls_score *= n_in_cls / (n_in_cls + n_out_of_cls) - out_of_cls_score *= n_out_of_cls / (n_in_cls + n_out_of_cls) - - return in_cls_score, out_of_cls_score - - def compose_score( scorer: Scorer, transformation: Callable[[float], float], diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 9dd9f5ad6..1af42f54d 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -5,32 +5,33 @@ import numbers from concurrent.futures import FIRST_COMPLETED, wait from copy import copy -from typing import cast +from typing import Callable, Optional, Tuple, cast import numpy as np +from numpy._typing import NDArray +from tqdm import tqdm from pydvl.utils import ( ParallelConfig, + Scorer, + SupervisedModel, Utility, effective_n_jobs, init_executor, init_parallel_backend, + random_powerset_group_conditional, ) - -__all__ = [ - "compute_classwise_shapley_values", -] - -from tqdm import tqdm - -from pydvl.utils.score import ClasswiseScorer from pydvl.value.result import ValuationResult -from pydvl.value.shapley.montecarlo import permutation_montecarlo_classwise_shapley +from pydvl.value.shapley.montecarlo import _permutation_montecarlo_shapley_rollout from pydvl.value.shapley.truncated import TruncationPolicy from pydvl.value.stopping import MaxChecks, StoppingCriterion logger = logging.getLogger(__name__) +__all__ = [ + "compute_classwise_shapley_values", +] + def compute_classwise_shapley_values( u: Utility, @@ -249,3 +250,223 @@ def _normalize_classwise_shapley_values( result.scale(in_cls_acc / sigma, indices=indices_label_set) return result + + +class ClasswiseScorer(Scorer): + """A Scorer which is applicable for valuation in classification problems. Its value + is based on in-cls and out-of-cls score :footcite:t:`schoch_csshapley_2022`. For + each class ``label`` it separates the elements into two groups, namely in-cls + instances and out-of-cls instances. The value function itself than estimates the + in-cls metric discounted by the out-of-cls metric. In other words the value function + for each element of one class is conditioned on the out-of-cls instances (or a + subset of it). The form of the value function can be written as + + .. math:: + v_{y_i}(D) = f(a_S(D_{y_i}))) * g(a_S(D_{-y_i}))) + + where f and g are continuous, monotonic functions and D is the test set. + + in order to produce meaningful results. For further reference see also section four + of :footcite:t:`schoch_csshapley_2022`. + + :param default: Score used when a model cannot be fit, e.g. when too little data is + passed, or errors arise. + :param range: Numerical range of the score function. Some Monte Carlo methods can + use this to estimate the number of samples required for a certain quality of + approximation. If not provided, it can be read from the ``scoring`` object if it + provides it, for instance if it was constructed with + :func:`~pydvl.utils.types.compose_score`. + :param in_class_discount_fn: Continuous, monotonic increasing function used to + discount the in-class score. + :param out_of_class_discount_fn: Continuous, monotonic increasing function used to + discount the out-of-class score. + :param initial_label: Set initial label (Doesn't require to set parameter ``label`` + on ``ClassWiseDiscountedScorer`` in first iteration) + :param name: Name of the scorer. If not provided, the name of the passed + function will be prefixed by 'classwise '. + + .. versionadded:: 0.7.0 + """ + + def __init__( + self, + scoring: str = "accuracy", + default: float = 0.0, + range: Tuple[float, float] = (-np.inf, np.inf), + in_class_discount_fn: Callable[[float], float] = lambda x: x, + out_of_class_discount_fn: Callable[[float], float] = np.exp, + initial_label: Optional[int] = None, + name: Optional[str] = None, + ): + disc_score_in_cls = in_class_discount_fn(range[1]) + disc_score_out_of_cls = out_of_class_discount_fn(range[1]) + transformed_range = (0, disc_score_in_cls * disc_score_out_of_cls) + super().__init__( + "accuracy", + range=transformed_range, + default=default, + name=name or f"classwise {scoring}", + ) + self._in_cls_discount_fn = in_class_discount_fn + self._out_of_cls_discount_fn = out_of_class_discount_fn + self.label = initial_label + + def __str__(self): + return self._name + + def __call__( + self: "ClasswiseScorer", + model: SupervisedModel, + x_test: NDArray[np.float_], + y_test: NDArray[np.int_], + ) -> float: + """ + :param model: Model used for computing the score on the validation set. + :param x_test: Array containing the features of the classification problem. + :param y_test: Array containing the labels of the classification problem. + :return: Calculated score. + """ + in_cls_score, out_of_cls_score = self.estimate_in_cls_and_out_of_cls_score( + model, x_test, y_test + ) + disc_score_in_cls = self._in_cls_discount_fn(in_cls_score) + disc_score_out_of_cls = self._out_of_cls_discount_fn(out_of_cls_score) + return disc_score_in_cls * disc_score_out_of_cls + + def estimate_in_cls_and_out_of_cls_score( + self, + model: SupervisedModel, + x_test: NDArray[np.float_], + y_test: NDArray[np.int_], + rescale_scores: bool = True, + ) -> Tuple[float, float]: + r""" + Computes in-class and out-of-class scores using the provided scoring function, + which can be expressed as: + + .. math:: + a_S(D=\{(\hat{x}_1, \hat{y}_1), \dots, (\hat{x}_K, \hat{y}_K)\}) &= + \frac{1}{N} \sum_k s(y(\hat{x}_k), \hat{y}_k) + + In this context, the computation is performed twice: once on D_i and once on D_o + to calculate the in-class and out-of-class scores. Here, D_i contains only + samples with the specified 'label' from the validation set, while D_o contains + all other samples. By default, the scores are scaled to have the same order of + magnitude. In such cases, the raw scores are multiplied by: + + .. math:: + N_{y_i} = \frac{a_S(D_{y_i})}{a_S(D_{y_i})+a_S(D_{-y_i})} \quad \text{and} + \quad N_{-y_i} = \frac{a_S(D_{-y_i})}{a_S(D_{y_i})+a_S(D_{-y_i})} + + :param model: Model used for computing the score on the validation set. + :param x_test: Array containing the features of the classification problem. + :param y_test: Array containing the labels of the classification problem. + :param rescale_scores: If set to True, the scores will be denormalized. This is + particularly useful when the inner score is calculated by an estimator of + the form 1/N sum_i x_i. + :return: Tuple containing the in-class and out-of-class scores. + """ + scorer = self._scorer + label_set_match = y_test == self.label + label_set = np.where(label_set_match)[0] + num_classes = len(np.unique(y_test)) + + if len(label_set) == 0: + return 0, 1 / (num_classes - 1) + + complement_label_set = np.where(~label_set_match)[0] + in_cls_score = scorer(model, x_test[label_set], y_test[label_set]) + out_of_cls_score = scorer( + model, x_test[complement_label_set], y_test[complement_label_set] + ) + + if rescale_scores: + n_in_cls = np.count_nonzero(y_test == self.label) + n_out_of_cls = len(y_test) - n_in_cls + in_cls_score *= n_in_cls / (n_in_cls + n_out_of_cls) + out_of_cls_score *= n_out_of_cls / (n_in_cls + n_out_of_cls) + + return in_cls_score, out_of_cls_score + + +def permutation_montecarlo_classwise_shapley( + u: Utility, + label: int, + *, + done: StoppingCriterion, + truncation: TruncationPolicy, + use_default_scorer_value: bool = True, + min_elements_per_label: int = 1, +) -> ValuationResult: + """ + Samples a random subset of the complement set and computes the truncated Monte Carlo + estimator. + + :param u: Utility object containing model, data, and scoring function. The scoring + function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + :param done: Function checking whether computation needs to stop. + :param label: The label for which to sample the complement (e.g. all other labels) + :param truncation: Callable which decides whether to interrupt processing a + permutation and set all subsequent marginals to zero. + :param use_default_scorer_value: Use default scorer value even if additional_indices + is not None. + :param min_elements_per_label: The minimum number of elements for each opposite + label. + :return: ValuationResult object containing computed data values. + """ + + algorithm_name = "classwise_shapley" + result = ValuationResult.zeros( + algorithm="classwise_shapley", + indices=u.data.indices, + data_names=u.data.data_names, + ) + + _, y_train = u.data.get_training_data(u.data.indices) + class_indices_set, class_complement_indices_set = split_indices_by_label( + u.data.indices, + y_train, + label, + ) + _, complement_y_train = u.data.get_training_data(class_complement_indices_set) + indices_permutation = np.random.permutation(class_indices_set) + + for subset_idx, subset_complement in enumerate( + random_powerset_group_conditional( + class_complement_indices_set, + complement_y_train, + min_elements_per_group=min_elements_per_label, + ) + ): + result += _permutation_montecarlo_shapley_rollout( + u, + indices_permutation, + additional_indices=subset_complement, + truncation=truncation, + algorithm_name=algorithm_name, + use_default_scorer_value=use_default_scorer_value, + ) + if done(result): + break + + return result + + +def split_indices_by_label( + indices: NDArray[np.int_], labels: NDArray[np.int_], label: int +) -> Tuple[NDArray[np.int_], NDArray[np.int_]]: + """ + Splits the indices into two sets based on the value of ``label``: those samples + with and without that label. + + :param indices: The indices to be used for referring to the data. + :param labels: Corresponding labels for the indices. + :param label: Label to be used for splitting. + :return: Tuple with two sets of indices. + """ + active_elements = labels == label + class_indices_set = np.where(active_elements)[0] + class_complement_indices_set = np.where(~active_elements)[0] + class_indices_set = indices[class_indices_set] + class_complement_indices_set = indices[class_complement_indices_set] + return class_indices_set, class_complement_indices_set diff --git a/src/pydvl/value/shapley/montecarlo.py b/src/pydvl/value/shapley/montecarlo.py index d44bfff26..9ca8c4471 100644 --- a/src/pydvl/value/shapley/montecarlo.py +++ b/src/pydvl/value/shapley/montecarlo.py @@ -35,13 +35,13 @@ import operator from functools import reduce from itertools import cycle, takewhile -from typing import Optional, Sequence, Tuple +from typing import Optional, Sequence import numpy as np from numpy.typing import NDArray from tqdm import tqdm -from pydvl.utils import Dataset, Utility, random_powerset_group_conditional +from pydvl.utils import Dataset from pydvl.utils.config import ParallelConfig from pydvl.utils.numeric import random_powerset from pydvl.utils.parallel import MapReduceJob @@ -54,7 +54,6 @@ __all__ = [ "permutation_montecarlo_shapley", - "permutation_montecarlo_classwise_shapley", "combinatorial_montecarlo_shapley", ] @@ -148,153 +147,6 @@ def permutation_montecarlo_shapley( return map_reduce_job() -def permutation_montecarlo_classwise_shapley( - u: Utility, - label: int, - *, - done: StoppingCriterion, - truncation: TruncationPolicy, - use_default_scorer_value: bool = True, - min_elements_per_label: int = 1, -) -> ValuationResult: - """ - Samples a random subset of the complement set and computes the truncated Monte Carlo - estimator. - - :param u: Utility object containing model, data, and scoring function. The scoring - function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. - :param done: Function checking whether computation needs to stop. - :param label: The label for which to sample the complement (e.g. all other labels) - :param truncation: Callable which decides whether to interrupt processing a - permutation and set all subsequent marginals to zero. - :param use_default_scorer_value: Use default scorer value even if additional_indices - is not None. - :param min_elements_per_label: The minimum number of elements for each opposite - label. - :return: ValuationResult object containing computed data values. - """ - - algorithm_name = "classwise_shapley" - result = ValuationResult.zeros( - algorithm="classwise_shapley", - indices=u.data.indices, - data_names=u.data.data_names, - ) - - _, y_train = u.data.get_training_data(u.data.indices) - class_indices_set, class_complement_indices_set = split_indices_by_label( - u.data.indices, - y_train, - label, - ) - _, complement_y_train = u.data.get_training_data(class_complement_indices_set) - indices_permutation = np.random.permutation(class_indices_set) - - for subset_idx, subset_complement in enumerate( - random_powerset_group_conditional( - class_complement_indices_set, - complement_y_train, - min_elements_per_group=min_elements_per_label, - ) - ): - result += _permutation_montecarlo_shapley_rollout( - u, - indices_permutation, - additional_indices=subset_complement, - truncation=truncation, - algorithm_name=algorithm_name, - use_default_scorer_value=use_default_scorer_value, - ) - if done(result): - break - - return result - - -def _permutation_montecarlo_shapley_rollout( - u: Utility, - permutation: NDArray[np.int_], - *, - truncation: TruncationPolicy, - algorithm_name: str, - additional_indices: Optional[NDArray[np.int_]] = None, - use_default_scorer_value: bool = True, -) -> ValuationResult: - """ - A truncated version of a permutation-based MC estimator for classwise Shapley - values. It generates a permutation p[i] of the class label indices and iterates over - all subsets starting from the empty set to the full set of indices. - - :param u: Utility object containing model, data, and scoring function. The scoring - function should to be of type :class:`~pydvl.utils.score.ClassWiseScorer`. - :param permutation: Permutation of indices to be considered. - :param truncation: Callable which decides whether to interrupt processing a - permutation and set all subsequent marginals to zero. - :param additional_indices: Set of additional indices for data points which should be - always considered. - :param use_default_scorer_value: Use default scorer value even if additional_indices - is not None. - :return: ValuationResult object containing computed data values. - """ - if ( - additional_indices is not None - and len(np.intersect1d(permutation, additional_indices)) > 0 - ): - raise ValueError( - "The class label set and the complement set have to be disjoint." - ) - - result = ValuationResult.zeros( - algorithm=algorithm_name, - indices=u.data.indices, - data_names=u.data.data_names, - ) - - prev_score = ( - u.default_score - if ( - use_default_scorer_value - or additional_indices is None - or additional_indices is not None - and len(additional_indices) == 0 - ) - else u(additional_indices) - ) - - truncation_u = u - if additional_indices is not None: - # hack to calculate the correct value in reset. - truncation_indices = np.sort(np.concatenate((permutation, additional_indices))) - truncation_u = Utility( - u.model, - Dataset( - u.data.x_train[truncation_indices], - u.data.y_train[truncation_indices], - u.data.x_test, - u.data.y_test, - ), - u.scorer, - ) - truncation.reset(truncation_u) - - is_terminated = False - for i, idx in enumerate(permutation): - if is_terminated or (is_terminated := truncation(i, prev_score)): - score = prev_score - else: - score = u( - np.concatenate((permutation[: i + 1], additional_indices)) - if additional_indices is not None and len(additional_indices) > 0 - else permutation[: i + 1] - ) - - marginal = score - prev_score - result.update(idx, marginal) - prev_score = score - - return result - - def _combinatorial_montecarlo_shapley( indices: Sequence[int], u: Utility, @@ -391,21 +243,84 @@ def combinatorial_montecarlo_shapley( return map_reduce_job() -def split_indices_by_label( - indices: NDArray[np.int_], labels: NDArray[np.int_], label: int -) -> Tuple[NDArray[np.int_], NDArray[np.int_]]: +def _permutation_montecarlo_shapley_rollout( + u: Utility, + permutation: NDArray[np.int_], + *, + truncation: TruncationPolicy, + algorithm_name: str, + additional_indices: Optional[NDArray[np.int_]] = None, + use_default_scorer_value: bool = True, +) -> ValuationResult: """ - Splits the indices into two sets based on the value of ``label``: those samples - with and without that label. + A truncated version of a permutation-based MC estimator. + values. It generates a permutation p[i] of the class label indices and iterates over + all subsets starting from the empty set to the full set of indices. - :param indices: The indices to be used for referring to the data. - :param labels: Corresponding labels for the indices. - :param label: Label to be used for splitting. - :return: Tuple with two sets of indices. + :param u: Utility object containing model, data, and scoring function. + :param permutation: Permutation of indices to be considered. + :param truncation: Callable which decides whether to interrupt processing a + permutation and set all subsequent marginals to zero. + :param additional_indices: Set of additional indices for data points which should be + always considered. + :param use_default_scorer_value: Use default scorer value even if additional_indices + is not None. + :return: ValuationResult object containing computed data values. """ - active_elements = labels == label - class_indices_set = np.where(active_elements)[0] - class_complement_indices_set = np.where(~active_elements)[0] - class_indices_set = indices[class_indices_set] - class_complement_indices_set = indices[class_complement_indices_set] - return class_indices_set, class_complement_indices_set + if ( + additional_indices is not None + and len(np.intersect1d(permutation, additional_indices)) > 0 + ): + raise ValueError( + "The class label set and the complement set have to be disjoint." + ) + + result = ValuationResult.zeros( + algorithm=algorithm_name, + indices=u.data.indices, + data_names=u.data.data_names, + ) + + prev_score = ( + u.default_score + if ( + use_default_scorer_value + or additional_indices is None + or additional_indices is not None + and len(additional_indices) == 0 + ) + else u(additional_indices) + ) + + truncation_u = u + if additional_indices is not None: + # hack to calculate the correct value in reset. + truncation_indices = np.sort(np.concatenate((permutation, additional_indices))) + truncation_u = Utility( + u.model, + Dataset( + u.data.x_train[truncation_indices], + u.data.y_train[truncation_indices], + u.data.x_test, + u.data.y_test, + ), + u.scorer, + ) + truncation.reset(truncation_u) + + is_terminated = False + for i, idx in enumerate(permutation): + if is_terminated or (is_terminated := truncation(i, prev_score)): + score = prev_score + else: + score = u( + np.concatenate((permutation[: i + 1], additional_indices)) + if additional_indices is not None and len(additional_indices) > 0 + else permutation[: i + 1] + ) + + marginal = score - prev_score + result.update(idx, marginal) + prev_score = score + + return result diff --git a/tests/conftest.py b/tests/conftest.py index d03779214..b325c7fa8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,8 +11,9 @@ from sklearn import datasets from sklearn.utils import Bunch -from pydvl.utils import ClasswiseScorer, Dataset, MemcachedClientConfig, Utility +from pydvl.utils import Dataset, MemcachedClientConfig, Utility from pydvl.utils.parallel.backend import available_cpus +from pydvl.value.shapley.classwise import ClasswiseScorer from tests.misc import ClosedFormLinearClassifier if TYPE_CHECKING: diff --git a/tests/utils/conftest.py b/tests/utils/conftest.py index 5783aba8a..f64330777 100644 --- a/tests/utils/conftest.py +++ b/tests/utils/conftest.py @@ -1,9 +1,5 @@ -from typing import Dict, Tuple - -import numpy as np import pytest import ray -from numpy.typing import NDArray from ray.cluster_utils import Cluster from pydvl.utils.config import ParallelConfig @@ -27,21 +23,3 @@ def parallel_config(request, num_workers): yield ParallelConfig(backend="ray", address=cluster.address) ray.shutdown() cluster.shutdown() - - -@pytest.fixture(scope="function") -def dataset_alt_seq_simple( - request, -) -> Tuple[NDArray[np.float_], NDArray[np.int_], Dict[str, float]]: - """ - The label set is represented as 0000011100011111, with adjustable left and right - margins. The left margin denotes the percentage of zeros at the beginning, while the - right margin denotes the percentage of ones at the end. Accuracy can be efficiently - calculated using a closed-form solution. - """ - n_element, left_margin, right_margin = request.param - x = np.linspace(0, 1, n_element) - y = ((left_margin <= x) & (x < 0.5)) | ((1 - right_margin) <= x) - y = y.astype(int) - x = np.expand_dims(x, -1) - return x, y, {"left_margin": left_margin, "right_margin": right_margin} diff --git a/tests/utils/test_score.py b/tests/utils/test_score.py index 15bd91d1e..078775240 100644 --- a/tests/utils/test_score.py +++ b/tests/utils/test_score.py @@ -1,19 +1,7 @@ -from typing import Dict, Tuple, cast - import numpy as np -import pandas as pd -import pytest from numpy.typing import NDArray -from pydvl.utils import Utility, powerset -from pydvl.utils.score import ( - ClasswiseScorer, - Scorer, - compose_score, - squashed_r2, - squashed_variance, -) -from tests.misc import ThresholdClassifier +from pydvl.utils.score import Scorer, compose_score, squashed_r2, squashed_variance sigmoid = lambda x: 1 / (1 + np.exp(-x)) @@ -81,115 +69,3 @@ def test_squashed_variance(): X = np.array([[1, 2], [3, 4]]) model = FittedLinearModel(coef) assert sigmoid(1.0) == squashed_variance(model, X, X @ coef) - - -@pytest.mark.parametrize( - "dataset_alt_seq_simple", - [((101, 0.3, 0.4))], - indirect=True, -) -def test_cs_scorer_on_dataset_alt_seq_simple(dataset_alt_seq_simple): - """ - Tests the class wise scorer. - """ - - scorer = ClasswiseScorer("accuracy", initial_label=0) - assert str(scorer) == "classwise accuracy" - assert repr(scorer) == "ClasswiseAccuracy (scorer=make_scorer(accuracy_score))" - - x, y, info = dataset_alt_seq_simple - n_element = len(x) - target_in_cls_acc_0 = (info["left_margin"] * 100 + 1) / n_element - target_out_of_cls_acc_0 = (info["right_margin"] * 100 + 1) / n_element - - model = ThresholdClassifier() - in_cls_acc_0, out_of_cls_acc_0 = scorer.estimate_in_cls_and_out_of_cls_score( - model, x, y - ) - assert np.isclose(in_cls_acc_0, target_in_cls_acc_0) - assert np.isclose(out_of_cls_acc_0, target_out_of_cls_acc_0) - - scorer.label = 1 - in_cls_acc_1, out_of_cls_acc_1 = scorer.estimate_in_cls_and_out_of_cls_score( - model, x, y - ) - assert in_cls_acc_1 == out_of_cls_acc_0 - assert in_cls_acc_0 == out_of_cls_acc_1 - - scorer.label = 0 - value = scorer(model, x, y) - assert np.isclose(value, in_cls_acc_0 * np.exp(out_of_cls_acc_0)) - - scorer.label = 1 - value = scorer(model, x, y) - assert np.isclose(value, in_cls_acc_1 * np.exp(out_of_cls_acc_1)) - - -def test_cs_scorer_on_alt_seq_cf_linear_classifier_cs_score( - linear_classifier_cs_scorer: Utility, -): - subsets_zero = list(powerset(np.array((0, 1)))) - subsets_one = list(powerset(np.array((2, 3)))) - subsets_zero = [tuple(s) for s in subsets_zero] - subsets_one = [tuple(s) for s in subsets_one] - target_betas = pd.DataFrame( - [ - [np.nan, 1 / 3, 1 / 4, 7 / 25], - [0, 3 / 10, 4 / 17, 7 / 26], - [0, 3 / 13, 1 / 5, 7 / 29], - [0, 3 / 14, 4 / 21, 7 / 30], - ], - index=subsets_zero, - columns=subsets_one, - ) - target_accuracies_zero = pd.DataFrame( - [ - [0, 1 / 4, 1 / 4, 1 / 4], - [3 / 4, 1 / 4, 1 / 2, 1 / 4], - [3 / 4, 1 / 2, 1 / 2, 1 / 2], - [3 / 4, 1 / 2, 1 / 2, 1 / 2], - ], - index=subsets_zero, - columns=subsets_one, - ) - target_accuracies_one = pd.DataFrame( - [ - [0, 1 / 4, 1 / 4, 1 / 4], - [0, 1 / 4, 1 / 4, 1 / 4], - [0, 1 / 4, 1 / 4, 1 / 4], - [0, 1 / 4, 1 / 4, 1 / 4], - ], - index=subsets_zero, - columns=subsets_one, - ) - model = linear_classifier_cs_scorer.model - scorer = cast(ClasswiseScorer, linear_classifier_cs_scorer.scorer) - scorer.label = 0 - - for set_zero_idx in range(len(subsets_zero)): - for set_one_idx in range(len(subsets_one)): - indices = list(subsets_zero[set_zero_idx] + subsets_one[set_one_idx]) - ( - x_train, - y_train, - ) = linear_classifier_cs_scorer.data.get_training_data(indices) - linear_classifier_cs_scorer.model.fit(x_train, y_train) - fitted_beta = linear_classifier_cs_scorer.model._beta # noqa - target_beta = target_betas.iloc[set_zero_idx, set_one_idx] - assert ( - np.isnan(fitted_beta) - if np.isnan(target_beta) - else fitted_beta == target_beta - ) - - ( - x_test, - y_test, - ) = linear_classifier_cs_scorer.data.get_test_data() - in_cls_acc_0, in_cls_acc_1 = scorer.estimate_in_cls_and_out_of_cls_score( - model, x_test, y_test - ) - assert ( - in_cls_acc_0 == target_accuracies_zero.iloc[set_zero_idx, set_one_idx] - ) - assert in_cls_acc_1 == target_accuracies_one.iloc[set_zero_idx, set_one_idx] diff --git a/tests/value/conftest.py b/tests/value/conftest.py index f806b3bb6..1520b1df3 100644 --- a/tests/value/conftest.py +++ b/tests/value/conftest.py @@ -1,6 +1,9 @@ +from typing import Dict, Tuple + import numpy as np import pytest import ray +from numpy._typing import NDArray from numpy.typing import NDArray from ray.cluster_utils import Cluster from sklearn.linear_model import LinearRegression @@ -136,3 +139,21 @@ def parallel_config(request): yield ParallelConfig(backend="ray", address=cluster.address) ray.shutdown() cluster.shutdown() + + +@pytest.fixture(scope="function") +def dataset_alt_seq_simple( + request, +) -> Tuple[NDArray[np.float_], NDArray[np.int_], Dict[str, float]]: + """ + The label set is represented as 0000011100011111, with adjustable left and right + margins. The left margin denotes the percentage of zeros at the beginning, while the + right margin denotes the percentage of ones at the end. Accuracy can be efficiently + calculated using a closed-form solution. + """ + n_element, left_margin, right_margin = request.param + x = np.linspace(0, 1, n_element) + y = ((left_margin <= x) & (x < 0.5)) | ((1 - right_margin) <= x) + y = y.astype(int) + x = np.expand_dims(x, -1) + return x, y, {"left_margin": left_margin, "right_margin": right_margin} diff --git a/tests/value/shapley/test_classwise.py b/tests/value/shapley/test_classwise.py index 1d263a7d3..3b607af53 100644 --- a/tests/value/shapley/test_classwise.py +++ b/tests/value/shapley/test_classwise.py @@ -3,15 +3,20 @@ """ import random from random import seed -from typing import Dict, Tuple +from typing import Dict, Tuple, cast import numpy as np +import pandas as pd import pytest -from pydvl.utils import Utility +from pydvl.utils import Utility, powerset from pydvl.value import MaxChecks, ValuationResult -from pydvl.value.shapley.classwise import compute_classwise_shapley_values +from pydvl.value.shapley.classwise import ( + ClasswiseScorer, + compute_classwise_shapley_values, +) from pydvl.value.shapley.truncated import NoTruncation +from tests.misc import ThresholdClassifier from tests.value import check_values @@ -784,3 +789,115 @@ def test_classwise_shapley( ) check_values(values, exact_solution, **check_args) assert np.all(values.counts == n_samples * n_resample_complement_sets) + + +@pytest.mark.parametrize( + "dataset_alt_seq_simple", + [((101, 0.3, 0.4))], + indirect=True, +) +def test_cs_scorer_on_dataset_alt_seq_simple(dataset_alt_seq_simple): + """ + Tests the class wise scorer. + """ + + scorer = ClasswiseScorer("accuracy", initial_label=0) + assert str(scorer) == "classwise accuracy" + assert repr(scorer) == "ClasswiseAccuracy (scorer=make_scorer(accuracy_score))" + + x, y, info = dataset_alt_seq_simple + n_element = len(x) + target_in_cls_acc_0 = (info["left_margin"] * 100 + 1) / n_element + target_out_of_cls_acc_0 = (info["right_margin"] * 100 + 1) / n_element + + model = ThresholdClassifier() + in_cls_acc_0, out_of_cls_acc_0 = scorer.estimate_in_cls_and_out_of_cls_score( + model, x, y + ) + assert np.isclose(in_cls_acc_0, target_in_cls_acc_0) + assert np.isclose(out_of_cls_acc_0, target_out_of_cls_acc_0) + + scorer.label = 1 + in_cls_acc_1, out_of_cls_acc_1 = scorer.estimate_in_cls_and_out_of_cls_score( + model, x, y + ) + assert in_cls_acc_1 == out_of_cls_acc_0 + assert in_cls_acc_0 == out_of_cls_acc_1 + + scorer.label = 0 + value = scorer(model, x, y) + assert np.isclose(value, in_cls_acc_0 * np.exp(out_of_cls_acc_0)) + + scorer.label = 1 + value = scorer(model, x, y) + assert np.isclose(value, in_cls_acc_1 * np.exp(out_of_cls_acc_1)) + + +def test_cs_scorer_on_alt_seq_cf_linear_classifier_cs_score( + linear_classifier_cs_scorer: Utility, +): + subsets_zero = list(powerset(np.array((0, 1)))) + subsets_one = list(powerset(np.array((2, 3)))) + subsets_zero = [tuple(s) for s in subsets_zero] + subsets_one = [tuple(s) for s in subsets_one] + target_betas = pd.DataFrame( + [ + [np.nan, 1 / 3, 1 / 4, 7 / 25], + [0, 3 / 10, 4 / 17, 7 / 26], + [0, 3 / 13, 1 / 5, 7 / 29], + [0, 3 / 14, 4 / 21, 7 / 30], + ], + index=subsets_zero, + columns=subsets_one, + ) + target_accuracies_zero = pd.DataFrame( + [ + [0, 1 / 4, 1 / 4, 1 / 4], + [3 / 4, 1 / 4, 1 / 2, 1 / 4], + [3 / 4, 1 / 2, 1 / 2, 1 / 2], + [3 / 4, 1 / 2, 1 / 2, 1 / 2], + ], + index=subsets_zero, + columns=subsets_one, + ) + target_accuracies_one = pd.DataFrame( + [ + [0, 1 / 4, 1 / 4, 1 / 4], + [0, 1 / 4, 1 / 4, 1 / 4], + [0, 1 / 4, 1 / 4, 1 / 4], + [0, 1 / 4, 1 / 4, 1 / 4], + ], + index=subsets_zero, + columns=subsets_one, + ) + model = linear_classifier_cs_scorer.model + scorer = cast(ClasswiseScorer, linear_classifier_cs_scorer.scorer) + scorer.label = 0 + + for set_zero_idx in range(len(subsets_zero)): + for set_one_idx in range(len(subsets_one)): + indices = list(subsets_zero[set_zero_idx] + subsets_one[set_one_idx]) + ( + x_train, + y_train, + ) = linear_classifier_cs_scorer.data.get_training_data(indices) + linear_classifier_cs_scorer.model.fit(x_train, y_train) + fitted_beta = linear_classifier_cs_scorer.model._beta # noqa + target_beta = target_betas.iloc[set_zero_idx, set_one_idx] + assert ( + np.isnan(fitted_beta) + if np.isnan(target_beta) + else fitted_beta == target_beta + ) + + ( + x_test, + y_test, + ) = linear_classifier_cs_scorer.data.get_test_data() + in_cls_acc_0, in_cls_acc_1 = scorer.estimate_in_cls_and_out_of_cls_score( + model, x_test, y_test + ) + assert ( + in_cls_acc_0 == target_accuracies_zero.iloc[set_zero_idx, set_one_idx] + ) + assert in_cls_acc_1 == target_accuracies_one.iloc[set_zero_idx, set_one_idx] From 078e7230bc01529b36389d3acc790c20e9cd8861 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Mon, 28 Aug 2023 02:51:27 +0200 Subject: [PATCH 04/43] Optimize stopping criterion as described in https://github.com/appliedAI-Initiative/pyDVL/pull/338#discussion_r1306500057. --- src/pydvl/value/shapley/classwise.py | 2 +- src/pydvl/value/stopping.py | 20 ++++++++++++-------- tests/value/shapley/test_classwise.py | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 1af42f54d..95a49d322 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -175,7 +175,7 @@ def _classwise_shapley_one_step( result += permutation_montecarlo_classwise_shapley( u, label, - done=MaxChecks(n_resample_complement_sets - 1), + done=MaxChecks(n_resample_complement_sets), truncation=truncation, use_default_scorer_value=use_default_scorer_value, min_elements_per_label=min_elements_per_label, diff --git a/src/pydvl/value/stopping.py b/src/pydvl/value/stopping.py index b235d2067..7270f717f 100644 --- a/src/pydvl/value/stopping.py +++ b/src/pydvl/value/stopping.py @@ -220,9 +220,9 @@ def completion(self) -> float: class AbsoluteStandardError(StoppingCriterion): r"""Determine convergence based on the standard error of the values. - If $s_i$ is the standard error for datum $i$ and $v_i$ its value, then this - criterion returns :attr:`~pydvl.utils.status.Status.Converged` if - $s_i < \epsilon$ for all $i$ and a threshold value $\epsilon \gt 0$. + If $s_i$ is the standard error for datum $i$, then this criterion returns + :attr:`~pydvl.utils.status.Status.Converged` if $s_i < \epsilon$ for all $i$ + and a threshold value $\epsilon \gt 0$. :param threshold: A value is considered to have converged if the standard error is below this value. A way of choosing it is to pick some @@ -279,21 +279,21 @@ class MaxChecks(StoppingCriterion): def __init__(self, n_checks: Optional[int], modify_result: bool = True): super().__init__(modify_result=modify_result) - if n_checks is not None and n_checks < 0: - raise ValueError("n_iterations must be at least 0 or None") + if n_checks is not None and n_checks < 1: + raise ValueError("n_iterations must be at least 1 or None") self.n_checks = n_checks self._count = 0 def _check(self, result: ValuationResult) -> Status: - if self.n_checks is not None: + if self.n_checks: self._count += 1 - if self._count > self.n_checks: + if self._count >= self.n_checks: self._converged = np.ones_like(result.values, dtype=bool) return Status.Converged return Status.Pending def completion(self) -> float: - if self.n_checks is not None: + if self.n_checks: return min(1.0, self._count / self.n_checks) return 0.0 @@ -302,6 +302,10 @@ class MaxUpdates(StoppingCriterion): """Terminate if any number of value updates exceeds or equals the given threshold. + .. note:: + If you want to ensure that **all** values have been updated, you probably + want :class:`MinUpdates` instead. + This checks the ``counts`` field of a :class:`~pydvl.value.result.ValuationResult`, i.e. the number of times that each index has been updated. For powerset samplers, the maximum of this diff --git a/tests/value/shapley/test_classwise.py b/tests/value/shapley/test_classwise.py index 3b607af53..c92ff6e34 100644 --- a/tests/value/shapley/test_classwise.py +++ b/tests/value/shapley/test_classwise.py @@ -781,7 +781,7 @@ def test_classwise_shapley( ) values = compute_classwise_shapley_values( linear_classifier_cs_scorer, - done=MaxChecks(n_samples - 1), + done=MaxChecks(n_samples), truncation=NoTruncation(), n_resample_complement_sets=n_resample_complement_sets, **args, From 684217cb410f6a2b442ead76fcb0452822c70efd Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Mon, 28 Aug 2023 02:53:56 +0200 Subject: [PATCH 05/43] Fix https://github.com/appliedAI-Initiative/pyDVL/pull/338#discussion_r1306500428 --- src/pydvl/value/shapley/truncated.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pydvl/value/shapley/truncated.py b/src/pydvl/value/shapley/truncated.py index 2945c95bf..19de69265 100644 --- a/src/pydvl/value/shapley/truncated.py +++ b/src/pydvl/value/shapley/truncated.py @@ -118,12 +118,11 @@ def __init__(self, u: Utility, rtol: float): def _check(self, idx: int, score: float) -> bool: return np.allclose(score, self.total_utility, rtol=self.rtol) - def reset(self, u: Optional[Utility] = None) -> float: + def reset(self, u: Optional[Utility] = None): if u is None: u = self._u self.total_utility = u(u.data.indices) - return self.total_utility class BootstrapTruncation(TruncationPolicy): From 4237e9460ba22d55d8709818b62e9354c03905be Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Mon, 28 Aug 2023 02:57:59 +0200 Subject: [PATCH 06/43] Move files from tests/misc.py file to test_classwise.py --- tests/conftest.py | 25 +----------- tests/misc.py | 36 ---------------- tests/value/shapley/test_classwise.py | 59 ++++++++++++++++++++++++++- 3 files changed, 58 insertions(+), 62 deletions(-) delete mode 100644 tests/misc.py diff --git a/tests/conftest.py b/tests/conftest.py index b325c7fa8..41244d275 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,10 +11,8 @@ from sklearn import datasets from sklearn.utils import Bunch -from pydvl.utils import Dataset, MemcachedClientConfig, Utility +from pydvl.utils import Dataset, MemcachedClientConfig from pydvl.utils.parallel.backend import available_cpus -from pydvl.value.shapley.classwise import ClasswiseScorer -from tests.misc import ClosedFormLinearClassifier if TYPE_CHECKING: from _pytest.config import Config @@ -413,24 +411,3 @@ def pytest_terminal_summary( ): tolerate_session = terminalreporter.config._tolerate_session tolerate_session.display(terminalreporter) - - -@pytest.fixture(scope="function") -def dataset_alt_seq_full() -> Dataset: - x_train = np.arange(1, 5).reshape([-1, 1]) - y_train = np.array([0, 0, 1, 1]) - x_test = x_train - y_test = np.array([0, 0, 0, 1]) - return Dataset(x_train, y_train, x_test, y_test) - - -@pytest.fixture(scope="function") -def linear_classifier_cs_scorer( - dataset_alt_seq_full: Dataset, -) -> Utility: - return Utility( - ClosedFormLinearClassifier(), - dataset_alt_seq_full, - ClasswiseScorer("accuracy"), - catch_errors=False, - ) diff --git a/tests/misc.py b/tests/misc.py deleted file mode 100644 index 2d12fb673..000000000 --- a/tests/misc.py +++ /dev/null @@ -1,36 +0,0 @@ -import numpy as np -from numpy._typing import NDArray - - -class ThresholdClassifier: - def fit(self, x: NDArray, y: NDArray) -> float: - raise NotImplementedError("Mock model") - - def predict(self, x: NDArray) -> NDArray: - y = 0.5 < x - return y[:, 0].astype(int) - - def score(self, x: NDArray, y: NDArray) -> float: - raise NotImplementedError("Mock model") - - -class ClosedFormLinearClassifier: - def __init__(self): - self._beta = None - - def fit(self, x: NDArray, y: NDArray) -> float: - v = x[:, 0] - self._beta = np.dot(v, y) / np.dot(v, v) - return -1 - - def predict(self, x: NDArray) -> NDArray: - if self._beta is None: - raise AttributeError("Model not fitted") - - x = x[:, 0] - probs = self._beta * x - return np.clip(np.round(probs + 1e-10), 0, 1).astype(int) - - def score(self, x: NDArray, y: NDArray) -> float: - pred_y = self.predict(x) - return np.sum(pred_y == y) / 4 diff --git a/tests/value/shapley/test_classwise.py b/tests/value/shapley/test_classwise.py index c92ff6e34..773f63d2c 100644 --- a/tests/value/shapley/test_classwise.py +++ b/tests/value/shapley/test_classwise.py @@ -8,15 +8,15 @@ import numpy as np import pandas as pd import pytest +from numpy._typing import NDArray -from pydvl.utils import Utility, powerset +from pydvl.utils import Dataset, Utility, powerset from pydvl.value import MaxChecks, ValuationResult from pydvl.value.shapley.classwise import ( ClasswiseScorer, compute_classwise_shapley_values, ) from pydvl.value.shapley.truncated import NoTruncation -from tests.misc import ThresholdClassifier from tests.value import check_values @@ -901,3 +901,58 @@ def test_cs_scorer_on_alt_seq_cf_linear_classifier_cs_score( in_cls_acc_0 == target_accuracies_zero.iloc[set_zero_idx, set_one_idx] ) assert in_cls_acc_1 == target_accuracies_one.iloc[set_zero_idx, set_one_idx] + + +class ThresholdClassifier: + def fit(self, x: NDArray, y: NDArray) -> float: + raise NotImplementedError("Mock model") + + def predict(self, x: NDArray) -> NDArray: + y = 0.5 < x + return y[:, 0].astype(int) + + def score(self, x: NDArray, y: NDArray) -> float: + raise NotImplementedError("Mock model") + + +class ClosedFormLinearClassifier: + def __init__(self): + self._beta = None + + def fit(self, x: NDArray, y: NDArray) -> float: + v = x[:, 0] + self._beta = np.dot(v, y) / np.dot(v, v) + return -1 + + def predict(self, x: NDArray) -> NDArray: + if self._beta is None: + raise AttributeError("Model not fitted") + + x = x[:, 0] + probs = self._beta * x + return np.clip(np.round(probs + 1e-10), 0, 1).astype(int) + + def score(self, x: NDArray, y: NDArray) -> float: + pred_y = self.predict(x) + return np.sum(pred_y == y) / 4 + + +@pytest.fixture(scope="function") +def linear_classifier_cs_scorer( + dataset_alt_seq_full: Dataset, +) -> Utility: + return Utility( + ClosedFormLinearClassifier(), + dataset_alt_seq_full, + ClasswiseScorer("accuracy"), + catch_errors=False, + ) + + +@pytest.fixture(scope="function") +def dataset_alt_seq_full() -> Dataset: + x_train = np.arange(1, 5).reshape([-1, 1]) + y_train = np.array([0, 0, 1, 1]) + x_test = x_train + y_test = np.array([0, 0, 0, 1]) + return Dataset(x_train, y_train, x_test, y_test) From cbeb5695bef93c7e158f695cf2f707766960cd17 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Mon, 28 Aug 2023 02:59:28 +0200 Subject: [PATCH 07/43] Fix https://github.com/appliedAI-Initiative/pyDVL/pull/338#discussion_r1306502017 Co-authored-by: Miguel de Benito Delgado --- src/pydvl/value/shapley/classwise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 95a49d322..5ffc64f2f 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -66,7 +66,7 @@ def compute_classwise_shapley_values( label. :param n_jobs: Number of parallel jobs to run. :param config: Parallel configuration. - :param progress: Whether to display progress bars for each job. + :param progress: Whether to display a progress bar. :return: ValuationResult object containing computed data values. """ From d8e218b14ef59ed616d35de6944a5474d1695795 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Mon, 28 Aug 2023 03:01:09 +0200 Subject: [PATCH 08/43] Remove processing details from comment. --- src/pydvl/value/shapley/classwise.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 5ffc64f2f..9b18efc2d 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -47,9 +47,9 @@ def compute_classwise_shapley_values( progress: bool = False, ) -> ValuationResult: """ - Computes the classwise Shapley value by parallel processing. Independent workers - are spawned to process the data in parallel. Once the data is aggregated, the values - can be optionally normalized, depending on ``normalize_values``. + Computes the classwise Shapley values as described in + footcite:t:`schoch_csshapley_2022`. The values can be optionally normalized, + depending on ``normalize_values``. :param u: Utility object containing model, data, and scoring function. The scoring function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. From 375c20061533d4cd07daa00ad12af06b5d39f584 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Mon, 28 Aug 2023 03:10:24 +0200 Subject: [PATCH 09/43] Make _permutation_montecarlo_classwise_shapley private. --- src/pydvl/value/shapley/classwise.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 9b18efc2d..d40346c6d 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -172,7 +172,7 @@ def _classwise_shapley_one_step( for label in unique_labels: u.scorer.label = label - result += permutation_montecarlo_classwise_shapley( + result += _permutation_montecarlo_classwise_shapley( u, label, done=MaxChecks(n_resample_complement_sets), @@ -389,7 +389,7 @@ def estimate_in_cls_and_out_of_cls_score( return in_cls_score, out_of_cls_score -def permutation_montecarlo_classwise_shapley( +def _permutation_montecarlo_classwise_shapley( u: Utility, label: int, *, From 9df9a9b33601dea70ed17693c8f6cfecbdc2667c Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Fri, 1 Sep 2023 22:16:34 +0200 Subject: [PATCH 10/43] Refactor executor pattern. --- src/pydvl/value/shapley/classwise.py | 34 +++++++--------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 9b1c5f5c9..07fc5c5cb 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -3,9 +3,9 @@ """ import logging import numbers -from concurrent.futures import FIRST_COMPLETED, wait +from concurrent.futures import FIRST_COMPLETED, Future, wait from copy import copy -from typing import Callable, Optional, Tuple, cast +from typing import Callable, Optional, Set, Tuple, cast import numpy as np from numpy.typing import NDArray @@ -72,10 +72,7 @@ def compute_classwise_shapley_values( parallel_backend = init_parallel_backend(config) u_ref = parallel_backend.put(u) - # This represents the number of jobs that are running n_jobs = effective_n_jobs(n_jobs, config) - # This determines the total number of submitted jobs - # including the ones that are running n_submitted_jobs = 2 * n_jobs pbar = tqdm(disable=not progress, position=0, total=100, unit="%") @@ -86,20 +83,8 @@ def compute_classwise_shapley_values( ) terminate_exec = False with init_executor(max_workers=n_jobs, config=config) as executor: - futures = set() - # Initial batch of computations - for _ in range(n_submitted_jobs): - future = executor.submit( - _classwise_shapley_one_step, - u_ref, - truncation=truncation, - n_resample_complement_sets=n_resample_complement_sets, - use_default_scorer_value=use_default_scorer_value, - min_elements_per_label=min_elements_per_label, - ) - futures.add(future) - while futures: - # Wait for the next futures to complete. + futures: Set[Future] = set() + while True: completed_futures, futures = wait( futures, timeout=60, return_when=FIRST_COMPLETED ) @@ -114,12 +99,9 @@ def compute_classwise_shapley_values( if terminate_exec: break - # Submit more computations - # The goal is to always have `n_jobs` - # computations running for _ in range(n_submitted_jobs - len(futures)): future = executor.submit( - _classwise_shapley_one_step, + _permutation_montecarlo_classwise_shapley, u_ref, truncation=truncation, n_resample_complement_sets=n_resample_complement_sets, @@ -135,7 +117,7 @@ def compute_classwise_shapley_values( return result -def _classwise_shapley_one_step( +def _permutation_montecarlo_classwise_shapley( u: Utility, *, truncation: TruncationPolicy, @@ -170,7 +152,7 @@ def _classwise_shapley_one_step( for label in unique_labels: u.scorer.label = label - result += _permutation_montecarlo_classwise_shapley( + result += _permutation_montecarlo_classwise_shapley_for_label( u, label, done=MaxChecks(n_resample_complement_sets), @@ -387,7 +369,7 @@ def estimate_in_cls_and_out_of_cls_score( return in_cls_score, out_of_cls_score -def _permutation_montecarlo_classwise_shapley( +def _permutation_montecarlo_classwise_shapley_for_label( u: Utility, label: int, *, From 382552fdc2499e89b579786daa14faee3f6b4167 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Fri, 1 Sep 2023 22:39:49 +0200 Subject: [PATCH 11/43] Remove writeable_array function. --- src/pydvl/utils/dataset.py | 5 +++++ src/pydvl/utils/util.py | 14 -------------- src/pydvl/value/result.py | 9 ++------- 3 files changed, 7 insertions(+), 21 deletions(-) delete mode 100644 src/pydvl/utils/util.py diff --git a/src/pydvl/utils/dataset.py b/src/pydvl/utils/dataset.py index 14c6b7edc..12a123806 100644 --- a/src/pydvl/utils/dataset.py +++ b/src/pydvl/utils/dataset.py @@ -429,6 +429,11 @@ def __init__( def __len__(self): return len(self.groups) + @property + def indices(self): + """Indices of the groups.""" + return self._indices + # FIXME this is a misnomer, should be `names` in `Dataset` so that here it # makes sense @property diff --git a/src/pydvl/utils/util.py b/src/pydvl/utils/util.py deleted file mode 100644 index d556b4d28..000000000 --- a/src/pydvl/utils/util.py +++ /dev/null @@ -1,14 +0,0 @@ -import numpy as np -from numpy.typing import NDArray - - -def arr_or_writeable_copy(arr: NDArray) -> NDArray: - """Return a copy of ``arr`` if it's not writeable, otherwise return ``arr``. - - :param arr: Array to copy if it's not writeable. - :return: Copy of ``arr`` if it's not writeable, otherwise ``arr``. - """ - if not arr.flags.writeable: - return np.copy(arr) - - return arr diff --git a/src/pydvl/value/result.py b/src/pydvl/value/result.py index cede7bf5f..76cd9110a 100644 --- a/src/pydvl/value/result.py +++ b/src/pydvl/value/result.py @@ -71,7 +71,6 @@ from pydvl.utils.dataset import Dataset from pydvl.utils.numeric import running_moments from pydvl.utils.status import Status -from pydvl.utils.util import arr_or_writeable_copy try: import pandas # Try to import here for the benefit of mypy @@ -244,12 +243,8 @@ def __init__( self._algorithm = algorithm self._status = Status(status) # Just in case we are given a string - self._values = arr_or_writeable_copy(values) - self._variances = ( - np.zeros_like(values) - if variances is None - else arr_or_writeable_copy(variances) - ) + self._values = values + self._variances = np.zeros_like(values) if variances is None else variances self._counts = np.ones_like(values) if counts is None else counts self._sort_order = None self._extra_values = extra_values or {} From 5869e6ebc391ebaddcd801b87319ae2380bacee2 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Fri, 1 Sep 2023 22:50:16 +0200 Subject: [PATCH 12/43] Fix comments. --- src/pydvl/utils/numeric.py | 19 ++++++++++++------- src/pydvl/value/result.py | 6 ++++-- src/pydvl/value/shapley/classwise.py | 13 ++++++++----- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/pydvl/utils/numeric.py b/src/pydvl/utils/numeric.py index afd19694a..a933647b7 100644 --- a/src/pydvl/utils/numeric.py +++ b/src/pydvl/utils/numeric.py @@ -17,6 +17,7 @@ Optional, Tuple, TypeVar, + Union, cast, overload, ) @@ -151,14 +152,18 @@ def random_powerset_group_conditional( that in each sampled set, each unique group is represented at least ``min_elements`` times. The groups are specified as integers for all elements of the set separately. - :param s: Vector of size N representing the set to sample elements from. - :param groups: Vector of size N containing the group as an integer for each element. - :param min_elements_per_group: The minimum number of elements for each group. + Args: + s: Vector of size N representing the set to sample elements from. + groups: Vector of size N containing the group as an integer for each element. + min_elements_per_group: The minimum number of elements for each group. + + Returns: + Generated draw from the power set of s with ``min_elements`` of each group. - :return: Generated draw from the power set of s with ``min_elements`` of each group. - :raises: TypeError: If the data ``s`` or ``groups`` is not a NumPy array. - :raises: ValueError: If the length of ``s``and ``groups`` different or - ``min_elements`` is smaller than 0. + Raises: + TypeError: If the data ``s`` or ``groups`` is not a NumPy array. + ValueError: If the length of ``s``and ``groups`` different or ``min_elements`` + is smaller than 0. """ if not isinstance(s, np.ndarray): raise TypeError("Set must be an NDArray") diff --git a/src/pydvl/value/result.py b/src/pydvl/value/result.py index 76cd9110a..e77f1665c 100644 --- a/src/pydvl/value/result.py +++ b/src/pydvl/value/result.py @@ -635,8 +635,10 @@ def update(self, idx: int, new_value: float) -> "ValuationResult": def scale(self, coefficient: float, indices: Optional[NDArray[IndexT]] = None): """ Scales the values and variances of the result by a coefficient. - :param coefficient: Coefficient to scale by. - :param indices: Indices to scale. If None, all values are scaled. + + Args: + coefficient: Coefficient to scale by. + indices: Indices to scale. If None, all values are scaled. """ self._values[self._sort_positions[indices]] *= coefficient self._variances[self._sort_positions[indices]] *= coefficient**2 diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 07fc5c5cb..c158b30b6 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -168,11 +168,14 @@ def _check_classwise_shapley_utility(u: Utility): """ Verifies if the provided utility object supports classwise Shapley values. - :param u: Utility object containing model, data, and scoring function. The scoring - function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. - :raises: ValueError: If ``u.data`` is not a classification problem. - :raises: ValueError: If ``u.scorer`` is not an instance of - :class:`~pydvl.utils.score.ClassWiseScorer` + Args: + u: Utility object containing model, data, and scoring function. The scoring + function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + + Raises: + ValueError: If ``u.data`` is not a classification problem. + ValueError: If ``u.scorer`` is not an instance of + :class:`~pydvl.utils.score.ClassWiseScorer` """ dim_correct = u.data.y_train.ndim == 1 and u.data.y_test.ndim == 1 From e64c63759ece3f63ad29f111840f2d3218710347 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Fri, 1 Sep 2023 23:31:13 +0200 Subject: [PATCH 13/43] Add short documentation. --- docs/value/shapley.md | 44 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/value/shapley.md b/docs/value/shapley.md index 77af2ae2b..f5765545f 100644 --- a/docs/value/shapley.md +++ b/docs/value/shapley.md @@ -78,6 +78,50 @@ stop condition. This is an instance of a [MaxTime][pydvl.value.stopping.MaxTime] and [AbsoluteStandardError][pydvl.value.stopping.AbsoluteStandardError]. +### Classwise Shapley + +**Classwise Shapley** [@schoch_csshapley_2022] is different Shapley schema applicable +for classification problems. The key insight is that samples can be beneficial +for overall performance, while being detrimental for their own class. CS-Shapley changes +the utility to account for this effect by decomposing it into a product of two +functions. It is the in-class accuracy multiplied by a discounter out-of-class accuracy. +The value is defined as: + +$$ +v_u(x_i) \approx \frac{1}{K \cdot L} +\sum_{S^{(k)}_{-y_i} \subseteq T_{-y_i} \setminus \{i\}} +\sum_{\sigma^{(l)} \in \Pi(T_{y_i} \setminus \{i\})} +[u( \sigma_{\colon i} \cup \{i\} | S_{-y_i} ) +− u( \sigma_{\colon i} | S_{-y_i})] +$$ + +where $K$ is the number of subsets $S^{(k)}_{-y_i}$ sampled from the class complement +set $T_{-y_i}$ of class c and $L$ is the number of permutations sampled from the class +indices set $T_{y_i}$. The scoring function used has the form + +$$u(S_{y_i}|S_{-y_i}) = a_S(D_{y_i}))) \exp\{a_S(D_{-y_i}))\}.$$ + +This can be further customised, but that form is shown by the authors to have certain +desirable properties. + + +```python +from pydvl.utils import Dataset, Utility +from pydvl.value import compute_classwise_shapley_values, ClasswiseScorer, HistoryDeviation +from pydvl.value.shapley.truncated import RelativeTruncation + +model = ... +scoring = ClasswiseScorer("accuracy") +data = Dataset(...) +utility = Utility(model, data, scoring) +values = compute_classwise_shapley_values( + utility, + done=HistoryDeviation(n_steps=500, rtol=1e-3), + truncation=RelativeTruncation(utility, rtol=0.01), + n_resample_complement_sets=10, + normalize_values=True +) +``` ### Owen sampling From 33bdf6f360f43b80d647184a2f71fcfabd718c3f Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Fri, 1 Sep 2023 23:50:34 +0200 Subject: [PATCH 14/43] Refactor stopping criterion and solve type issue. --- src/pydvl/value/shapley/classwise.py | 131 +++++++++++++++----------- tests/utils/test_numeric.py | 8 +- tests/value/shapley/test_classwise.py | 11 +-- 3 files changed, 84 insertions(+), 66 deletions(-) diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index c158b30b6..627c0ae45 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -24,7 +24,7 @@ from pydvl.value.result import ValuationResult from pydvl.value.shapley.montecarlo import permutation_montecarlo_shapley_rollout from pydvl.value.shapley.truncated import TruncationPolicy -from pydvl.value.stopping import MaxChecks, StoppingCriterion +from pydvl.value.stopping import MaxChecks, MaxUpdates, StoppingCriterion logger = logging.getLogger(__name__) @@ -36,8 +36,8 @@ def compute_classwise_shapley_values( *, done: StoppingCriterion, truncation: TruncationPolicy, + done_sample_complements: Optional[StoppingCriterion] = None, normalize_values: bool = True, - n_resample_complement_sets: int = 1, use_default_scorer_value: bool = True, min_elements_per_label: int = 1, n_jobs: int = 1, @@ -49,23 +49,32 @@ def compute_classwise_shapley_values( footcite:t:`schoch_csshapley_2022`. The values can be optionally normalized, depending on ``normalize_values``. - :param u: Utility object containing model, data, and scoring function. The scoring - function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. - :param done: Function that checks whether the computation needs to stop. - :param truncation: Callable function that decides whether to interrupt processing a - permutation and set subsequent marginals to zero. - :param normalize_values: Indicates whether to normalize the values by the variation - in each class times their in-class accuracy. - :param n_resample_complement_sets: Number of times to resample the complement set - for each permutation. - :param use_default_scorer_value: Use default scorer value even if additional_indices - is not None. - :param min_elements_per_label: The minimum number of elements for each opposite - label. - :param n_jobs: Number of parallel jobs to run. - :param config: Parallel configuration. - :param progress: Whether to display a progress bar. - :return: ValuationResult object containing computed data values. + Args: + u: Utility object containing model, data, and scoring function. The scoring + function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + done: Function that checks whether the computation needs to stop. + truncation: Callable function that decides whether to interrupt processing a + permutation and set subsequent marginals to zero. + done_sample_complements: Function checking whether computation needs to stop. + Otherwise, it will resample conditional sets until the stopping criterion is + met. + normalize_values: Indicates whether to normalize the values by the variation + in each class times their in-class accuracy. + done_sample_complements: Number of times to resample the complement set + for each permutation. + use_default_scorer_value: The first set of indices is the sampled complement + set. Unless not otherwise specified, the default scorer value is used for + this. If it is set to false, the base score is calculated from the utility. + min_elements_per_label: The minimum number of elements for each opposite + label. + n_jobs: Number of parallel jobs to run. + config: Parallel configuration. + progress: Whether to display a progress bar. + + Returns: + ValuationResult object containing computed data values. + + !!! tip "New in version 0.7.0" """ _check_classwise_shapley_utility(u) @@ -104,7 +113,7 @@ def compute_classwise_shapley_values( _permutation_montecarlo_classwise_shapley, u_ref, truncation=truncation, - n_resample_complement_sets=n_resample_complement_sets, + done_sample_complements=done_sample_complements, use_default_scorer_value=use_default_scorer_value, min_elements_per_label=min_elements_per_label, ) @@ -120,26 +129,34 @@ def compute_classwise_shapley_values( def _permutation_montecarlo_classwise_shapley( u: Utility, *, + done_sample_complements: StoppingCriterion = None, truncation: TruncationPolicy, - n_resample_complement_sets: int = 1, use_default_scorer_value: bool = True, min_elements_per_label: int = 1, ) -> ValuationResult: """Computes classwise Shapley value using truncated Monte Carlo permutation sampling for the subsets. - :param u: Utility object containing model, data, and scoring function. The scoring - function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. - :param truncation: Callable function that decides whether to interrupt processing a - permutation and set subsequent marginals to zero. - :param n_resample_complement_sets: Number of times to resample the complement set - for each permutation. - :param use_default_scorer_value: Use default scorer value even if additional_indices - is not None. - :param min_elements_per_label: The minimum number of elements for each opposite - label. - :return: ValuationResult object containing computed data values. + Args: + u: Utility object containing model, data, and scoring function. The scoring + function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + done_sample_complements: Function checking whether computation needs to stop. + Otherwise, it will resample conditional sets until the stopping criterion is + met. + truncation: Callable function that decides whether to interrupt processing a + permutation and set subsequent marginals to zero. + use_default_scorer_value: The first set of indices is the sampled complement + set. Unless not otherwise specified, the default scorer value is used for + this. If it is set to false, the base score is calculated from the utility. + min_elements_per_label: The minimum number of elements for each opposite + label. + + Returns: + ValuationResult object containing computed data values. """ + if done_sample_complements is None: + done_sample_complements = MaxChecks(1) + result = ValuationResult.zeros( algorithm="classwise_shapley", indices=u.data.indices, @@ -155,7 +172,7 @@ def _permutation_montecarlo_classwise_shapley( result += _permutation_montecarlo_classwise_shapley_for_label( u, label, - done=MaxChecks(n_resample_complement_sets), + done=done_sample_complements, truncation=truncation, use_default_scorer_value=use_default_scorer_value, min_elements_per_label=min_elements_per_label, @@ -208,9 +225,13 @@ def _normalize_classwise_shapley_values( belonging to the currently viewed class. See footcite:t:`schoch_csshapley_2022` for more details. - :param result: ValuationResult object to be normalized. - :param u: Utility object containing model, data, and scoring function. The scoring - function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + Args: + result: ValuationResult object to be normalized. + u: Utility object containing model, data, and scoring function. The scoring + function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + + Returns: + Normalized ValuationResult object. """ y_train = u.data.y_train unique_labels = np.unique(np.concatenate((y_train, u.data.y_test))) @@ -252,23 +273,24 @@ class ClasswiseScorer(Scorer): in order to produce meaningful results. For further reference see also section four of :footcite:t:`schoch_csshapley_2022`. - :param default: Score used when a model cannot be fit, e.g. when too little data is - passed, or errors arise. - :param range: Numerical range of the score function. Some Monte Carlo methods can - use this to estimate the number of samples required for a certain quality of - approximation. If not provided, it can be read from the ``scoring`` object if it - provides it, for instance if it was constructed with - :func:`~pydvl.utils.types.compose_score`. - :param in_class_discount_fn: Continuous, monotonic increasing function used to - discount the in-class score. - :param out_of_class_discount_fn: Continuous, monotonic increasing function used to - discount the out-of-class score. - :param initial_label: Set initial label (Doesn't require to set parameter ``label`` - on ``ClassWiseDiscountedScorer`` in first iteration) - :param name: Name of the scorer. If not provided, the name of the passed - function will be prefixed by 'classwise '. - - .. versionadded:: 0.7.0 + Args: + default: Score used when a model cannot be fit, e.g. when too little data is + passed, or errors arise. + range: Numerical range of the score function. Some Monte Carlo methods can + use this to estimate the number of samples required for a certain quality of + approximation. If not provided, it can be read from the ``scoring`` object + if it provides it, for instance if it was constructed with + :func:`~pydvl.utils.types.compose_score`. + in_class_discount_fn: Continuous, monotonic increasing function used to + discount the in-class score. + out_of_class_discount_fn: Continuous, monotonic increasing function used to + discount the out-of-class score. + initial_label: Set initial label (Doesn't require to set parameter ``label`` + on ``ClassWiseDiscountedScorer`` in first iteration) + name: Name of the scorer. If not provided, the name of the passed + function will be prefixed by 'classwise '. + + !!! tip "New in version 0.7.0" """ def __init__( @@ -387,7 +409,8 @@ def _permutation_montecarlo_classwise_shapley_for_label( :param u: Utility object containing model, data, and scoring function. The scoring function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. - :param done: Function checking whether computation needs to stop. + :param done: Function checking whether computation needs to stop. Otherwise, it will + resample conditional sets until the stopping criterion is met. :param label: The label for which to sample the complement (e.g. all other labels) :param truncation: Callable which decides whether to interrupt processing a permutation and set all subsequent marginals to zero. diff --git a/tests/utils/test_numeric.py b/tests/utils/test_numeric.py index 5b3d2582e..9c9841337 100644 --- a/tests/utils/test_numeric.py +++ b/tests/utils/test_numeric.py @@ -141,10 +141,10 @@ def test_running_moments(): assert np.allclose(variances, true_variances) -@pytest.mark.parametrize("min_elements", [1, 2]) -@pytest.mark.parametrize("elements_per_group", [10]) -@pytest.mark.parametrize("num_groups", [3]) -@pytest.mark.parametrize("check_num_samples", [10]) +@pytest.mark.parametrize( + "min_elements,elements_per_group,num_groups,check_num_samples", + [(1, 10, 3, 3), (2, 10, 3, 3)], +) def test_random_powerset_group_conditional( min_elements: int, elements_per_group: int, diff --git a/tests/value/shapley/test_classwise.py b/tests/value/shapley/test_classwise.py index 77baca787..140ab6ddb 100644 --- a/tests/value/shapley/test_classwise.py +++ b/tests/value/shapley/test_classwise.py @@ -781,7 +781,7 @@ def test_classwise_shapley( linear_classifier_cs_scorer, done=MaxChecks(n_samples), truncation=NoTruncation(), - n_resample_complement_sets=n_resample_complement_sets, + done_sample_complements=MaxChecks(n_resample_complement_sets), **args, progress=True, ) @@ -789,11 +789,7 @@ def test_classwise_shapley( assert np.all(values.counts == n_samples * n_resample_complement_sets) -@pytest.mark.parametrize( - "dataset_alt_seq_simple", - [((101, 0.3, 0.4))], - indirect=True, -) +@pytest.mark.parametrize("n_element, left_margin, right_margin", [(101, 0.3, 0.4)]) def test_cs_scorer_on_dataset_alt_seq_simple(dataset_alt_seq_simple): """ Tests the class wise scorer. @@ -958,7 +954,7 @@ def dataset_alt_seq_full() -> Dataset: @pytest.fixture(scope="function") def dataset_alt_seq_simple( - request, + n_element: int, left_margin: float, right_margin: float ) -> Tuple[NDArray[np.float_], NDArray[np.int_], Dict[str, float]]: """ The label set is represented as 0000011100011111, with adjustable left and right @@ -966,7 +962,6 @@ def dataset_alt_seq_simple( right margin denotes the percentage of ones at the end. Accuracy can be efficiently calculated using a closed-form solution. """ - n_element, left_margin, right_margin = request.param x = np.linspace(0, 1, n_element) y = ((left_margin <= x) & (x < 0.5)) | ((1 - right_margin) <= x) y = y.astype(int) From f4c9024a12526453f435297381ad9a405506c204 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Sat, 2 Sep 2023 01:30:03 +0200 Subject: [PATCH 15/43] Fix test case regarding max checks. --- tests/value/test_stopping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/value/test_stopping.py b/tests/value/test_stopping.py index 0c876fbf7..1f732a1e7 100644 --- a/tests/value/test_stopping.py +++ b/tests/value/test_stopping.py @@ -193,6 +193,6 @@ def test_max_checks(): assert not done(v) done = MaxChecks(5) - for _ in range(5): + for _ in range(4): assert not done(v) assert done(v) From 62c0f09c866bf0d73b1abdf750b2a1e47a52a9b7 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Sat, 2 Sep 2023 13:49:25 +0200 Subject: [PATCH 16/43] Move cs-shap specific function to its file --- src/pydvl/utils/numeric.py | 4 +- src/pydvl/value/shapley/classwise.py | 94 ++++++++++++++++++++++++++-- 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/src/pydvl/utils/numeric.py b/src/pydvl/utils/numeric.py index b0fe3f08c..fdb5bca60 100644 --- a/src/pydvl/utils/numeric.py +++ b/src/pydvl/utils/numeric.py @@ -160,9 +160,9 @@ def random_powerset_group_conditional( s: NDArray[T], groups: NDArray[np.int_], min_elements_per_group: int = 1, - seed: Optional[Seed] = None + seed: Optional[Seed] = None, ) -> Generator[NDArray[T], None, None]: - """ Draws random group-conditional subsets from `s`. + """Draws random group-conditional subsets from `s`. It is ensured that in each sampled set, each unique group is represented at least `min_elements` times. The groups are specified as integers for all diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 627c0ae45..adb9b452a 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -1,5 +1,5 @@ """ -Implementation of the algorithm footcite:t:`schoch_csshapley_2022`. +Implementation of Class-wise Shapley, introduced in [@schoch_csshapley_2022]. """ import logging import numbers @@ -12,6 +12,7 @@ from tqdm import tqdm from pydvl.utils import ( + Dataset, ParallelConfig, Scorer, SupervisedModel, @@ -22,9 +23,8 @@ random_powerset_group_conditional, ) from pydvl.value.result import ValuationResult -from pydvl.value.shapley.montecarlo import permutation_montecarlo_shapley_rollout from pydvl.value.shapley.truncated import TruncationPolicy -from pydvl.value.stopping import MaxChecks, MaxUpdates, StoppingCriterion +from pydvl.value.stopping import MaxChecks, StoppingCriterion logger = logging.getLogger(__name__) @@ -444,7 +444,7 @@ def _permutation_montecarlo_classwise_shapley_for_label( min_elements_per_group=min_elements_per_label, ) ): - result += permutation_montecarlo_shapley_rollout( + result += _permutation_montecarlo_shapley_rollout( u, indices_permutation, additional_indices=subset_complement, @@ -458,6 +458,92 @@ def _permutation_montecarlo_classwise_shapley_for_label( return result +def _permutation_montecarlo_shapley_rollout( + u: Utility, + permutation: NDArray[np.int_], + truncation: TruncationPolicy, + algorithm_name: str, + additional_indices: Optional[NDArray[np.int_]] = None, + use_default_scorer_value: bool = True, +) -> ValuationResult: + """ + A truncated version of a permutation-based MC estimator. + values. It generates a permutation p[i] of the class label indices and iterates over + all subsets starting from the empty set to the full set of indices. + + Args: + u: Utility object containing model, data, and scoring function. + permutation: Permutation of indices to be considered. + truncation: Callable which decides whether to interrupt processing a + permutation and set all subsequent marginals to zero. + algorithm_name: For the results object. Used internally by different + variants of Shapley using this subroutine + additional_indices: Set of additional indices for data points which should be + always considered. + use_default_scorer_value: Use default scorer value even if additional_indices + is not None. + Returns: + ValuationResult object containing computed data values. + """ + if ( + additional_indices is not None + and len(np.intersect1d(permutation, additional_indices)) > 0 + ): + raise ValueError( + "The class label set and the complement set have to be disjoint." + ) + + result = ValuationResult.zeros( + algorithm=algorithm_name, + indices=u.data.indices, + data_names=u.data.data_names, + ) + + prev_score = ( + u.default_score + if ( + use_default_scorer_value + or additional_indices is None + or additional_indices is not None + and len(additional_indices) == 0 + ) + else u(additional_indices) + ) + + truncation_u = u + if additional_indices is not None: + # hack to calculate the correct value in reset. + truncation_indices = np.sort(np.concatenate((permutation, additional_indices))) + truncation_u = Utility( + u.model, + Dataset( + u.data.x_train[truncation_indices], + u.data.y_train[truncation_indices], + u.data.x_test, + u.data.y_test, + ), + u.scorer, + ) + truncation.reset(truncation_u) + + is_terminated = False + for i, idx in enumerate(permutation): + if is_terminated or (is_terminated := truncation(i, prev_score)): + score = prev_score + else: + score = u( + np.concatenate((permutation[: i + 1], additional_indices)) + if additional_indices is not None and len(additional_indices) > 0 + else permutation[: i + 1] + ) + + marginal = score - prev_score + result.update(idx, marginal) + prev_score = score + + return result + + def split_indices_by_label( indices: NDArray[np.int_], labels: NDArray[np.int_], label: int ) -> Tuple[NDArray[np.int_], NDArray[np.int_]]: From bfce98438559c544d57759f9ae82471c0a8b36f1 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Sat, 2 Sep 2023 16:59:45 +0200 Subject: [PATCH 17/43] Fix docstring --- src/pydvl/value/shapley/truncated.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/pydvl/value/shapley/truncated.py b/src/pydvl/value/shapley/truncated.py index 0583bd158..d4b783e9b 100644 --- a/src/pydvl/value/shapley/truncated.py +++ b/src/pydvl/value/shapley/truncated.py @@ -90,12 +90,14 @@ def reset(self, u: Optional[Utility] = None): class FixedTruncation(TruncationPolicy): """Break a permutation after computing a fixed number of marginals. - The experiments in Appendix B of (Ghorbani and Zou, 2019)1 - show that when the training set size is large enough, one can simply truncate the iteration - over permutations after a fixed number of steps. This happens because beyond - a certain number of samples in a training set, the model becomes insensitive - to new ones. Alas, this strongly depends on the data distribution and the - model and there is no automatic way of estimating this number. + The experiments in Appendix B of + (Ghorbani and Zou, 2019)1 + show that when the training set size is large enough, one can simply + truncate the iteration over permutations after a fixed number of steps. This + happens because beyond a certain number of samples in a training set, the + model becomes insensitive to new ones. Alas, this strongly depends on the + data distribution and the model and there is no automatic way of estimating + this number. Args: u: Utility object with model, data, and scoring function From b4f50fc3ca6367ee2fbc71e305dac0b3c867ff9c Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Sat, 2 Sep 2023 17:49:03 +0200 Subject: [PATCH 18/43] Fix reference --- src/pydvl/value/shapley/classwise.py | 23 +++++++++++++++++------ tests/value/shapley/test_classwise.py | 2 +- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index adb9b452a..04b0b7381 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -1,5 +1,15 @@ """ -Implementation of Class-wise Shapley, introduced in [@schoch_csshapley_2022]. +Implementation of Class-wise Shapley, introduced in (Schoch, Haifeng and Ji, +2022)[^1]. + +# References + +[^1]: Schoch, Stephanie, Haifeng Xu, and + Yangfeng Ji. [CS-Shapley: Class-Wise Shapley Values for Data Valuation in + Classification](https://openreview.net/forum?id=KTOcrOR5mQ9). In Proc. of + the Thirty-Sixth Conference on Neural Information Processing Systems + (NeurIPS). New Orleans, Louisiana, USA, 2022. + """ import logging import numbers @@ -45,13 +55,14 @@ def compute_classwise_shapley_values( progress: bool = False, ) -> ValuationResult: """ - Computes the classwise Shapley values as described in - footcite:t:`schoch_csshapley_2022`. The values can be optionally normalized, - depending on ``normalize_values``. + Computes the class-wise Shapley values as described in (Schoch, Haifeng and + Ji, 2022)1. + The values can be optionally normalized, depending on `normalize_values`. Args: - u: Utility object containing model, data, and scoring function. The scoring - function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + u: Utility object containing model, data, and scoring function. The + scorer must be of type + [ClassWiseScorer][pydvl.value.shapley.classwise.ClasswiseScorer]. done: Function that checks whether the computation needs to stop. truncation: Callable function that decides whether to interrupt processing a permutation and set subsequent marginals to zero. diff --git a/tests/value/shapley/test_classwise.py b/tests/value/shapley/test_classwise.py index 140ab6ddb..923abbeac 100644 --- a/tests/value/shapley/test_classwise.py +++ b/tests/value/shapley/test_classwise.py @@ -6,7 +6,7 @@ import numpy as np import pandas as pd import pytest -from numpy._typing import NDArray +from numpy.typing import NDArray from pydvl.utils import Dataset, Utility, powerset from pydvl.value import MaxChecks, ValuationResult From 587a56bcc0973958d7a53e7588252a01f828ed8a Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Mon, 4 Sep 2023 03:27:59 +0200 Subject: [PATCH 19/43] Add seed parameter to class wise shapley. --- src/pydvl/utils/numeric.py | 7 +++-- src/pydvl/value/shapley/classwise.py | 42 +++++++++++++++++++++------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/pydvl/utils/numeric.py b/src/pydvl/utils/numeric.py index fdb5bca60..3ec8677e8 100644 --- a/src/pydvl/utils/numeric.py +++ b/src/pydvl/utils/numeric.py @@ -173,8 +173,7 @@ def random_powerset_group_conditional( groups: Vector of size N containing the group as an integer for each element. min_elements_per_group: The minimum number of elements for each group. - seed: Either an instance of a numpy random number generator or a seed - for it. + seed: Either an instance of a numpy random number generator or a seed for it. Returns: Generated draw from the powerset of s with `min_elements` of each group. @@ -211,7 +210,9 @@ def random_powerset_group_conditional( ) ) if subset_length > 0: - subsets.append(random_subset_of_size(s[label_indices], subset_length)) + subsets.append( + random_subset_of_size(s[label_indices], subset_length, seed=rng) + ) if len(subsets) > 0: subset = np.concatenate(tuple(subsets)) diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 04b0b7381..be28047b5 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -25,9 +25,11 @@ Dataset, ParallelConfig, Scorer, + Seed, SupervisedModel, Utility, effective_n_jobs, + ensure_seed_sequence, init_executor, init_parallel_backend, random_powerset_group_conditional, @@ -53,6 +55,7 @@ def compute_classwise_shapley_values( n_jobs: int = 1, config: ParallelConfig = ParallelConfig(), progress: bool = False, + seed: Optional[Seed] = None, ) -> ValuationResult: """ Computes the class-wise Shapley values as described in (Schoch, Haifeng and @@ -81,6 +84,7 @@ def compute_classwise_shapley_values( n_jobs: Number of parallel jobs to run. config: Parallel configuration. progress: Whether to display a progress bar. + seed: Either an instance of a numpy random number generator or a seed for it. Returns: ValuationResult object containing computed data values. @@ -102,11 +106,13 @@ def compute_classwise_shapley_values( data_names=u.data.data_names, ) terminate_exec = False + seed_sequence = ensure_seed_sequence(seed) + with init_executor(max_workers=n_jobs, config=config) as executor: - futures: Set[Future] = set() + pending: Set[Future] = set() while True: - completed_futures, futures = wait( - futures, timeout=60, return_when=FIRST_COMPLETED + completed_futures, pending = wait( + pending, timeout=60, return_when=FIRST_COMPLETED ) for future in completed_futures: accumulated_result += future.result() @@ -119,7 +125,9 @@ def compute_classwise_shapley_values( if terminate_exec: break - for _ in range(n_submitted_jobs - len(futures)): + n_remaining_slots = n_submitted_jobs - len(pending) + seeds = seed_sequence.spawn(n_remaining_slots) + for i in range(n_remaining_slots): future = executor.submit( _permutation_montecarlo_classwise_shapley, u_ref, @@ -127,8 +135,9 @@ def compute_classwise_shapley_values( done_sample_complements=done_sample_complements, use_default_scorer_value=use_default_scorer_value, min_elements_per_label=min_elements_per_label, + seed=seeds[i], ) - futures.add(future) + pending.add(future) result = accumulated_result if normalize_values: @@ -144,6 +153,7 @@ def _permutation_montecarlo_classwise_shapley( truncation: TruncationPolicy, use_default_scorer_value: bool = True, min_elements_per_label: int = 1, + seed: Optional[Seed] = None, ) -> ValuationResult: """Computes classwise Shapley value using truncated Monte Carlo permutation sampling for the subsets. @@ -161,6 +171,7 @@ def _permutation_montecarlo_classwise_shapley( this. If it is set to false, the base score is calculated from the utility. min_elements_per_label: The minimum number of elements for each opposite label. + seed: Either an instance of a numpy random number generator or a seed for it. Returns: ValuationResult object containing computed data values. @@ -173,6 +184,7 @@ def _permutation_montecarlo_classwise_shapley( indices=u.data.indices, data_names=u.data.data_names, ) + rng = np.random.default_rng(seed) x_train, y_train = u.data.get_training_data(u.data.indices) unique_labels = np.unique(y_train) scorer = cast(ClasswiseScorer, copy(u.scorer)) @@ -187,6 +199,7 @@ def _permutation_montecarlo_classwise_shapley( truncation=truncation, use_default_scorer_value=use_default_scorer_value, min_elements_per_label=min_elements_per_label, + seed=rng, ) return result @@ -337,10 +350,13 @@ def __call__( y_test: NDArray[np.int_], ) -> float: """ - :param model: Model used for computing the score on the validation set. - :param x_test: Array containing the features of the classification problem. - :param y_test: Array containing the labels of the classification problem. - :return: Calculated score. + Args: + model: Model used for computing the score on the validation set. + x_test: Array containing the features of the classification problem. + y_test: Array containing the labels of the classification problem. + + Returns: + Calculated score. """ in_cls_score, out_of_cls_score = self.estimate_in_cls_and_out_of_cls_score( model, x_test, y_test @@ -413,11 +429,13 @@ def _permutation_montecarlo_classwise_shapley_for_label( truncation: TruncationPolicy, use_default_scorer_value: bool = True, min_elements_per_label: int = 1, + seed: Optional[Seed] = None, ) -> ValuationResult: """ Samples a random subset of the complement set and computes the truncated Monte Carlo estimator. + Args: :param u: Utility object containing model, data, and scoring function. The scoring function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. :param done: Function checking whether computation needs to stop. Otherwise, it will @@ -429,6 +447,8 @@ def _permutation_montecarlo_classwise_shapley_for_label( is not None. :param min_elements_per_label: The minimum number of elements for each opposite label. + :param seed: Either an instance of a numpy random number generator or a seed for it. + :return: ValuationResult object containing computed data values. """ @@ -439,6 +459,7 @@ def _permutation_montecarlo_classwise_shapley_for_label( data_names=u.data.data_names, ) + rng = np.random.default_rng(seed) _, y_train = u.data.get_training_data(u.data.indices) class_indices_set, class_complement_indices_set = split_indices_by_label( u.data.indices, @@ -446,13 +467,14 @@ def _permutation_montecarlo_classwise_shapley_for_label( label, ) _, complement_y_train = u.data.get_training_data(class_complement_indices_set) - indices_permutation = np.random.permutation(class_indices_set) + indices_permutation = rng.permutation(class_indices_set) for subset_idx, subset_complement in enumerate( random_powerset_group_conditional( class_complement_indices_set, complement_y_train, min_elements_per_group=min_elements_per_label, + seed=rng, ) ): result += _permutation_montecarlo_shapley_rollout( From faf16dfae712661bf4df4c82e6f9b745aa039e97 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Tue, 12 Sep 2023 02:14:53 +0200 Subject: [PATCH 20/43] Fix styling of comments, improve comments, rename methods and introduce reset method to StoppingCriterion. --- docs/value/shapley.md | 49 +- docs_includes/abbreviations.md | 1 + src/pydvl/utils/numeric.py | 51 +- src/pydvl/utils/score.py | 11 +- src/pydvl/value/result.py | 10 +- src/pydvl/value/shapley/classwise.py | 586 +++++++++--------- src/pydvl/value/shapley/truncated.py | 19 +- src/pydvl/value/stopping.py | 12 + tests/utils/test_numeric.py | 26 +- tests/value/shapley/test_classwise.py | 857 +++++--------------------- 10 files changed, 537 insertions(+), 1085 deletions(-) diff --git a/docs/value/shapley.md b/docs/value/shapley.md index f5765545f..954c7821d 100644 --- a/docs/value/shapley.md +++ b/docs/value/shapley.md @@ -78,47 +78,50 @@ stop condition. This is an instance of a [MaxTime][pydvl.value.stopping.MaxTime] and [AbsoluteStandardError][pydvl.value.stopping.AbsoluteStandardError]. -### Classwise Shapley +### Class-wise Shapley -**Classwise Shapley** [@schoch_csshapley_2022] is different Shapley schema applicable -for classification problems. The key insight is that samples can be beneficial -for overall performance, while being detrimental for their own class. CS-Shapley changes -the utility to account for this effect by decomposing it into a product of two -functions. It is the in-class accuracy multiplied by a discounter out-of-class accuracy. -The value is defined as: +Class-wise Shapley [@schoch_csshapley_2022] offers a distinct Shapley framework tailored +for classification problems. Let $D$ be the dataset, $D_{y_i}$ be the subset of $D$ with +labels $y_i$, and $D_{-y_i}$ be the complement of $D_{y_i}$ in $D$. The key idea is that +a sample $(x_i, y_i)$, might enhance the overall performance on $D$, while being +detrimental for the performance on $D_{y_i}$. To address this nuanced behavior, the +authors introduced the estimator $$ -v_u(x_i) \approx \frac{1}{K \cdot L} -\sum_{S^{(k)}_{-y_i} \subseteq T_{-y_i} \setminus \{i\}} -\sum_{\sigma^{(l)} \in \Pi(T_{y_i} \setminus \{i\})} -[u( \sigma_{\colon i} \cup \{i\} | S_{-y_i} ) -− u( \sigma_{\colon i} | S_{-y_i})] +v_u(i) = \frac{1}{2^{|D_{-y_i}|}} \sum_{S_{-y_i}} \frac{1}{|D_{y_i}|!} +\sum_{S_{y_i}} \binom{|D_{y_i}|-1}{|S_{y_i}|}^{-1} +[u( S_{y_i} \cup \{i\} | S_{-y_i} ) − u( S_{y_i} | S_{-y_i})], $$ -where $K$ is the number of subsets $S^{(k)}_{-y_i}$ sampled from the class complement -set $T_{-y_i}$ of class c and $L$ is the number of permutations sampled from the class -indices set $T_{y_i}$. The scoring function used has the form +where $S_{y_i} \subseteq D_{y_i} \setminus \{i\}$ and $S_{-y_i} \subseteq D_{-y_i}$. In +other words, the summations are over the powerset of $D_{y_i} \setminus \{i\}$ and +$D_{-y_i}$ respectively. The estimator employs a specialized utility function -$$u(S_{y_i}|S_{-y_i}) = a_S(D_{y_i}))) \exp\{a_S(D_{-y_i}))\}.$$ +$$ +u(S_{y_i}|S_{-y_i}) = a_S(D_{y_i}) \exp(a_S(D_{-y_i})), +$$ -This can be further customised, but that form is shown by the authors to have certain -desirable properties. +where $S=S_{y_i} \cup S_{-y_i}$ and $a_S(D)$ is the accuracy of the model trained on $S$ +and evaluated on $D$.In practical applications, the evaluation of this estimator +leverages both Monte Carlo sampling and permutation Monte Carlo sampling +[@castro_polynomial_2009]. ```python from pydvl.utils import Dataset, Utility -from pydvl.value import compute_classwise_shapley_values, ClasswiseScorer, HistoryDeviation -from pydvl.value.shapley.truncated import RelativeTruncation +from pydvl.value import HistoryDeviation, MaxChecks, RelativeTruncation +from pydvl.value.shapley.classwise import compute_classwise_shapley_values, \ + ClasswiseScorer model = ... -scoring = ClasswiseScorer("accuracy") data = Dataset(...) +scoring = ClasswiseScorer("accuracy") utility = Utility(model, data, scoring) values = compute_classwise_shapley_values( utility, - done=HistoryDeviation(n_steps=500, rtol=1e-3), + done=HistoryDeviation(n_steps=500, rtol=5e-2), truncation=RelativeTruncation(utility, rtol=0.01), - n_resample_complement_sets=10, + done_sample_complements=MaxChecks(1), normalize_values=True ) ``` diff --git a/docs_includes/abbreviations.md b/docs_includes/abbreviations.md index e0fa67a4c..6cc7cb258 100644 --- a/docs_includes/abbreviations.md +++ b/docs_includes/abbreviations.md @@ -1,4 +1,5 @@ *[CSP]: Constraint Satisfaction Problem +*[CS]: Class-wise Shapley *[GT]: Group Testing *[LC]: Least Core *[LOO]: Leave-One-Out diff --git a/src/pydvl/utils/numeric.py b/src/pydvl/utils/numeric.py index 3ec8677e8..6399d0d98 100644 --- a/src/pydvl/utils/numeric.py +++ b/src/pydvl/utils/numeric.py @@ -34,7 +34,7 @@ "random_matrix_with_condition_number", "random_subset", "random_powerset", - "random_powerset_group_conditional", + "random_powerset_label_min", "random_subset_of_size", "top_k_value_accuracy", ] @@ -156,56 +156,49 @@ def random_powerset( total += 1 -def random_powerset_group_conditional( +def random_powerset_label_min( s: NDArray[T], - groups: NDArray[np.int_], - min_elements_per_group: int = 1, + labels: NDArray[np.int_], + min_elements_per_label: int = 1, seed: Optional[Seed] = None, ) -> Generator[NDArray[T], None, None]: - """Draws random group-conditional subsets from `s`. - - It is ensured that in each sampled set, each unique group is represented at - least `min_elements` times. The groups are specified as integers for all - elements of the set separately. + """Draws random subsets from `s`, while ensuring that at least + `min_elements_per_label` elements per label are included in the draw. It can be used + for classification problems to ensure that a set contains information for all labels + (or not if `min_elements_per_label=0`). Args: - s: Vector of size N representing the set to sample elements from. - groups: Vector of size N containing the group as an integer for each - element. - min_elements_per_group: The minimum number of elements for each group. + s: Set to sample from + labels: Labels for the samples + min_elements_per_label: Minimum number of elements for each label. seed: Either an instance of a numpy random number generator or a seed for it. Returns: - Generated draw from the powerset of s with `min_elements` of each group. + Generated draw from the powerset of s with `min_elements_per_label` for each + label. Raises: - ValueError: If `s` and `groups` are of different length or - `min_elements` is smaller than 0. + ValueError: If `s` and `labels` are of different length or + `min_elements_per_label` is smaller than 0. """ - if len(groups) != len(s): + if len(labels) != len(s): raise ValueError("Set and labels have to be of same size.") - if min_elements_per_group < 0: + if min_elements_per_label < 0: raise ValueError( - f"Parameter min_elements={min_elements_per_group} needs to be bigger or equal to 0." - ) - - if min_elements_per_group == 0: - logger.warning( - "It is recommended to ensure at least one element of each group is" - " contained in the sampled and yielded set." + f"Parameter min_elements={min_elements_per_label} needs to be bigger or equal to 0." ) rng = np.random.default_rng(seed) - unique_labels = np.unique(groups) + unique_labels = np.unique(labels) while True: subsets: List[NDArray[T]] = [] for label in unique_labels: - label_indices = np.asarray(np.where(groups == label)[0]) + label_indices = np.asarray(np.where(labels == label)[0]) subset_length = int( rng.integers( - min(min_elements_per_group, len(label_indices)), + min(min_elements_per_label, len(label_indices)), len(label_indices) + 1, ) ) @@ -219,7 +212,7 @@ def random_powerset_group_conditional( rng.shuffle(subset) yield subset else: - yield np.array([], dtype=int) + yield np.array([], dtype=s.dtype) def random_subset_of_size( diff --git a/src/pydvl/utils/score.py b/src/pydvl/utils/score.py index fcbe23862..578ed76ae 100644 --- a/src/pydvl/utils/score.py +++ b/src/pydvl/utils/score.py @@ -2,7 +2,7 @@ This module provides a [Scorer][pydvl.utils.score.Scorer] class that wraps scoring functions with additional information. -Scorers can be constructed in the same way as in scikit-learn: either from +Scorers can be constructed in the same way as in scikit-learn: either from known strings or from a callable. Greater values must be better. If they are not, a negated version can be used, see scikit-learn's [make_scorer()](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html). @@ -22,12 +22,7 @@ from pydvl.utils.types import SupervisedModel -__all__ = [ - "Scorer", - "compose_score", - "squashed_r2", - "squashed_variance", -] +__all__ = ["Scorer", "compose_score", "squashed_r2", "squashed_variance"] class ScorerCallable(Protocol): @@ -64,7 +59,7 @@ class Scorer: def __init__( self, scoring: Union[str, ScorerCallable], - default: float = 0.0, + default: float = np.nan, range: Tuple = (-np.inf, np.inf), name: Optional[str] = None, ): diff --git a/src/pydvl/value/result.py b/src/pydvl/value/result.py index b501a061b..86085d77f 100644 --- a/src/pydvl/value/result.py +++ b/src/pydvl/value/result.py @@ -542,7 +542,7 @@ def __add__(self, other: "ValuationResult") -> "ValuationResult": xm[other_pos] = other._values vm[other_pos] = other._variances - # np.maximum(1, n + m) covers case n = m = 0 with + # np.maximum(1, n + m) covers case n = m = 0. n_m_sum = np.maximum(1, n + m) # Sample mean of n+m samples from two means of n and m samples @@ -635,16 +635,16 @@ def update(self, idx: int, new_value: float) -> "ValuationResult": ) return self - def scale(self, coefficient: float, indices: Optional[NDArray[IndexT]] = None): + def scale(self, factor: float, indices: Optional[NDArray[IndexT]] = None): """ Scales the values and variances of the result by a coefficient. Args: - coefficient: Coefficient to scale by. + factor: Factor to scale by. indices: Indices to scale. If None, all values are scaled. """ - self._values[self._sort_positions[indices]] *= coefficient - self._variances[self._sort_positions[indices]] *= coefficient**2 + self._values[self._sort_positions[indices]] *= factor + self._variances[self._sort_positions[indices]] *= factor**2 def get(self, idx: Integral) -> ValueItem: """Retrieves a ValueItem by data index, as opposed to sort index, like diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index be28047b5..0e0647b97 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -1,11 +1,61 @@ -""" -Implementation of Class-wise Shapley, introduced in (Schoch, Haifeng and Ji, -2022)[^1]. +r""" +Implementation of Class-wise Shapley, introduced in (Schoch et al., 2022)[^1]. + +Class-wise Shapley (Schoch et al., 2022)[^1] offers a distinct Shapley framework tailored +for classification problems. Let $D$ be the dataset, $D_{y_i}$ be the subset of $D$ with +labels $y_i$, and $D_{-y_i}$ be the complement of $D_{y_i}$ in $D$. The key idea is that +a sample $(x_i, y_i)$, might enhance the overall performance on $D$, while being +detrimental for the performance on $D_{y_i}$. To address this nuanced behavior, the +authors introduced the estimator + +$$ +v_u(i) = \frac{1}{2^{|D_{-y_i}|}} \sum_{S_{-y_i}} \frac{1}{|D_{y_i}|!} +\sum_{S_{y_i}} \binom{|D_{y_i}|-1}{|S_{y_i}|}^{-1} +[u( S_{y_i} \cup \{i\} | S_{-y_i} ) − u( S_{y_i} | S_{-y_i})], +$$ + +where $S_{y_i} \subseteq D_{y_i} \setminus \{i\}$ and $S_{-y_i} \subseteq D_{-y_i}$. In +other words, the summations are over the powerset of $D_{y_i} \setminus \{i\}$ and +$D_{-y_i}$ respectively. The estimator employs a specialized utility function + +$$ +u(S_{y_i}|S_{-y_i}) = a_S(D_{y_i}) \exp(a_S(D_{-y_i})), +$$ + +where $S=S_{y_i} \cup S_{-y_i}$ and $a_S(D)$ is the accuracy of the model trained on $S$ +and evaluated on $D$. + +## Monte Carlo sampling + Permutation Monte Carlo sampling + +In practical applications, the evaluation of this estimator leverages both Monte Carlo +sampling and permutation Monte Carlo sampling. This results in the estimator + +$$ +v_u(i) = \frac{1}{K} \sum_k \frac{1}{L} \sum_l +[u(\pi^{(l)}_{:i} \cup \{i\} | S^{(k)} ) − u( \pi^{(l)}_{:i} | S^{(k)})], +$$ + +with $S^{(1)}, \dots, S^{(K)} \subseteq T_{-y_i}$ and + $\pi^{(1)}, \dots, \pi^{(L)} \in \Pi(T_{y_i}\setminus\{i\})$. + +## Derivation of test case + +Let $D=\{(1,0),(2,0),(3,0),(4,1)\}$ be the test set and $T=\{(1,0),(2,0),(3,1),(4,1)\}$ +the train set. This specific dataset is chosen as it allows to solve the model + +$$y = \max(0, \min(1, \text{round}(\beta^T x)))$$ + +in closed form $\beta = \frac{\text{dot}(x, y)}{\text{dot}(x, x)}$. By using the tables +that represent in-class accuracy $a_S(D_{y_i})$ and out-of-class accuracy +$a_S(D_{-y_i})$ the Monte Carlo estimator with $\{S^{(1)}, \dots, S^{(K)}\} += 2^{T_{-y_i}}$ and $\{\pi^{(1)}, \dots, pi^{(L)}\} = \Pi(T_{y_i}\setminus\{i\})$ can be +evaluated. Note that $2^M$ is the powerset of $M$. The details are left out to the +curious reader. # References [^1]: Schoch, Stephanie, Haifeng Xu, and - Yangfeng Ji. [CS-Shapley: Class-Wise Shapley Values for Data Valuation in + Yangfeng Ji. [CS-Shapley: Class-wise Shapley Values for Data Valuation in Classification](https://openreview.net/forum?id=KTOcrOR5mQ9). In Proc. of the Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). New Orleans, Louisiana, USA, 2022. @@ -14,10 +64,11 @@ import logging import numbers from concurrent.futures import FIRST_COMPLETED, Future, wait -from copy import copy +from copy import copy, deepcopy from typing import Callable, Optional, Set, Tuple, cast import numpy as np +from numpy.random import SeedSequence from numpy.typing import NDArray from tqdm import tqdm @@ -32,7 +83,7 @@ ensure_seed_sequence, init_executor, init_parallel_backend, - random_powerset_group_conditional, + random_powerset_label_min, ) from pydvl.value.result import ValuationResult from pydvl.value.shapley.truncated import TruncationPolicy @@ -40,7 +91,145 @@ logger = logging.getLogger(__name__) -__all__ = ["compute_classwise_shapley_values", "ClasswiseScorer"] +__all__ = ["ClasswiseScorer", "compute_classwise_shapley_values"] + + +class ClasswiseScorer(Scorer): + r"""A Scorer designed for evaluation in classification problems. Its value is + derived from both in-class and out-of-class scores (Schoch et al., 2022) + 1. Let $S$ represent the training + set and $D$ be the test set. For each label $c$, the test set $D$ is factorized into + two disjoint sets: $D_c$ for in-class instances and $D_{-c}$ for out-of-class + instances. The score function itself than estimates the in-class metric, adjusted by + the discounted out-of-class metric. In essence, the score function for each element + of label $c$ is conditioned on the out-of-class instances (or a subset thereof). + Both the in-class and out-of-class metrics are determined by an inner score function + $a_S$ trained on the train set $S$. The expression for the outer score function is + + $${ + u(S_{y_i}) = f(a_S(D_{y_i}))) g(a_S(D_{-y_i}))), + }$$ + + where $f$ and $g$ are continuous, monotonic functions. For a detailed explanation, + refer to section four of (Schoch et al., 2022) + 1. + + Args: + default: Score used when a model cannot be fit, e.g. when too little data is + passed, or errors arise. + range: Numerical range of the score function. Some Monte Carlo methods can + use this to estimate the number of samples required for a certain quality of + approximation. If not provided, it can be read from the `scoring` object + if it provides it, for instance if it was constructed with + [compose_score][pydvl.utils.score.compose_score]. + in_class_discount_fn: Continuous, monotonic increasing function used to + discount the in-class score. + out_of_class_discount_fn: Continuous, monotonic increasing function used to + discount the out-of-class score. + initial_label: Set initial label (Doesn't require to set parameter `label` + on [ClasswiseScorer][pydvl.value.shapley.classwise.ClasswiseScorer] in first + iteration) + name: Name of the scorer. If not provided, the name of the passed + function will be prefixed by 'classwise '. + + !!! tip "New in version 0.7.1" + """ + + def __init__( + self, + scoring: str = "accuracy", + default: float = 0.0, + range: Tuple[float, float] = (0, np.inf), + in_class_discount_fn: Callable[[float], float] = lambda x: x, + out_of_class_discount_fn: Callable[[float], float] = np.exp, + initial_label: Optional[int] = None, + name: Optional[str] = None, + ): + disc_score_in_class = in_class_discount_fn(range[1]) + disc_score_out_of_class = out_of_class_discount_fn(range[1]) + transformed_range = (0, disc_score_in_class * disc_score_out_of_class) + super().__init__( + "accuracy", + range=transformed_range, + default=default, + name=name or f"classwise {scoring}", + ) + self._in_class_discount_fn = in_class_discount_fn + self._out_of_class_discount_fn = out_of_class_discount_fn + self.label = initial_label + + def __str__(self): + return self._name + + def __call__( + self: "ClasswiseScorer", + model: SupervisedModel, + x_test: NDArray[np.float_], + y_test: NDArray[np.int_], + ) -> float: + ( + in_class_score, + out_of_class_score, + ) = self.estimate_in_class_and_out_of_class_score(model, x_test, y_test) + disc_score_in_class = self._in_class_discount_fn(in_class_score) + disc_score_out_of_class = self._out_of_class_discount_fn(out_of_class_score) + return disc_score_in_class * disc_score_out_of_class + + def estimate_in_class_and_out_of_class_score( + self, + model: SupervisedModel, + x_test: NDArray[np.float_], + y_test: NDArray[np.int_], + rescale_scores: bool = True, + ) -> Tuple[float, float]: + r""" + Computes in-class and out-of-class scores using the provided scoring function + $s$. The result can be expressed as + + $${ + a_S(D=\{(x_1, y_1), \dots, (x_K, y_K)\}) = \frac{1}{N} \sum_k s(y(x_k), y_k). + }$$ + + In this context, for label $c$ calculations are executed twice: once for $D_c$ + and once for $D_{-c}$ to determine the in-class and out-of-class scores, + respectively. By default, the raw scores are multiplied by $\frac{|D_c|}{|D|}$ + and $\frac{|D_{-c}|}{|D|}$, respectively. This is done to ensure that both + scores are of the same order of magnitude. This normalization is particularly + useful when the inner score function $a_S$ is calculated by an estimator of the + form $\frac{1}{N} \sum_i x_i$, e.g. the accuracy. + + Args: + model: Model used for computing the score on the validation set. + x_test: Array containing the features of the classification problem. + y_test: Array containing the labels of the classification problem. + rescale_scores: If set to True, the scores will be denormalized. This is + particularly useful when the inner score function $a_S$ is calculated by + an estimator of the form $\frac{1}{N} \sum_i x_i$. + + Returns: + Tuple containing the in-class and out-of-class scores. + """ + scorer = self._scorer + label_set_match = y_test == self.label + label_set = np.where(label_set_match)[0] + num_classes = len(np.unique(y_test)) + + if len(label_set) == 0: + return 0, 1 / (num_classes - 1) + + complement_label_set = np.where(~label_set_match)[0] + in_class_score = scorer(model, x_test[label_set], y_test[label_set]) + out_of_class_score = scorer( + model, x_test[complement_label_set], y_test[complement_label_set] + ) + + if rescale_scores: + n_in_class = np.count_nonzero(y_test == self.label) + n_out_of_class = len(y_test) - n_in_class + in_class_score *= n_in_class / (n_in_class + n_out_of_class) + out_of_class_score *= n_out_of_class / (n_in_class + n_out_of_class) + + return in_class_score, out_of_class_score def compute_classwise_shapley_values( @@ -57,15 +246,25 @@ def compute_classwise_shapley_values( progress: bool = False, seed: Optional[Seed] = None, ) -> ValuationResult: - """ - Computes the class-wise Shapley values as described in (Schoch, Haifeng and - Ji, 2022)1. - The values can be optionally normalized, depending on `normalize_values`. + r""" + Computes an approximate Class-wise Shapley value by sampling independent + permutations of the index set for each label and unifying it with index sets sampled + from the complement (with respect to the currently evaluated label), approximating + the sum: + + $$ + v_u(i) = \frac{1}{K} \sum_k \frac{1}{L} \sum_l + [u(\sigma^{(l)}_{:i} \cup \{i\} | S^{(k)} ) − u( \sigma^{(l)}_{:i} | S^{(k)})], + $$ + + where $\sigma_{:i}$ denotes the set of indices in permutation sigma before + the position where $i$ appears and $S$ is a subset of the index set of all other + labels(see [[data-valuation]] for details). Args: u: Utility object containing model, data, and scoring function. The scorer must be of type - [ClassWiseScorer][pydvl.value.shapley.classwise.ClasswiseScorer]. + [ClasswiseScorer][pydvl.value.shapley.classwise.ClasswiseScorer]. done: Function that checks whether the computation needs to stop. truncation: Callable function that decides whether to interrupt processing a permutation and set subsequent marginals to zero. @@ -89,10 +288,24 @@ def compute_classwise_shapley_values( Returns: ValuationResult object containing computed data values. - !!! tip "New in version 0.7.0" + !!! tip "New in version 0.7.1" """ + dim_correct = u.data.y_train.ndim == 1 and u.data.y_test.ndim == 1 + is_integral = all( + map( + lambda v: isinstance(v, numbers.Integral), (*u.data.y_train, *u.data.y_test) + ) + ) + if not dim_correct or not is_integral: + raise ValueError( + "The supplied dataset has to be a 1-dimensional classification dataset." + ) - _check_classwise_shapley_utility(u) + if not isinstance(u.scorer, ClasswiseScorer): + raise ValueError( + "Please set a subclass of ClasswiseScorer object as scorer object of the" + " utility. See scoring argument of Utility." + ) parallel_backend = init_parallel_backend(config) u_ref = parallel_backend.put(u) @@ -100,8 +313,9 @@ def compute_classwise_shapley_values( n_submitted_jobs = 2 * n_jobs pbar = tqdm(disable=not progress, position=0, total=100, unit="%") + algorithm = "classwise_shapley" accumulated_result = ValuationResult.zeros( - algorithm="classwise_shapley", + algorithm=algorithm, indices=u.data.indices, data_names=u.data.data_names, ) @@ -129,12 +343,13 @@ def compute_classwise_shapley_values( seeds = seed_sequence.spawn(n_remaining_slots) for i in range(n_remaining_slots): future = executor.submit( - _permutation_montecarlo_classwise_shapley, + _permutation_montecarlo_classwise_shapley_one_step, u_ref, truncation=truncation, done_sample_complements=done_sample_complements, use_default_scorer_value=use_default_scorer_value, min_elements_per_label=min_elements_per_label, + algorithm_name=algorithm, seed=seeds[i], ) pending.add(future) @@ -146,21 +361,24 @@ def compute_classwise_shapley_values( return result -def _permutation_montecarlo_classwise_shapley( +def _permutation_montecarlo_classwise_shapley_one_step( u: Utility, *, done_sample_complements: StoppingCriterion = None, truncation: TruncationPolicy, use_default_scorer_value: bool = True, min_elements_per_label: int = 1, - seed: Optional[Seed] = None, + algorithm_name: str = "classwise_shapley", + seed: Optional[SeedSequence] = None, ) -> ValuationResult: - """Computes classwise Shapley value using truncated Monte Carlo permutation - sampling for the subsets. + """Helper function for [compute_classwise_shapley_values()] + [pydvl.value.shapley.classwise.compute_classwise_shapley_values]. + Args: - u: Utility object containing model, data, and scoring function. The scoring - function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + u: Utility object containing model, data, and scoring function. The + scorer must be of type [ClasswiseScorer] + [pydvl.value.shapley.classwise.ClasswiseScorer]. done_sample_complements: Function checking whether computation needs to stop. Otherwise, it will resample conditional sets until the stopping criterion is met. @@ -171,6 +389,7 @@ def _permutation_montecarlo_classwise_shapley( this. If it is set to false, the base score is calculated from the utility. min_elements_per_label: The minimum number of elements for each opposite label. + algorithm_name: For the results object. seed: Either an instance of a numpy random number generator or a seed for it. Returns: @@ -180,7 +399,7 @@ def _permutation_montecarlo_classwise_shapley( done_sample_complements = MaxChecks(1) result = ValuationResult.zeros( - algorithm="classwise_shapley", + algorithm=algorithm_name, indices=u.data.indices, data_names=u.data.data_names, ) @@ -192,67 +411,54 @@ def _permutation_montecarlo_classwise_shapley( for label in unique_labels: u.scorer.label = label - result += _permutation_montecarlo_classwise_shapley_for_label( - u, + class_indices_set, class_complement_indices_set = _split_indices_by_label( + u.data.indices, + y_train, label, - done=done_sample_complements, - truncation=truncation, - use_default_scorer_value=use_default_scorer_value, - min_elements_per_label=min_elements_per_label, - seed=rng, ) + _, complement_y_train = u.data.get_training_data(class_complement_indices_set) + indices_permutation = rng.permutation(class_indices_set) + done_sample_complements.reset() + + for subset_idx, subset_complement in enumerate( + random_powerset_label_min( + class_complement_indices_set, + complement_y_train, + min_elements_per_label=min_elements_per_label, + seed=rng, + ) + ): + result += _permutation_montecarlo_shapley_rollout( + u, + indices_permutation, + additional_indices=subset_complement, + truncation=truncation, + algorithm_name=algorithm_name, + use_default_scorer_value=use_default_scorer_value, + ) + if done_sample_complements(result): + break return result -def _check_classwise_shapley_utility(u: Utility): - """ - Verifies if the provided utility object supports classwise Shapley values. - - Args: - u: Utility object containing model, data, and scoring function. The scoring - function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. - - Raises: - ValueError: If ``u.data`` is not a classification problem. - ValueError: If ``u.scorer`` is not an instance of - :class:`~pydvl.utils.score.ClassWiseScorer` - """ - - dim_correct = u.data.y_train.ndim == 1 and u.data.y_test.ndim == 1 - is_integral = all( - map( - lambda v: isinstance(v, numbers.Integral), (*u.data.y_train, *u.data.y_test) - ) - ) - if not dim_correct or not is_integral: - raise ValueError( - "The supplied dataset has to be a 1-dimensional classification dataset." - ) - - if not isinstance(u.scorer, ClasswiseScorer): - raise ValueError( - "Please set a subclass of ClassWiseScorer object as scorer object of the" - " utility. See scoring argument of Utility." - ) - - def _normalize_classwise_shapley_values( result: ValuationResult, u: Utility, ) -> ValuationResult: - """ + r""" Normalize a valuation result specific to classwise Shapley. - Each value corresponds to a class c and gets normalized by multiplying - `in-class-score / sigma`. In this context `sigma` is the magnitude of all values - belonging to the currently viewed class. See footcite:t:`schoch_csshapley_2022` for - more details. + Each value $v_i$ associated with the sample $(x_i, y_i)$ is normalized by + multiplying it with $a_S(D_{y_i})$ and dividing by $\sum_{j \in D_{y_i}} v_j$. For + more details, see (Schoch et al., 2022) 1 + . Args: result: ValuationResult object to be normalized. - u: Utility object containing model, data, and scoring function. The scoring - function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. + u: Utility object containing model, data, and scoring function. The + scorer must be of type [ClasswiseScorer] + [pydvl.value.shapley.classwise.ClasswiseScorer]. Returns: Normalized ValuationResult object. @@ -269,224 +475,13 @@ def _normalize_classwise_shapley_values( u.model.fit(u.data.x_train, u.data.y_train) scorer.label = label - in_cls_acc, _ = scorer.estimate_in_cls_and_out_of_cls_score( + in_class_acc, _ = scorer.estimate_in_class_and_out_of_class_score( u.model, u.data.x_test, u.data.y_test ) sigma = np.sum(result.values[indices_label_set]) if sigma != 0: - result.scale(in_cls_acc / sigma, indices=indices_label_set) - - return result - - -class ClasswiseScorer(Scorer): - """A Scorer which is applicable for valuation in classification problems. Its value - is based on in-cls and out-of-cls score :footcite:t:`schoch_csshapley_2022`. For - each class ``label`` it separates the elements into two groups, namely in-cls - instances and out-of-cls instances. The value function itself than estimates the - in-cls metric discounted by the out-of-cls metric. In other words the value function - for each element of one class is conditioned on the out-of-cls instances (or a - subset of it). The form of the value function can be written as - - .. math:: - v_{y_i}(D) = f(a_S(D_{y_i}))) * g(a_S(D_{-y_i}))) - - where f and g are continuous, monotonic functions and D is the test set. - - in order to produce meaningful results. For further reference see also section four - of :footcite:t:`schoch_csshapley_2022`. - - Args: - default: Score used when a model cannot be fit, e.g. when too little data is - passed, or errors arise. - range: Numerical range of the score function. Some Monte Carlo methods can - use this to estimate the number of samples required for a certain quality of - approximation. If not provided, it can be read from the ``scoring`` object - if it provides it, for instance if it was constructed with - :func:`~pydvl.utils.types.compose_score`. - in_class_discount_fn: Continuous, monotonic increasing function used to - discount the in-class score. - out_of_class_discount_fn: Continuous, monotonic increasing function used to - discount the out-of-class score. - initial_label: Set initial label (Doesn't require to set parameter ``label`` - on ``ClassWiseDiscountedScorer`` in first iteration) - name: Name of the scorer. If not provided, the name of the passed - function will be prefixed by 'classwise '. - - !!! tip "New in version 0.7.0" - """ - - def __init__( - self, - scoring: str = "accuracy", - default: float = 0.0, - range: Tuple[float, float] = (-np.inf, np.inf), - in_class_discount_fn: Callable[[float], float] = lambda x: x, - out_of_class_discount_fn: Callable[[float], float] = np.exp, - initial_label: Optional[int] = None, - name: Optional[str] = None, - ): - disc_score_in_cls = in_class_discount_fn(range[1]) - disc_score_out_of_cls = out_of_class_discount_fn(range[1]) - transformed_range = (0, disc_score_in_cls * disc_score_out_of_cls) - super().__init__( - "accuracy", - range=transformed_range, - default=default, - name=name or f"classwise {scoring}", - ) - self._in_cls_discount_fn = in_class_discount_fn - self._out_of_cls_discount_fn = out_of_class_discount_fn - self.label = initial_label - - def __str__(self): - return self._name - - def __call__( - self: "ClasswiseScorer", - model: SupervisedModel, - x_test: NDArray[np.float_], - y_test: NDArray[np.int_], - ) -> float: - """ - Args: - model: Model used for computing the score on the validation set. - x_test: Array containing the features of the classification problem. - y_test: Array containing the labels of the classification problem. - - Returns: - Calculated score. - """ - in_cls_score, out_of_cls_score = self.estimate_in_cls_and_out_of_cls_score( - model, x_test, y_test - ) - disc_score_in_cls = self._in_cls_discount_fn(in_cls_score) - disc_score_out_of_cls = self._out_of_cls_discount_fn(out_of_cls_score) - return disc_score_in_cls * disc_score_out_of_cls - - def estimate_in_cls_and_out_of_cls_score( - self, - model: SupervisedModel, - x_test: NDArray[np.float_], - y_test: NDArray[np.int_], - rescale_scores: bool = True, - ) -> Tuple[float, float]: - r""" - Computes in-class and out-of-class scores using the provided scoring function, - which can be expressed as: - - .. math:: - a_S(D=\{(\hat{x}_1, \hat{y}_1), \dots, (\hat{x}_K, \hat{y}_K)\}) &= - \frac{1}{N} \sum_k s(y(\hat{x}_k), \hat{y}_k) - - In this context, the computation is performed twice: once on D_i and once on D_o - to calculate the in-class and out-of-class scores. Here, D_i contains only - samples with the specified 'label' from the validation set, while D_o contains - all other samples. By default, the scores are scaled to have the same order of - magnitude. In such cases, the raw scores are multiplied by: - - .. math:: - N_{y_i} = \frac{a_S(D_{y_i})}{a_S(D_{y_i})+a_S(D_{-y_i})} \quad \text{and} - \quad N_{-y_i} = \frac{a_S(D_{-y_i})}{a_S(D_{y_i})+a_S(D_{-y_i})} - - :param model: Model used for computing the score on the validation set. - :param x_test: Array containing the features of the classification problem. - :param y_test: Array containing the labels of the classification problem. - :param rescale_scores: If set to True, the scores will be denormalized. This is - particularly useful when the inner score is calculated by an estimator of - the form 1/N sum_i x_i. - :return: Tuple containing the in-class and out-of-class scores. - """ - scorer = self._scorer - label_set_match = y_test == self.label - label_set = np.where(label_set_match)[0] - num_classes = len(np.unique(y_test)) - - if len(label_set) == 0: - return 0, 1 / (num_classes - 1) - - complement_label_set = np.where(~label_set_match)[0] - in_cls_score = scorer(model, x_test[label_set], y_test[label_set]) - out_of_cls_score = scorer( - model, x_test[complement_label_set], y_test[complement_label_set] - ) - - if rescale_scores: - n_in_cls = np.count_nonzero(y_test == self.label) - n_out_of_cls = len(y_test) - n_in_cls - in_cls_score *= n_in_cls / (n_in_cls + n_out_of_cls) - out_of_cls_score *= n_out_of_cls / (n_in_cls + n_out_of_cls) - - return in_cls_score, out_of_cls_score - - -def _permutation_montecarlo_classwise_shapley_for_label( - u: Utility, - label: int, - *, - done: StoppingCriterion, - truncation: TruncationPolicy, - use_default_scorer_value: bool = True, - min_elements_per_label: int = 1, - seed: Optional[Seed] = None, -) -> ValuationResult: - """ - Samples a random subset of the complement set and computes the truncated Monte Carlo - estimator. - - Args: - :param u: Utility object containing model, data, and scoring function. The scoring - function should be of type :class:`~pydvl.utils.score.ClassWiseScorer`. - :param done: Function checking whether computation needs to stop. Otherwise, it will - resample conditional sets until the stopping criterion is met. - :param label: The label for which to sample the complement (e.g. all other labels) - :param truncation: Callable which decides whether to interrupt processing a - permutation and set all subsequent marginals to zero. - :param use_default_scorer_value: Use default scorer value even if additional_indices - is not None. - :param min_elements_per_label: The minimum number of elements for each opposite - label. - :param seed: Either an instance of a numpy random number generator or a seed for it. - - :return: ValuationResult object containing computed data values. - """ - - algorithm_name = "classwise_shapley" - result = ValuationResult.zeros( - algorithm="classwise_shapley", - indices=u.data.indices, - data_names=u.data.data_names, - ) - - rng = np.random.default_rng(seed) - _, y_train = u.data.get_training_data(u.data.indices) - class_indices_set, class_complement_indices_set = split_indices_by_label( - u.data.indices, - y_train, - label, - ) - _, complement_y_train = u.data.get_training_data(class_complement_indices_set) - indices_permutation = rng.permutation(class_indices_set) - - for subset_idx, subset_complement in enumerate( - random_powerset_group_conditional( - class_complement_indices_set, - complement_y_train, - min_elements_per_group=min_elements_per_label, - seed=rng, - ) - ): - result += _permutation_montecarlo_shapley_rollout( - u, - indices_permutation, - additional_indices=subset_complement, - truncation=truncation, - algorithm_name=algorithm_name, - use_default_scorer_value=use_default_scorer_value, - ) - if done(result): - break + result.scale(in_class_acc / sigma, indices=indices_label_set) return result @@ -500,9 +495,14 @@ def _permutation_montecarlo_shapley_rollout( use_default_scorer_value: bool = True, ) -> ValuationResult: """ - A truncated version of a permutation-based MC estimator. - values. It generates a permutation p[i] of the class label indices and iterates over - all subsets starting from the empty set to the full set of indices. + Represents a truncated version of a permutation-based MC estimator. It iterates over + all subsets starting from the empty set to the full set of indices as specified by + `permutation`. For each subset, the marginal contribution is computed and added to + the result. The computation is interrupted if the truncation policy returns `True`. + + !!! Todo + Reuse in [permutation_montecarlo_shapley()] + [pydvl.value.shapley.montecarlo.permutation_montecarlo_shapley] Args: u: Utility object containing model, data, and scoring function. @@ -515,6 +515,7 @@ def _permutation_montecarlo_shapley_rollout( always considered. use_default_scorer_value: Use default scorer value even if additional_indices is not None. + Returns: ValuationResult object containing computed data values. """ @@ -577,17 +578,20 @@ def _permutation_montecarlo_shapley_rollout( return result -def split_indices_by_label( +def _split_indices_by_label( indices: NDArray[np.int_], labels: NDArray[np.int_], label: int ) -> Tuple[NDArray[np.int_], NDArray[np.int_]]: """ - Splits the indices into two sets based on the value of ``label``: those samples + Splits the indices into two sets based on the value of `label`, e.g. those samples with and without that label. - :param indices: The indices to be used for referring to the data. - :param labels: Corresponding labels for the indices. - :param label: Label to be used for splitting. - :return: Tuple with two sets of indices. + Args: + indices: The indices to be used for referring to the data. + labels: Corresponding labels for the indices. + label: Label to be used for splitting. + + Returns: + Tuple with two sets of indices. """ active_elements = labels == label class_indices_set = np.where(active_elements)[0] diff --git a/src/pydvl/value/shapley/truncated.py b/src/pydvl/value/shapley/truncated.py index 9e1bbd812..4bd094bb6 100644 --- a/src/pydvl/value/shapley/truncated.py +++ b/src/pydvl/value/shapley/truncated.py @@ -91,14 +91,12 @@ def reset(self, u: Optional[Utility] = None): class FixedTruncation(TruncationPolicy): """Break a permutation after computing a fixed number of marginals. - The experiments in Appendix B of - (Ghorbani and Zou, 2019)1 - show that when the training set size is large enough, one can simply - truncate the iteration over permutations after a fixed number of steps. This - happens because beyond a certain number of samples in a training set, the - model becomes insensitive to new ones. Alas, this strongly depends on the - data distribution and the model and there is no automatic way of estimating - this number. + The experiments in Appendix B of (Ghorbani and Zou, 2019)1 + show that when the training set size is large enough, one can simply truncate the iteration + over permutations after a fixed number of steps. This happens because beyond + a certain number of samples in a training set, the model becomes insensitive + to new ones. Alas, this strongly depends on the data distribution and the + model and there is no automatic way of estimating this number. Args: u: Utility object with model, data, and scoring function @@ -135,6 +133,7 @@ class RelativeTruncation(TruncationPolicy): def __init__(self, u: Utility, rtol: float): super().__init__() self.rtol = rtol + logger.info("Computing total utility for permutation truncation.") self.total_utility = self.reset(u) self._u = u @@ -191,8 +190,8 @@ def reset(self, u: Optional[Utility] = None): @deprecated( target=True, - deprecated_in="0.6.1", - remove_in="0.7.0", + deprecated_in="0.7.0", + remove_in="0.8.0", args_mapping=dict(coordinator_update_period=None, worker_update_period=None), ) def truncated_montecarlo_shapley( diff --git a/src/pydvl/value/stopping.py b/src/pydvl/value/stopping.py index 257a6b70f..2c7327eac 100644 --- a/src/pydvl/value/stopping.py +++ b/src/pydvl/value/stopping.py @@ -147,6 +147,9 @@ def completion(self) -> float: return 0.0 return float(np.mean(self.converged).item()) + def reset(self): + pass + @property def converged(self) -> NDArray[np.bool_]: """Returns a boolean array indicating whether the values have converged @@ -333,6 +336,9 @@ def completion(self) -> float: return min(1.0, self._count / self.n_checks) return 0.0 + def reset(self): + self._count = 0 + class MaxUpdates(StoppingCriterion): """Terminate if any number of value updates exceeds or equals the given @@ -447,6 +453,9 @@ def completion(self) -> float: return 0.0 return (time() - self.start) / self.max_seconds + def reset(self): + self.start = time() + class HistoryDeviation(StoppingCriterion): r"""A simple check for relative distance to a previous step in the @@ -527,3 +536,6 @@ def _check(self, r: ValuationResult) -> Status: if np.all(self._converged): return Status.Converged return Status.Pending + + def reset(self): + self._memory = None # type: ignore diff --git a/tests/utils/test_numeric.py b/tests/utils/test_numeric.py index 71eb8826e..a899f4dd7 100644 --- a/tests/utils/test_numeric.py +++ b/tests/utils/test_numeric.py @@ -6,7 +6,7 @@ powerset, random_matrix_with_condition_number, random_powerset, - random_powerset_group_conditional, + random_powerset_label_min, random_subset_of_size, running_moments, ) @@ -253,26 +253,24 @@ def test_running_moments(): @pytest.mark.parametrize( - "min_elements,elements_per_group,num_groups,check_num_samples", - [(1, 10, 3, 3), (2, 10, 3, 3)], + "min_elements_per_label,num_elements_per_label,num_labels,check_num_samples", + [(0, 10, 3, 1000), (1, 10, 3, 1000), (2, 10, 3, 1000)], ) -def test_random_powerset_group_conditional( - min_elements: int, - elements_per_group: int, - num_groups: int, +def test_random_powerset_label_min( + min_elements_per_label: int, + num_elements_per_label: int, + num_labels: int, check_num_samples: int, ): - s = np.arange(num_groups * elements_per_group) - groups = np.arange(num_groups).repeat(elements_per_group) + s = np.arange(num_labels * num_elements_per_label) + labels = np.arange(num_labels).repeat(num_elements_per_label) for idx, subset in enumerate( - random_powerset_group_conditional(s, groups, min_elements) + random_powerset_label_min(s, labels, min_elements_per_label) ): assert np.all(np.isin(subset, s)) - assert np.all(np.unique(groups[subset]) == np.unique(groups)) - - for group in np.unique(groups): - assert np.sum(group == groups[subset]) >= min_elements + for group in np.unique(labels): + assert np.sum(group == labels[subset]) >= min_elements_per_label if idx == check_num_samples: break diff --git a/tests/value/shapley/test_classwise.py b/tests/value/shapley/test_classwise.py index 923abbeac..bd4f55a5d 100644 --- a/tests/value/shapley/test_classwise.py +++ b/tests/value/shapley/test_classwise.py @@ -1,6 +1,3 @@ -""" -Test cases for the class wise shapley value. -""" from typing import Dict, Tuple, cast import numpy as np @@ -19,216 +16,9 @@ @pytest.fixture(scope="function") -def linear_classifier_cs_scorer_args_exact_solution_use_default_score() -> Tuple[ - Dict, ValuationResult, Dict -]: - r""" - Returns the exact solution for the class wise shapley value of the training and - validation set of the `utility_alt_seq_cf_linear_classifier_cs_scorer` fixture. - - =========================== - CS-Shapley Manual Derivation - =========================== - - :Author: Markus Semmler - :Date: August 2023 - - Dataset description - =================== - - We have a training and a test dataset. We want to model a simple XOR dataset. The - development set :math:`D` is given by - - .. math:: - \begin{aligned} - \hat{x}_0 &= 1 \quad &\hat{y}_0 = 0 \\ - \hat{x}_1 &= 2 \quad &\hat{y}_1 = 0 \\ - \hat{x}_2 &= 3 \quad &\hat{y}_2 = 0 \\ - \hat{x}_3 &= 4 \quad &\hat{y}_3 = 1 \\ - \end{aligned} - - and the training set :math:`T` is given by - - .. math:: - \begin{aligned} - x_0 &= 1 \quad &y_0 = 0 \\ - x_1 &= 2 \quad &y_1 = 0 \\ - x_2 &= 3 \quad &y_2 = 1 \\ - x_3 &= 4 \quad &y_3 = 1 \\ - \end{aligned} - - Note that the training set and the development set contain the same - inputs x, but differ in the label :math:`\hat{y}_2 \neq y_2` - - Model - ===== - - We use an adapted version of linear regression - - .. math:: y = \max(0, \min(1, \text{round}(\beta^T x))) - - for classification, with the closed form solution - - .. math:: \beta = \frac{\text{dot}(x, y)}{\text{dot}(x, x)} - - Fitted model - ============ - - The hyperparameters for all combinations are - - .. container:: tabular - - | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & - :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & - :math:`\{x_2, x_3\}` - | :math:`\emptyset` & nan & :math:`\frac{1}{3}` & :math:`\frac{1}{4}` - & :math:`\frac{7}{25}` - | :math:`\{x_0\}` & :math:`0` & :math:`\frac{3}{10}` & - :math:`\frac{4}{17}` & :math:`\frac{7}{26}` - | :math:`\{x_1\}` & :math:`0` & :math:`\frac{3}{13}` & - :math:`\frac{1}{5}` &\ :math:`\frac{7}{29}` - | :math:`\{x_0, x_1 \}` & :math:`0` & :math:`\frac{3}{14}` & - :math:`\frac{4}{21}` & :math:`\frac{7}{30}` - - Accuracy tables on development set :math:`D` - ============================================ - - (*) Note that the algorithm described in the paper overwrites these - values with 0. - - .. container:: tabular - - | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & - :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & - :math:`\{x_2, x_3\}` - | :math:`\emptyset` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` & :math:`\frac{1}{4}` - | :math:`\{x_0\}` & :math:`\frac{3}{4}` & :math:`\frac{1}{4}` & - :math:`\frac{1}{2}` & :math:`\frac{1}{4}` - | :math:`\{x_1\}` & :math:`\frac{3}{4}` & :math:`\frac{1}{2}` & - :math:`\frac{1}{2}` &\ :math:`\frac{1}{2}` - | :math:`\{x_0, x_1 \}` & :math:`\frac{3}{4}` & :math:`\frac{1}{2}` & - :math:`\frac{1}{2}` & :math:`\frac{1}{2}` - - .. container:: tabular - - | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & - :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & - :math:`\{x_2, x_3\}` - | :math:`\emptyset` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` & :math:`\frac{1}{4}` - | :math:`\{x_0\}` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` & :math:`\frac{1}{4}` - | :math:`\{x_1\}` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` &\ :math:`\frac{1}{4}` - | :math:`\{x_0, x_1 \}` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` & :math:`\frac{1}{4}` - - CS-Shapley - ========== - - The formulas of the algorithm are given by - - .. math:: - - \begin{aligned} - \delta(\pi, S_{-y_i}, i) &= v_{y_i}(\pi_{:i} \cup \{ i \} | S_{-y_i}) - - v_{y_i}(\pi_{:i} | S_{-y_i}) \\ - \left [ \phi_i | S_{-y_i} \right ] &= \frac{1}{|T_{y_i}|!} - \sum_{\pi \in \Pi(T_{y_i})} \delta(\pi, S_{-y_i}, i) \\ - \phi_i &= \frac{1}{2^{|T_{-y_i}|}-1} \left [\sum_{\emptyset \subset S_{-y_i} - \subseteq T_{-y_i}} \left [ \phi_i | S_{-y_i} \right ] \right ] - \end{aligned} - - Valuation of :math:`x_0` - ======================== - - .. math:: - \begin{aligned} - \delta((x_0, x_1), \{ x_2 \}, 0) &= \frac{1}{4} e^\frac{1}{4} &\quad - \delta((x_1, x_0), \{ x_2 \}, 0) &= 0 \\ - \delta((x_0, x_1), \{ x_3 \}, 0) &= \frac{1}{2} e^\frac{1}{4} &\quad - \delta((x_1, x_0), \{ x_3 \}, 0) &= 0 \\ - \delta((x_0, x_1), \{ x_2, x_3 \}, 0) &= \frac{1}{4} e^\frac{1}{4} &\quad - \delta((x_1, x_0), \{ x_2, x_3 \}, 0) &= 0 - \end{aligned} - - .. math:: - \begin{aligned} - \left [ \phi_0 | \{ x_2 \} \right] &= \frac{1}{8} e^\frac{1}{4} \\ - \left [ \phi_0 | \{ x_3 \} \right] &= \frac{1}{4} e^\frac{1}{4} \\ - \left [ \phi_0 | \{ x_2, x_3 \} \right] &= \frac{1}{8} e^\frac{1}{4} - \end{aligned} - - .. math:: \phi_0 = \frac{1}{6} e^\frac{1}{4} \approx 0.214 - - Valuation of :math:`x_1` - ======================== - - .. math:: - \begin{aligned} - \delta((x_0, x_1), \{ x_2 \}, 1) &= \frac{1}{4} e^\frac{1}{4} &\quad - \delta((x_1, x_0), \{ x_2 \}, 1) &= \frac{1}{2} e^\frac{1}{4} \\ - \delta((x_0, x_1), \{ x_3 \}, 1) &= 0 &\quad - \delta((x_1, x_0), \{ x_3 \}, 1) &= \frac{1}{2} e^\frac{1}{4} \\ - \delta((x_0, x_1), \{ x_2, x_3 \}, 1) &= \frac{1}{4} e^\frac{1}{4} &\quad - \delta((x_1, x_0), \{ x_2, x_3 \}, 1) &= \frac{1}{2} e^\frac{1}{4} - \end{aligned} - - .. math:: - \begin{aligned} - \left [ \phi_1 | \{ x_2 \} \right] &= \frac{3}{8} e^\frac{1}{4} \\ - \left [ \phi_1 | \{ x_3 \} \right] &= \frac{1}{4} e^\frac{1}{4} \\ - \left [ \phi_1 | \{ x_2, x_3 \} \right] &= \frac{3}{8} e^\frac{1}{4} - \end{aligned} - - .. math:: \phi_0 = \frac{1}{3} e^\frac{1}{4} \approx 0.428 - - Valuation of :math:`x_2` - ======================== - - .. math:: - \begin{aligned} - \delta((x_2, x_3), \{ x_0 \}, 2) &= \frac{1}{4} e^\frac{1}{4} &\quad - \delta((x_3, x_2), \{ x_0 \}, 2) - &= \frac{1}{4} e^\frac{1}{4} - \frac{1}{4} e^\frac{1}{2} \\ - \delta((x_2, x_3), \{ x_1 \}, 2) &= \frac{1}{4} e^\frac{1}{2} &\quad - \delta((x_3, x_2), \{ x_1 \}, 2) &= 0 \\ - \delta((x_2, x_3), \{ x_0, x_1 \}, 2) &= \frac{1}{4} e^\frac{1}{2} &\quad - \delta((x_3, x_2), \{ x_0, x_1 \}, 2) &= 0 - \end{aligned} - - .. math:: - \begin{aligned} - \left [ \phi_2 | \{ x_0 \} \right] - &= \frac{1}{4} e^\frac{1}{4} - \frac{1}{8} e^\frac{1}{2} \\ - \left [ \phi_2 | \{ x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ - \left [ \phi_2 | \{ x_0, x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} - \end{aligned} - - .. math:: \phi_2 = \frac{1}{12} e^\frac{1}{4} + \frac{1}{24} e^\frac{1}{2} \approx 0.1757 - - Valuation of :math:`x_3` - ======================== - - .. math:: - \begin{aligned} - \delta((x_2, x_3), \{ x_0 \}, 3) &= 0 &\quad - \delta((x_3, x_2), \{ x_0 \}, 3) &= \frac{1}{4} e^\frac{1}{2} \\ - \delta((x_2, x_3), \{ x_1 \}, 3) &= 0 &\quad - \delta((x_3, x_2), \{ x_1 \}, 3) &= \frac{1}{4} e^\frac{1}{2} \\ - \delta((x_2, x_3), \{ x_0, x_1 \}, 3) &= 0 &\quad - \delta((x_3, x_2), \{ x_0, x_1 \}, 3) &= \frac{1}{4} e^\frac{1}{2} - \end{aligned} - - .. math:: - \begin{aligned} - \left [ \phi_3 | \{ x_0 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ - \left [ \phi_3 | \{ x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ - \left [ \phi_3 | \{ x_0, x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} - \end{aligned} - - .. math:: \phi_3 = \frac{1}{8} e^\frac{1}{2} \approx 0.2061 +def classwise_shapley_exact_solution() -> Tuple[Dict, ValuationResult, Dict]: + """ + See [classwise.py][pydvl.value.shapley.classwise] for details of the derivation. """ return ( { @@ -249,17 +39,14 @@ def linear_classifier_cs_scorer_args_exact_solution_use_default_score() -> Tuple @pytest.fixture(scope="function") -def linear_classifier_cs_scorer_args_exact_solution_use_default_score_norm( - linear_classifier_cs_scorer_args_exact_solution_use_default_score: Tuple[ - Dict, ValuationResult, Dict - ] +def classwise_shapley_exact_solution_normalized( + classwise_shapley_exact_solution, ) -> Tuple[Dict, ValuationResult, Dict]: """ - Same as :func:`linear_classifier_cs_scorer_args_exact_solution_use_default_score` - but with normalization. The values of label c are normalized by the in-class score - of label c divided by the sum of values of that specific label. + It additionally normalizes the values using the argument `normalize_values`. See + [classwise.py][pydvl.value.shapley.classwise] for details of the derivation. """ - values = linear_classifier_cs_scorer_args_exact_solution_use_default_score[1].values + values = classwise_shapley_exact_solution[1].values label_zero_coefficient = 1 / np.exp(1 / 4) label_one_coefficient = 1 / (1 / 3 * np.exp(1 / 4) + 2 / 3 * np.exp(1 / 2)) @@ -282,213 +69,11 @@ def linear_classifier_cs_scorer_args_exact_solution_use_default_score_norm( @pytest.fixture(scope="function") -def linear_classifier_cs_scorer_args_exact_solution_use_add_idx() -> Tuple[ - Dict, ValuationResult, Dict -]: - r""" - Returns the exact solution for the class wise shapley value of the training and - validation set of the `utility_alt_seq_cf_linear_classifier_cs_scorer` fixture. - - =========================== - CS-Shapley Manual Derivation - =========================== - - :Author: Markus Semmler - :Date: August 2023 - - Dataset description - =================== - - We have a training and a test dataset. We want to model a simple XOR dataset. The - development set :math:`D` is given by - - .. math:: - \begin{aligned} - \hat{x}_0 &= 1 \quad &\hat{y}_0 = 0 \\ - \hat{x}_1 &= 2 \quad &\hat{y}_1 = 0 \\ - \hat{x}_2 &= 3 \quad &\hat{y}_2 = 0 \\ - \hat{x}_3 &= 4 \quad &\hat{y}_3 = 1 \\ - \end{aligned} - - and the training set :math:`T` is given by - - .. math:: - \begin{aligned} - x_0 &= 1 \quad &y_0 = 0 \\ - x_1 &= 2 \quad &y_1 = 0 \\ - x_2 &= 3 \quad &y_2 = 1 \\ - x_3 &= 4 \quad &y_3 = 1 \\ - \end{aligned} - - Note that the training set and the development set contain the same - inputs x, but differ in the label :math:`\hat{y}_2 \neq y_2` - - Model - ===== - - We use an adapted version of linear regression - - .. math:: y = \max(0, \min(1, \text{round}(\beta^T x))) - - for classification, with the closed form solution - - .. math:: \beta = \frac{\text{dot}(x, y)}{\text{dot}(x, x)} - - Fitted model - ============ - - The hyperparameters for all combinations are - - .. container:: tabular - - | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & - :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & - :math:`\{x_2, x_3\}` - | :math:`\emptyset` & nan & :math:`\frac{1}{3}` & :math:`\frac{1}{4}` - & :math:`\frac{7}{25}` - | :math:`\{x_0\}` & :math:`0` & :math:`\frac{3}{10}` & - :math:`\frac{4}{17}` & :math:`\frac{7}{26}` - | :math:`\{x_1\}` & :math:`0` & :math:`\frac{3}{13}` & - :math:`\frac{1}{5}` &\ :math:`\frac{7}{29}` - | :math:`\{x_0, x_1 \}` & :math:`0` & :math:`\frac{3}{14}` & - :math:`\frac{4}{21}` & :math:`\frac{7}{30}` - - Accuracy tables on development set :math:`D` - ============================================ - - .. container:: tabular - - | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & - :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & - :math:`\{x_2, x_3\}` - | :math:`\emptyset` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` & :math:`\frac{1}{4}` - | :math:`\{x_0\}` & :math:`\frac{3}{4}` & :math:`\frac{1}{4}` & - :math:`\frac{1}{2}` & :math:`\frac{1}{4}` - | :math:`\{x_1\}` & :math:`\frac{3}{4}` & :math:`\frac{1}{2}` & - :math:`\frac{1}{2}` &\ :math:`\frac{1}{2}` - | :math:`\{x_0, x_1 \}` & :math:`\frac{3}{4}` & :math:`\frac{1}{2}` & - :math:`\frac{1}{2}` & :math:`\frac{1}{2}` - - .. container:: tabular - - | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & - :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & - :math:`\{x_2, x_3\}` - | :math:`\emptyset` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` & :math:`\frac{1}{4}` - | :math:`\{x_0\}` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` & :math:`\frac{1}{4}` - | :math:`\{x_1\}` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` &\ :math:`\frac{1}{4}` - | :math:`\{x_0, x_1 \}` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` & :math:`\frac{1}{4}` - - CS-Shapley - ========== - - The formulas of the algorithm are given by - - .. math:: - - \begin{aligned} - \delta(\pi, S_{-y_i}, i) &= v_{y_i}(\pi_{:i} \cup \{ i \} | S_{-y_i}) - - v_{y_i}(\pi_{:i} | S_{-y_i}) \\ - \left [ \phi_i | S_{-y_i} \right ] &= \frac{1}{|T_{y_i}|!} - \sum_{\pi \in \Pi(T_{y_i})} \delta(\pi, S_{-y_i}, i) \\ - \phi_i &= \frac{1}{2^{|T_{-y_i}|}-1} \left [\sum_{\emptyset \subset S_{-y_i} - \subseteq T_{-y_i}} \left [ \phi_i | S_{-y_i} \right ] \right ] - \end{aligned} - - Valuation of :math:`x_0` - ======================== - - .. math:: - \begin{aligned} - \delta((x_0, x_1), \{ x_2 \}, 0) &= 0 &\quad - \delta((x_1, x_0), \{ x_2 \}, 0) &= 0 \\ - \delta((x_0, x_1), \{ x_3 \}, 0) &= \frac{1}{4} e^\frac{1}{4} &\quad - \delta((x_1, x_0), \{ x_3 \}, 0) &= 0 \\ - \delta((x_0, x_1), \{ x_2, x_3 \}, 0) &= 0 &\quad - \delta((x_1, x_0), \{ x_2, x_3 \}, 0) &= 0 - \end{aligned} - - .. math:: - \begin{aligned} - \left [ \phi_0 | \{ x_2 \} \right] &= 0 \\ - \left [ \phi_0 | \{ x_3 \} \right] &= \frac{1}{8} e^\frac{1}{4} \\ - \left [ \phi_0 | \{ x_2, x_3 \} \right] &= 0 - \end{aligned} - - .. math:: \phi_0 = \frac{1}{24} e^\frac{1}{4} \approx 0.0535 - - Valuation of :math:`x_1` - ======================== - - .. math:: - \begin{aligned} - \delta((x_0, x_1), \{ x_2 \}, 1) &= \frac{1}{4} e^\frac{1}{4} &\quad - \delta((x_1, x_0), \{ x_2 \}, 1) &= \frac{1}{4} e^\frac{1}{4} \\ - \delta((x_0, x_1), \{ x_3 \}, 1) &= 0 &\quad - \delta((x_1, x_0), \{ x_3 \}, 1) &= \frac{1}{4} e^\frac{1}{4} \\ - \delta((x_0, x_1), \{ x_2, x_3 \}, 1) &= \frac{1}{4} e^\frac{1}{4} &\quad - \delta((x_1, x_0), \{ x_2, x_3 \}, 1) &= \frac{1}{4} e^\frac{1}{4} - \end{aligned} - - .. math:: - \begin{aligned} - \left [ \phi_1 | \{ x_2 \} \right] &= \frac{1}{4} e^\frac{1}{4} \\ - \left [ \phi_1 | \{ x_3 \} \right] &= \frac{1}{8} e^\frac{1}{4} \\ - \left [ \phi_1 | \{ x_2, x_3 \} \right] &= \frac{1}{4} e^\frac{1}{4} - \end{aligned} - - .. math:: \phi_0 = \frac{5}{24} e^\frac{1}{4} \approx 0.2675 - - Valuation of :math:`x_2` - ======================== - - .. math:: - \begin{aligned} - \delta((x_2, x_3), \{ x_0 \}, 2) &= \frac{1}{4} e^\frac{1}{4} &\quad - \delta((x_3, x_2), \{ x_0 \}, 2) - &= \frac{1}{4} e^\frac{1}{4} - \frac{1}{4} e^\frac{1}{2} \\ - \delta((x_2, x_3), \{ x_1 \}, 2) &= \frac{1}{4} e^\frac{1}{2} &\quad - \delta((x_3, x_2), \{ x_1 \}, 2) &= 0 \\ - \delta((x_2, x_3), \{ x_0, x_1 \}, 2) &= \frac{1}{4} e^\frac{1}{2} &\quad - \delta((x_3, x_2), \{ x_0, x_1 \}, 2) &= 0 - \end{aligned} - - .. math:: - \begin{aligned} - \left [ \phi_2 | \{ x_0 \} \right] - &= \frac{1}{4} e^\frac{1}{4} - \frac{1}{8} e^\frac{1}{2} \\ - \left [ \phi_2 | \{ x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ - \left [ \phi_2 | \{ x_0, x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} - \end{aligned} - - .. math:: \phi_2 = \frac{1}{12} e^\frac{1}{4} + \frac{1}{24} e^\frac{1}{2} \approx 0.1757 - - Valuation of :math:`x_3` - ======================== - - .. math:: - \begin{aligned} - \delta((x_2, x_3), \{ x_0 \}, 3) &= 0 &\quad - \delta((x_3, x_2), \{ x_0 \}, 3) &= \frac{1}{4} e^\frac{1}{2} \\ - \delta((x_2, x_3), \{ x_1 \}, 3) &= 0 &\quad - \delta((x_3, x_2), \{ x_1 \}, 3) &= \frac{1}{4} e^\frac{1}{2} \\ - \delta((x_2, x_3), \{ x_0, x_1 \}, 3) &= 0 &\quad - \delta((x_3, x_2), \{ x_0, x_1 \}, 3) &= \frac{1}{4} e^\frac{1}{2} - \end{aligned} - - .. math:: - \begin{aligned} - \left [ \phi_3 | \{ x_0 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ - \left [ \phi_3 | \{ x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ - \left [ \phi_3 | \{ x_0, x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} - \end{aligned} - - .. math:: \phi_3 = \frac{1}{8} e^\frac{1}{2} \approx 0.2061 +def classwise_shapley_exact_solution_no_default() -> Tuple[Dict, ValuationResult, Dict]: + """ + Note that this special case doesn't set the utility to 0 if the permutation is + empty. See [classwise.py][pydvl.value.shapley.classwise] for details of the + derivation. """ return ( { @@ -510,227 +95,13 @@ def linear_classifier_cs_scorer_args_exact_solution_use_add_idx() -> Tuple[ @pytest.fixture(scope="function") -def linear_classifier_cs_scorer_args_exact_solution_use_add_idx_empty_set() -> Tuple[ - Dict, ValuationResult, Dict -]: +def classwise_shapley_exact_solution_no_default_allow_empty_set() -> ( + Tuple[Dict, ValuationResult, Dict] +): r""" - Returns the exact solution for the class wise shapley value of the training and - validation set of the `utility_alt_seq_cf_linear_classifier_cs_scorer` fixture. - - =========================== - CS-Shapley Manual Derivation - =========================== - - :Author: Markus Semmler - :Date: August 2023 - - Dataset description - =================== - - We have a training and a test dataset. We want to model a simple XOR dataset. The - development set :math:`D` is given by - - .. math:: - \begin{aligned} - \hat{x}_0 &= 1 \quad &\hat{y}_0 = 0 \\ - \hat{x}_1 &= 2 \quad &\hat{y}_1 = 0 \\ - \hat{x}_2 &= 3 \quad &\hat{y}_2 = 0 \\ - \hat{x}_3 &= 4 \quad &\hat{y}_3 = 1 \\ - \end{aligned} - - and the training set :math:`T` is given by - - .. math:: - \begin{aligned} - x_0 &= 1 \quad &y_0 = 0 \\ - x_1 &= 2 \quad &y_1 = 0 \\ - x_2 &= 3 \quad &y_2 = 1 \\ - x_3 &= 4 \quad &y_3 = 1 \\ - \end{aligned} - - Note that the training set and the development set contain the same - inputs x, but differ in the label :math:`\hat{y}_2 \neq y_2` - - Model - ===== - - We use an adapted version of linear regression - - .. math:: y = \max(0, \min(1, \text{round}(\beta^T x))) - - for classification, with the closed form solution - - .. math:: \beta = \frac{\text{dot}(x, y)}{\text{dot}(x, x)} - - Fitted model - ============ - - The hyperparameters for all combinations are - - .. container:: tabular - - | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & - :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & - :math:`\{x_2, x_3\}` - | :math:`\emptyset` & nan & :math:`\frac{1}{3}` & :math:`\frac{1}{4}` - & :math:`\frac{7}{25}` - | :math:`\{x_0\}` & :math:`0` & :math:`\frac{3}{10}` & - :math:`\frac{4}{17}` & :math:`\frac{7}{26}` - | :math:`\{x_1\}` & :math:`0` & :math:`\frac{3}{13}` & - :math:`\frac{1}{5}` &\ :math:`\frac{7}{29}` - | :math:`\{x_0, x_1 \}` & :math:`0` & :math:`\frac{3}{14}` & - :math:`\frac{4}{21}` & :math:`\frac{7}{30}` - - Accuracy tables on development set :math:`D` - ============================================ - - .. container:: tabular - - | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & - :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & - :math:`\{x_2, x_3\}` - | :math:`\emptyset` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` & :math:`\frac{1}{4}` - | :math:`\{x_0\}` & :math:`\frac{3}{4}` & :math:`\frac{1}{4}` & - :math:`\frac{1}{2}` & :math:`\frac{1}{4}` - | :math:`\{x_1\}` & :math:`\frac{3}{4}` & :math:`\frac{1}{2}` & - :math:`\frac{1}{2}` &\ :math:`\frac{1}{2}` - | :math:`\{x_0, x_1 \}` & :math:`\frac{3}{4}` & :math:`\frac{1}{2}` & - :math:`\frac{1}{2}` & :math:`\frac{1}{2}` - - .. container:: tabular - - | \|c||Sc \| Sc \| Sc \| Sc \| :math:`S_1 \cup S_2` & - :math:`\emptyset` & :math:`\{x_2\}` & :math:`\{x_3\}` & - :math:`\{x_2, x_3\}` - | :math:`\emptyset` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` & :math:`\frac{1}{4}` - | :math:`\{x_0\}` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` & :math:`\frac{1}{4}` - | :math:`\{x_1\}` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` &\ :math:`\frac{1}{4}` - | :math:`\{x_0, x_1 \}` & :math:`0` & :math:`\frac{1}{4}` & - :math:`\frac{1}{4}` & :math:`\frac{1}{4}` - - CS-Shapley - ========== - - The formulas of the algorithm are given by - - .. math:: - - \begin{aligned} - \delta(\pi, S_{-y_i}, i) &= v_{y_i}(\pi_{:i} \cup \{ i \} | S_{-y_i}) - - v_{y_i}(\pi_{:i} | S_{-y_i}) \\ - \left [ \phi_i | S_{-y_i} \right ] &= \frac{1}{|T_{y_i}|!} - \sum_{\pi \in \Pi(T_{y_i})} \delta(\pi, S_{-y_i}, i) \\ - \phi_i &= \frac{1}{2^{|T_{-y_i}|}} \left [\sum_{S_{-y_i} - \subseteq T_{-y_i}} \left [ \phi_i | S_{-y_i} \right ] \right ] - \end{aligned} - - Valuation of :math:`x_0` - ======================== - - .. math:: - \begin{aligned} - \delta((x_0, x_1), \emptyset, 0) &= \frac{3}{4} &\quad - \delta((x_1, x_0), \emptyset, 0) &= 0 \\ - \delta((x_0, x_1), \{ x_2 \}, 0) &= 0 &\quad - \delta((x_1, x_0), \{ x_2 \}, 0) &= 0 \\ - \delta((x_0, x_1), \{ x_3 \}, 0) &= \frac{1}{4} e^\frac{1}{4} &\quad - \delta((x_1, x_0), \{ x_3 \}, 0) &= 0 \\ - \delta((x_0, x_1), \{ x_2, x_3 \}, 0) &= 0 &\quad - \delta((x_1, x_0), \{ x_2, x_3 \}, 0) &= 0 - \end{aligned} - - .. math:: - \begin{aligned} - \left [ \phi_0 | \emptyset \right] &= \frac{3}{8} \\ - \left [ \phi_0 | \{ x_2 \} \right] &= 0 \\ - \left [ \phi_0 | \{ x_3 \} \right] &= \frac{1}{8} e^\frac{1}{4} \\ - \left [ \phi_0 | \{ x_2, x_3 \} \right] &= 0 - \end{aligned} - - .. math:: \phi_0 = \frac{3}{32} + \frac{1}{32} e^\frac{1}{4} \approx 0.1339 - - Valuation of :math:`x_1` - ======================== - - .. math:: - \begin{aligned} - \delta((x_0, x_1), \emptyset, 1) &= 0 &\quad - \delta((x_1, x_0), \emptyset, 1) &= \frac{3}{4} \\ - \delta((x_0, x_1), \{ x_2 \}, 1) &= \frac{1}{4} e^\frac{1}{4} &\quad - \delta((x_1, x_0), \{ x_2 \}, 1) &= \frac{1}{4} e^\frac{1}{4} \\ - \delta((x_0, x_1), \{ x_3 \}, 1) &= 0 &\quad - \delta((x_1, x_0), \{ x_3 \}, 1) &= \frac{1}{4} e^\frac{1}{4} \\ - \delta((x_0, x_1), \{ x_2, x_3 \}, 1) &= \frac{1}{4} e^\frac{1}{4} &\quad - \delta((x_1, x_0), \{ x_2, x_3 \}, 1) &= \frac{1}{4} e^\frac{1}{4} - \end{aligned} - - .. math:: - \begin{aligned} - \left [ \phi_1 | \emptyset \right] &= \frac{3}{8} \\ - \left [ \phi_1 | \{ x_2 \} \right] &= \frac{1}{4} e^\frac{1}{4} \\ - \left [ \phi_1 | \{ x_3 \} \right] &= \frac{1}{8} e^\frac{1}{4} \\ - \left [ \phi_1 | \{ x_2, x_3 \} \right] &= \frac{1}{4} e^\frac{1}{4} - \end{aligned} - - .. math:: \phi_0 = \frac{3}{32} + \frac{5}{32} e^\frac{1}{4} \approx 0.2944 - - Valuation of :math:`x_2` - ======================== - - .. math:: - \begin{aligned} - \delta((x_2, x_3), \emptyset, 2) &= \frac{1}{4} e^\frac{1}{4} &\quad - \delta((x_3, x_2), \emptyset, 2) &= 0 \\ - \delta((x_2, x_3), \{ x_0 \}, 2) &= \frac{1}{4} e^\frac{1}{4} &\quad - \delta((x_3, x_2), \{ x_0 \}, 2) - &= \frac{1}{4} e^\frac{1}{4} - \frac{1}{4} e^\frac{1}{2} \\ - \delta((x_2, x_3), \{ x_1 \}, 2) &= \frac{1}{4} e^\frac{1}{2} &\quad - \delta((x_3, x_2), \{ x_1 \}, 2) &= 0 \\ - \delta((x_2, x_3), \{ x_0, x_1 \}, 2) &= \frac{1}{4} e^\frac{1}{2} &\quad - \delta((x_3, x_2), \{ x_0, x_1 \}, 2) &= 0 - \end{aligned} - - .. math:: - \begin{aligned} - \left [ \phi_2 | \emptyset \right] &= \frac{1}{8} e^\frac{1}{4} \\ - \left [ \phi_2 | \{ x_0 \} \right] - &= \frac{1}{4} e^\frac{1}{4} - \frac{1}{8} e^\frac{1}{2} \\ - \left [ \phi_2 | \{ x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ - \left [ \phi_2 | \{ x_0, x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} - \end{aligned} - - .. math:: - \phi_2 = \frac{5}{32} e^\frac{1}{4} + \frac{1}{32} e^\frac{1}{2} \approx 0.2522 - - Valuation of :math:`x_3` - ======================== - - .. math:: - \begin{aligned} - \delta((x_2, x_3), \emptyset, 3) &= 0 &\quad - \delta((x_3, x_2), \emptyset, 3) &= \frac{1}{4} e^\frac{1}{4} \\ - \delta((x_2, x_3), \{ x_0 \}, 3) &= 0 &\quad - \delta((x_3, x_2), \{ x_0 \}, 3) &= \frac{1}{4} e^\frac{1}{2} \\ - \delta((x_2, x_3), \{ x_1 \}, 3) &= 0 &\quad - \delta((x_3, x_2), \{ x_1 \}, 3) &= \frac{1}{4} e^\frac{1}{2} \\ - \delta((x_2, x_3), \{ x_0, x_1 \}, 3) &= 0 &\quad - \delta((x_3, x_2), \{ x_0, x_1 \}, 3) &= \frac{1}{4} e^\frac{1}{2} - \end{aligned} - - .. math:: - \begin{aligned} - \left [ \phi_3 | \emptyset \right] &= \frac{1}{8} e^\frac{1}{4} \\ - \left [ \phi_3 | \{ x_0 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ - \left [ \phi_3 | \{ x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} \\ - \left [ \phi_3 | \{ x_0, x_1 \} \right] &= \frac{1}{8} e^\frac{1}{2} - \end{aligned} - - .. math:: - \phi_3 = \frac{1}{32} e^\frac{1}{4} + \frac{3}{32} e^\frac{1}{2} \approx 0.1947 + Note that this special case doesn't set the utility to 0 if the permutation is + empty and additionally allows $S^{(k)} = \emptyset$. See + [classwise.py][pydvl.value.shapley.classwise] for details of the derivation. """ return ( { @@ -759,26 +130,24 @@ def linear_classifier_cs_scorer_args_exact_solution_use_add_idx_empty_set() -> T ids=lambda x: "n_resample_complement_sets={}".format(x), ) @pytest.mark.parametrize( - "linear_classifier_cs_scorer_args_exact_solution", + "exact_solution", [ - "linear_classifier_cs_scorer_args_exact_solution_use_default_score", - "linear_classifier_cs_scorer_args_exact_solution_use_default_score_norm", - "linear_classifier_cs_scorer_args_exact_solution_use_add_idx", - "linear_classifier_cs_scorer_args_exact_solution_use_add_idx_empty_set", + "classwise_shapley_exact_solution", + "classwise_shapley_exact_solution_normalized", + "classwise_shapley_exact_solution_no_default", + "classwise_shapley_exact_solution_no_default_allow_empty_set", ], ) def test_classwise_shapley( - linear_classifier_cs_scorer: Utility, - linear_classifier_cs_scorer_args_exact_solution: Tuple[Dict, ValuationResult], + classwise_shapley_utility: Utility, + exact_solution: Tuple[Dict, ValuationResult, Dict], n_samples: int, n_resample_complement_sets: int, request, ): - args, exact_solution, check_args = request.getfixturevalue( - linear_classifier_cs_scorer_args_exact_solution - ) + args, exact_solution, check_args = request.getfixturevalue(exact_solution) values = compute_classwise_shapley_values( - linear_classifier_cs_scorer, + classwise_shapley_utility, done=MaxChecks(n_samples), truncation=NoTruncation(), done_sample_complements=MaxChecks(n_resample_complement_sets), @@ -789,61 +158,79 @@ def test_classwise_shapley( assert np.all(values.counts == n_samples * n_resample_complement_sets) -@pytest.mark.parametrize("n_element, left_margin, right_margin", [(101, 0.3, 0.4)]) -def test_cs_scorer_on_dataset_alt_seq_simple(dataset_alt_seq_simple): +def test_classwise_scorer_representation(): """ - Tests the class wise scorer. + Tests the (string) representation of the ClassWiseScorer. """ scorer = ClasswiseScorer("accuracy", initial_label=0) assert str(scorer) == "classwise accuracy" assert repr(scorer) == "ClasswiseAccuracy (scorer=make_scorer(accuracy_score))" - x, y, info = dataset_alt_seq_simple + +@pytest.mark.parametrize("n_element, left_margin, right_margin", [(101, 0.3, 0.4)]) +def test_classwise_scorer_utility(dataset_left_right_margins): + """ + Tests whether the ClassWiseScorer returns the expected utility value. + See [classwise.py][pydvl.value.shapley.classwise] for more details. + """ + scorer = ClasswiseScorer("accuracy", initial_label=0) + x, y, info = dataset_left_right_margins n_element = len(x) target_in_cls_acc_0 = (info["left_margin"] * 100 + 1) / n_element target_out_of_cls_acc_0 = (info["right_margin"] * 100 + 1) / n_element model = ThresholdClassifier() - in_cls_acc_0, out_of_cls_acc_0 = scorer.estimate_in_cls_and_out_of_cls_score( + in_cls_acc_0, out_of_cls_acc_0 = scorer.estimate_in_class_and_out_of_class_score( model, x, y ) assert np.isclose(in_cls_acc_0, target_in_cls_acc_0) assert np.isclose(out_of_cls_acc_0, target_out_of_cls_acc_0) - scorer.label = 1 - in_cls_acc_1, out_of_cls_acc_1 = scorer.estimate_in_cls_and_out_of_cls_score( - model, x, y - ) - assert in_cls_acc_1 == out_of_cls_acc_0 - assert in_cls_acc_0 == out_of_cls_acc_1 - - scorer.label = 0 value = scorer(model, x, y) assert np.isclose(value, in_cls_acc_0 * np.exp(out_of_cls_acc_0)) scorer.label = 1 value = scorer(model, x, y) - assert np.isclose(value, in_cls_acc_1 * np.exp(out_of_cls_acc_1)) + assert np.isclose(value, out_of_cls_acc_0 * np.exp(in_cls_acc_0)) + + +@pytest.mark.parametrize("n_element, left_margin, right_margin", [(101, 0.3, 0.4)]) +def test_classwise_scorer_is_symmetric( + dataset_left_right_margins, +): + """ + Tests whether the ClassWiseScorer is symmetric. For a two-class classification the + in-class accuracy for the first label needs to match the out-of-class accuracy for + the second label. See [classwise.py][pydvl.value.shapley.classwise] for more + details. + """ + scorer = ClasswiseScorer("accuracy", initial_label=0) + x, y, info = dataset_left_right_margins + model = ThresholdClassifier() + in_cls_acc_0, out_of_cls_acc_0 = scorer.estimate_in_class_and_out_of_class_score( + model, x, y + ) + scorer.label = 1 + in_cls_acc_1, out_of_cls_acc_1 = scorer.estimate_in_class_and_out_of_class_score( + model, x, y + ) + assert in_cls_acc_1 == out_of_cls_acc_0 + assert in_cls_acc_0 == out_of_cls_acc_1 -def test_cs_scorer_on_alt_seq_cf_linear_classifier_cs_score( - linear_classifier_cs_scorer: Utility, +def test_classwise_scorer_accuracies_manual_derivation( + classwise_shapley_utility: Utility, ): + """ + Tests whether the model of the scorer is fitted correctly and returns the expected + in-class and out-of-class accuracies. See + [classwise.py][pydvl.value.shapley.classwise] for more details. + """ subsets_zero = list(powerset(np.array((0, 1)))) subsets_one = list(powerset(np.array((2, 3)))) subsets_zero = [tuple(s) for s in subsets_zero] subsets_one = [tuple(s) for s in subsets_one] - target_betas = pd.DataFrame( - [ - [np.nan, 1 / 3, 1 / 4, 7 / 25], - [0, 3 / 10, 4 / 17, 7 / 26], - [0, 3 / 13, 1 / 5, 7 / 29], - [0, 3 / 14, 4 / 21, 7 / 30], - ], - index=subsets_zero, - columns=subsets_one, - ) target_accuracies_zero = pd.DataFrame( [ [0, 1 / 4, 1 / 4, 1 / 4], @@ -864,8 +251,8 @@ def test_cs_scorer_on_alt_seq_cf_linear_classifier_cs_score( index=subsets_zero, columns=subsets_one, ) - model = linear_classifier_cs_scorer.model - scorer = cast(ClasswiseScorer, linear_classifier_cs_scorer.scorer) + model = classwise_shapley_utility.model + scorer = cast(ClasswiseScorer, classwise_shapley_utility.scorer) scorer.label = 0 for set_zero_idx in range(len(subsets_zero)): @@ -874,29 +261,86 @@ def test_cs_scorer_on_alt_seq_cf_linear_classifier_cs_score( ( x_train, y_train, - ) = linear_classifier_cs_scorer.data.get_training_data(indices) - linear_classifier_cs_scorer.model.fit(x_train, y_train) - fitted_beta = linear_classifier_cs_scorer.model._beta # noqa - target_beta = target_betas.iloc[set_zero_idx, set_one_idx] - assert ( - np.isnan(fitted_beta) - if np.isnan(target_beta) - else fitted_beta == target_beta - ) + ) = classwise_shapley_utility.data.get_training_data(indices) + classwise_shapley_utility.model.fit(x_train, y_train) ( x_test, y_test, - ) = linear_classifier_cs_scorer.data.get_test_data() - in_cls_acc_0, in_cls_acc_1 = scorer.estimate_in_cls_and_out_of_cls_score( - model, x_test, y_test - ) + ) = classwise_shapley_utility.data.get_test_data() + ( + in_cls_acc_0, + in_cls_acc_1, + ) = scorer.estimate_in_class_and_out_of_class_score(model, x_test, y_test) assert ( in_cls_acc_0 == target_accuracies_zero.iloc[set_zero_idx, set_one_idx] ) assert in_cls_acc_1 == target_accuracies_one.iloc[set_zero_idx, set_one_idx] +@pytest.mark.parametrize("n_element, left_margin, right_margin", [(101, 0.3, 0.4)]) +def test_classwise_scorer_accuracies_left_right_margins(dataset_left_right_margins): + """ + Tests whether the model of the scorer is fitted correctly and returns the expected + in-class and out-of-class accuracies. See + [classwise.py][pydvl.value.shapley.classwise] for more details. + """ + scorer = ClasswiseScorer("accuracy", initial_label=0) + x, y, info = dataset_left_right_margins + n_element = len(x) + + target_in_cls_acc_0 = (info["left_margin"] * 100 + 1) / n_element + target_out_of_cls_acc_0 = (info["right_margin"] * 100 + 1) / n_element + + model = ThresholdClassifier() + in_cls_acc_0, out_of_cls_acc_0 = scorer.estimate_in_class_and_out_of_class_score( + model, x, y + ) + assert np.isclose(in_cls_acc_0, target_in_cls_acc_0) + assert np.isclose(out_of_cls_acc_0, target_out_of_cls_acc_0) + + +def test_closed_form_linear_classifier( + classwise_shapley_utility: Utility, +): + """ + Tests whether the model is fitted correctly and contains the right $\beta$ + parameter. See [classwise.py][pydvl.value.shapley.classwise] for more details. + """ + subsets_zero = list(powerset(np.array((0, 1)))) + subsets_one = list(powerset(np.array((2, 3)))) + subsets_zero = [tuple(s) for s in subsets_zero] + subsets_one = [tuple(s) for s in subsets_one] + target_betas = pd.DataFrame( + [ + [np.nan, 1 / 3, 1 / 4, 7 / 25], + [0, 3 / 10, 4 / 17, 7 / 26], + [0, 3 / 13, 1 / 5, 7 / 29], + [0, 3 / 14, 4 / 21, 7 / 30], + ], + index=subsets_zero, + columns=subsets_one, + ) + scorer = cast(ClasswiseScorer, classwise_shapley_utility.scorer) + scorer.label = 0 + + for set_zero_idx in range(len(subsets_zero)): + for set_one_idx in range(len(subsets_one)): + indices = list(subsets_zero[set_zero_idx] + subsets_one[set_one_idx]) + ( + x_train, + y_train, + ) = classwise_shapley_utility.data.get_training_data(indices) + classwise_shapley_utility.model.fit(x_train, y_train) + fitted_beta = classwise_shapley_utility.model._beta # noqa + target_beta = target_betas.iloc[set_zero_idx, set_one_idx] + assert ( + np.isnan(fitted_beta) + if np.isnan(target_beta) + else fitted_beta == target_beta + ) + + class ThresholdClassifier: def fit(self, x: NDArray, y: NDArray) -> float: raise NotImplementedError("Mock model") @@ -932,19 +376,22 @@ def score(self, x: NDArray, y: NDArray) -> float: @pytest.fixture(scope="function") -def linear_classifier_cs_scorer( - dataset_alt_seq_full: Dataset, +def classwise_shapley_utility( + dataset_manual_derivation: Dataset, ) -> Utility: return Utility( ClosedFormLinearClassifier(), - dataset_alt_seq_full, + dataset_manual_derivation, ClasswiseScorer("accuracy"), catch_errors=False, ) @pytest.fixture(scope="function") -def dataset_alt_seq_full() -> Dataset: +def dataset_manual_derivation() -> Dataset: + """ + See [classwise.py][pydvl.value.shapley.classwise] for more details. + """ x_train = np.arange(1, 5).reshape([-1, 1]) y_train = np.array([0, 0, 1, 1]) x_test = x_train @@ -953,7 +400,7 @@ def dataset_alt_seq_full() -> Dataset: @pytest.fixture(scope="function") -def dataset_alt_seq_simple( +def dataset_left_right_margins( n_element: int, left_margin: float, right_margin: float ) -> Tuple[NDArray[np.float_], NDArray[np.int_], Dict[str, float]]: """ From 2554c9fc026fc5d8d2716af075756ba7668caa38 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Wed, 13 Sep 2023 04:57:16 +0200 Subject: [PATCH 21/43] Change pi to sigma. --- src/pydvl/value/shapley/classwise.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 0e0647b97..d7fb48253 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -32,11 +32,11 @@ $$ v_u(i) = \frac{1}{K} \sum_k \frac{1}{L} \sum_l -[u(\pi^{(l)}_{:i} \cup \{i\} | S^{(k)} ) − u( \pi^{(l)}_{:i} | S^{(k)})], +[u(\sigma^{(l)}_{:i} \cup \{i\} | S^{(k)} ) − u( \sigma^{(l)}_{:i} | S^{(k)})], $$ with $S^{(1)}, \dots, S^{(K)} \subseteq T_{-y_i}$ and - $\pi^{(1)}, \dots, \pi^{(L)} \in \Pi(T_{y_i}\setminus\{i\})$. + $\sigma^{(1)}, \dots, \sigma^{(L)} \in \Pi(T_{y_i}\setminus\{i\})$. ## Derivation of test case @@ -48,9 +48,9 @@ in closed form $\beta = \frac{\text{dot}(x, y)}{\text{dot}(x, x)}$. By using the tables that represent in-class accuracy $a_S(D_{y_i})$ and out-of-class accuracy $a_S(D_{-y_i})$ the Monte Carlo estimator with $\{S^{(1)}, \dots, S^{(K)}\} -= 2^{T_{-y_i}}$ and $\{\pi^{(1)}, \dots, pi^{(L)}\} = \Pi(T_{y_i}\setminus\{i\})$ can be -evaluated. Note that $2^M$ is the powerset of $M$. The details are left out to the -curious reader. += 2^{T_{-y_i}}$ and $\{\sigma^{(1)}, \dots, sigma^{(L)}\} = \Pi(T_{y_i}\setminus\{i\})$ +can be evaluated. Note that $2^M$ is the powerset of $M$. The details are left out to +the curious reader. # References From a95dae1514563e3390a30d74f378e7eb59887e8e Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Wed, 13 Sep 2023 15:55:10 +0200 Subject: [PATCH 22/43] Rename variable from `subset_length` to `subset_size`. --- src/pydvl/utils/numeric.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pydvl/utils/numeric.py b/src/pydvl/utils/numeric.py index 6399d0d98..ca41da43b 100644 --- a/src/pydvl/utils/numeric.py +++ b/src/pydvl/utils/numeric.py @@ -196,15 +196,15 @@ def random_powerset_label_min( subsets: List[NDArray[T]] = [] for label in unique_labels: label_indices = np.asarray(np.where(labels == label)[0]) - subset_length = int( + subset_size = int( rng.integers( min(min_elements_per_label, len(label_indices)), len(label_indices) + 1, ) ) - if subset_length > 0: + if subset_size > 0: subsets.append( - random_subset_of_size(s[label_indices], subset_length, seed=rng) + random_subset_of_size(s[label_indices], subset_size, seed=rng) ) if len(subsets) > 0: From 9f6ec301e5b7ef4ecffe2fa9cff31b51369887f4 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Wed, 13 Sep 2023 16:02:53 +0200 Subject: [PATCH 23/43] Adapt documentation. --- src/pydvl/value/shapley/classwise.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index d7fb48253..1470ca385 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -1,6 +1,4 @@ r""" -Implementation of Class-wise Shapley, introduced in (Schoch et al., 2022)[^1]. - Class-wise Shapley (Schoch et al., 2022)[^1] offers a distinct Shapley framework tailored for classification problems. Let $D$ be the dataset, $D_{y_i}$ be the subset of $D$ with labels $y_i$, and $D_{-y_i}$ be the complement of $D_{y_i}$ in $D$. The key idea is that @@ -45,12 +43,13 @@ $$y = \max(0, \min(1, \text{round}(\beta^T x)))$$ -in closed form $\beta = \frac{\text{dot}(x, y)}{\text{dot}(x, x)}$. By using the tables -that represent in-class accuracy $a_S(D_{y_i})$ and out-of-class accuracy -$a_S(D_{-y_i})$ the Monte Carlo estimator with $\{S^{(1)}, \dots, S^{(K)}\} -= 2^{T_{-y_i}}$ and $\{\sigma^{(1)}, \dots, sigma^{(L)}\} = \Pi(T_{y_i}\setminus\{i\})$ -can be evaluated. Note that $2^M$ is the powerset of $M$. The details are left out to -the curious reader. +in closed form $\beta = \frac{\text{dot}(x, y)}{\text{dot}(x, x)}$. From the closed-form +solution, the tables for in-class accuracy $a_S(D_{y_i})$ and out-of-class accuracy +$a_S(D_{-y_i})$ can be calculated. By using these tables and setting +$\{S^{(1)}, \dots, S^{(K)}\} = 2^{T_{-y_i}}$ and +$\{\sigma^{(1)}, \dots, \sigma^{(L)}\} = \Pi(T_{y_i}\setminus\{i\})$, +the Monte Carlo estimator can be evaluated ($2^M$ is the powerset of $M$). +The details of the derivation are left to the eager reader. # References From 037fa66fbfdbcc112d461f63df7cd335b3deeb7d Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Sat, 23 Sep 2023 02:46:52 +0200 Subject: [PATCH 24/43] Move imports fix some stuff and add to CHANGELOG.md --- CHANGELOG.md | 3 +++ src/pydvl/utils/numeric.py | 16 +++------------- src/pydvl/value/shapley/classwise.py | 10 ++++++---- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d548ff43d..c990ca1d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,9 @@ randomness. `compute_beta_shapley_semivalues`, `compute_shapley_semivalues` and `compute_generic_semivalues`. [PR #428](https://github.com/aai-institute/pyDVL/pull/428) +- Added classwise Shapley as proposed by (Schoch et al. 2021) + [https://arxiv.org/abs/2211.06800] + [PR #338](https://github.com/aai-institute/pyDVL/pull/338) ### Changed diff --git a/src/pydvl/utils/numeric.py b/src/pydvl/utils/numeric.py index dd22dad9a..679573a82 100644 --- a/src/pydvl/utils/numeric.py +++ b/src/pydvl/utils/numeric.py @@ -4,10 +4,6 @@ """ from __future__ import annotations -import logging -import os -import random -import time from itertools import chain, combinations from typing import ( Collection, @@ -17,8 +13,6 @@ Optional, Tuple, TypeVar, - Union, - cast, overload, ) @@ -39,10 +33,6 @@ "top_k_value_accuracy", ] - -logger = logging.getLogger(__name__) - - T = TypeVar("T", bound=np.generic) @@ -132,8 +122,7 @@ def random_powerset( Defaults to `np.iinfo(np.int32).max` q: Sampling probability for elements. The default 0.5 yields a uniform distribution over the power set of s. - seed: Either an instance of a numpy random number generator or a seed - for it. + seed: Either an instance of a numpy random number generator or a seed for it. Returns: Samples from the power set of `s`. @@ -184,7 +173,8 @@ def random_powerset_label_min( if min_elements_per_label < 0: raise ValueError( - f"Parameter min_elements={min_elements_per_label} needs to be bigger or equal to 0." + f"Parameter min_elements={min_elements_per_label} needs to be bigger or " + f"equal to 0." ) rng = np.random.default_rng(seed) diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 1470ca385..cfc6226d3 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -71,17 +71,19 @@ from numpy.typing import NDArray from tqdm import tqdm +from pydvl.parallel import ( + ParallelConfig, + effective_n_jobs, + init_executor, + init_parallel_backend, +) from pydvl.utils import ( Dataset, - ParallelConfig, Scorer, Seed, SupervisedModel, Utility, - effective_n_jobs, ensure_seed_sequence, - init_executor, - init_parallel_backend, random_powerset_label_min, ) from pydvl.value.result import ValuationResult From 94228d815f08f2b09b4836af79db947866676d23 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Mon, 25 Sep 2023 02:35:57 +0200 Subject: [PATCH 25/43] Fix that batch_size is overwritten. --- src/pydvl/value/semivalues.py | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/src/pydvl/value/semivalues.py b/src/pydvl/value/semivalues.py index 6876c27bd..3e876dcda 100644 --- a/src/pydvl/value/semivalues.py +++ b/src/pydvl/value/semivalues.py @@ -171,14 +171,6 @@ def _marginal( # deprecated_in="0.8.0", # remove_in="0.9.0", # ) -@deprecated( - target=True, - deprecated_in="0.7.0", - remove_in="0.9.0", - args_mapping={"batch_size": None}, - template_mgs="batch_size is for experimental use and will be removed" - "in future versions.", -) def compute_generic_semivalues( sampler: PowersetSampler[IndexT], u: Utility, @@ -307,14 +299,6 @@ def beta_coefficient_w(n: int, k: int) -> float: return cast(SVCoefficient, beta_coefficient_w) -@deprecated( - target=True, - deprecated_in="0.7.0", - remove_in="0.9.0", - args_mapping={"batch_size": None}, - template_mgs="batch_size is for experimental use and will be removed" - "in future versions.", -) def compute_shapley_semivalues( u: Utility, *, @@ -367,14 +351,6 @@ def compute_shapley_semivalues( ) -@deprecated( - target=True, - deprecated_in="0.7.0", - remove_in="0.9.0", - args_mapping={"batch_size": None}, - template_mgs="batch_size is for experimental use and will be removed" - "in future versions.", -) def compute_banzhaf_semivalues( u: Utility, *, @@ -425,14 +401,6 @@ def compute_banzhaf_semivalues( ) -@deprecated( - target=True, - deprecated_in="0.7.0", - remove_in="0.9.0", - args_mapping={"batch_size": None}, - template_mgs="batch_size is for experimental use and will be removed" - "in future versions.", -) def compute_beta_shapley_semivalues( u: Utility, *, From 5526031a4b2a1e9ffbf2eac8e757cde63d35cb4e Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Mon, 25 Sep 2023 21:03:25 +0200 Subject: [PATCH 26/43] Rework documentation. --- ...se-shapley-discounted-utility-function.svg | 67571 ++++++++++++++++ docs/value/shapley.md | 20 +- src/pydvl/value/shapley/classwise.py | 57 +- 3 files changed, 67608 insertions(+), 40 deletions(-) create mode 100644 docs/api/pydvl/value/shapley/classwise/img/classwise-shapley-discounted-utility-function.svg diff --git a/docs/api/pydvl/value/shapley/classwise/img/classwise-shapley-discounted-utility-function.svg b/docs/api/pydvl/value/shapley/classwise/img/classwise-shapley-discounted-utility-function.svg new file mode 100644 index 000000000..8fc402f9d --- /dev/null +++ b/docs/api/pydvl/value/shapley/classwise/img/classwise-shapley-discounted-utility-function.svg @@ -0,0 +1,67571 @@ + + + + + + + + 2023-09-25T20:19:20.731288 + image/svg+xml + + + Matplotlib v3.7.0, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/value/shapley.md b/docs/value/shapley.md index 954c7821d..4d1c78677 100644 --- a/docs/value/shapley.md +++ b/docs/value/shapley.md @@ -84,7 +84,7 @@ Class-wise Shapley [@schoch_csshapley_2022] offers a distinct Shapley framework for classification problems. Let $D$ be the dataset, $D_{y_i}$ be the subset of $D$ with labels $y_i$, and $D_{-y_i}$ be the complement of $D_{y_i}$ in $D$. The key idea is that a sample $(x_i, y_i)$, might enhance the overall performance on $D$, while being -detrimental for the performance on $D_{y_i}$. To address this nuanced behavior, the +detrimental for the performance on $D_{y_i}$. To address this issue, the authors introduced the estimator $$ @@ -95,17 +95,7 @@ $$ where $S_{y_i} \subseteq D_{y_i} \setminus \{i\}$ and $S_{-y_i} \subseteq D_{-y_i}$. In other words, the summations are over the powerset of $D_{y_i} \setminus \{i\}$ and -$D_{-y_i}$ respectively. The estimator employs a specialized utility function - -$$ -u(S_{y_i}|S_{-y_i}) = a_S(D_{y_i}) \exp(a_S(D_{-y_i})), -$$ - -where $S=S_{y_i} \cup S_{-y_i}$ and $a_S(D)$ is the accuracy of the model trained on $S$ -and evaluated on $D$.In practical applications, the evaluation of this estimator -leverages both Monte Carlo sampling and permutation Monte Carlo sampling -[@castro_polynomial_2009]. - +$D_{-y_i}$ respectively. The algorithm can be applied by using the snippet ```python from pydvl.utils import Dataset, Utility @@ -115,7 +105,7 @@ from pydvl.value.shapley.classwise import compute_classwise_shapley_values, \ model = ... data = Dataset(...) -scoring = ClasswiseScorer("accuracy") +scoring = ("accuracy") utility = Utility(model, data, scoring) values = compute_classwise_shapley_values( utility, @@ -126,6 +116,10 @@ values = compute_classwise_shapley_values( ) ``` +where `ClasswiseScorer` is a special type of scorer only applicable for classification +problems. In practical applications, the evaluation of this estimator leverages both +Monte Carlo sampling and permutation Monte Carlo sampling [@castro_polynomial_2009]. + ### Owen sampling **Owen Sampling** [@okhrati_multilinear_2021] is a practical algorithm based on diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index cfc6226d3..408bb50e4 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -3,8 +3,8 @@ for classification problems. Let $D$ be the dataset, $D_{y_i}$ be the subset of $D$ with labels $y_i$, and $D_{-y_i}$ be the complement of $D_{y_i}$ in $D$. The key idea is that a sample $(x_i, y_i)$, might enhance the overall performance on $D$, while being -detrimental for the performance on $D_{y_i}$. To address this nuanced behavior, the -authors introduced the estimator +detrimental for the performance on $D_{y_i}$. The valuation formula of the paper is +given by $$ v_u(i) = \frac{1}{2^{|D_{-y_i}|}} \sum_{S_{-y_i}} \frac{1}{|D_{y_i}|!} @@ -14,19 +14,9 @@ where $S_{y_i} \subseteq D_{y_i} \setminus \{i\}$ and $S_{-y_i} \subseteq D_{-y_i}$. In other words, the summations are over the powerset of $D_{y_i} \setminus \{i\}$ and -$D_{-y_i}$ respectively. The estimator employs a specialized utility function - -$$ -u(S_{y_i}|S_{-y_i}) = a_S(D_{y_i}) \exp(a_S(D_{-y_i})), -$$ - -where $S=S_{y_i} \cup S_{-y_i}$ and $a_S(D)$ is the accuracy of the model trained on $S$ -and evaluated on $D$. - -## Monte Carlo sampling + Permutation Monte Carlo sampling - -In practical applications, the evaluation of this estimator leverages both Monte Carlo -sampling and permutation Monte Carlo sampling. This results in the estimator +$D_{-y_i}$ respectively. In practical applications, the implementation of this estimator +leverages both Monte Carlo sampling and permutation Monte Carlo sampling. Applying these +techniques results in the estimator $$ v_u(i) = \frac{1}{K} \sum_k \frac{1}{L} \sum_l @@ -34,22 +24,29 @@ $$ with $S^{(1)}, \dots, S^{(K)} \subseteq T_{-y_i}$ and - $\sigma^{(1)}, \dots, \sigma^{(L)} \in \Pi(T_{y_i}\setminus\{i\})$. +$\sigma^{(1)}, \dots, \sigma^{(L)} \in \Pi(T_{y_i}\setminus\{i\})$. Furthermore, the +estimator employs a specialized utility function + +$$ +u(S_{y_i}|S_{-y_i}) = a_S(D_{y_i}) \exp(a_S(D_{-y_i})), +$$ -## Derivation of test case +where $S=S_{y_i} \cup S_{-y_i}$ and $a_S(D)$ is the accuracy of the model trained on $S$ +and evaluated on $D$. -Let $D=\{(1,0),(2,0),(3,0),(4,1)\}$ be the test set and $T=\{(1,0),(2,0),(3,1),(4,1)\}$ -the train set. This specific dataset is chosen as it allows to solve the model +!!! info "Notes for derivation of test cases." + Let $D=\{(1,0),(2,0),(3,0),(4,1)\}$ be the test set and $T=\{(1,0),(2,0),(3,1),(4,1)\}$ + the train set. This specific dataset is chosen as it allows to solve the model -$$y = \max(0, \min(1, \text{round}(\beta^T x)))$$ + $$y = \max(0, \min(1, \text{round}(\beta^T x)))$$ -in closed form $\beta = \frac{\text{dot}(x, y)}{\text{dot}(x, x)}$. From the closed-form -solution, the tables for in-class accuracy $a_S(D_{y_i})$ and out-of-class accuracy -$a_S(D_{-y_i})$ can be calculated. By using these tables and setting -$\{S^{(1)}, \dots, S^{(K)}\} = 2^{T_{-y_i}}$ and -$\{\sigma^{(1)}, \dots, \sigma^{(L)}\} = \Pi(T_{y_i}\setminus\{i\})$, -the Monte Carlo estimator can be evaluated ($2^M$ is the powerset of $M$). -The details of the derivation are left to the eager reader. + in closed form $\beta = \frac{\text{dot}(x, y)}{\text{dot}(x, x)}$. From the closed-form + solution, the tables for in-class accuracy $a_S(D_{y_i})$ and out-of-class accuracy + $a_S(D_{-y_i})$ can be calculated. By using these tables and setting + $\{S^{(1)}, \dots, S^{(K)}\} = 2^{T_{-y_i}}$ and + $\{\sigma^{(1)}, \dots, \sigma^{(L)}\} = \Pi(T_{y_i}\setminus\{i\})$, + the Monte Carlo estimator can be evaluated ($2^M$ is the powerset of $M$). + The details of the derivation are left to the eager reader. # References @@ -115,6 +112,12 @@ class ClasswiseScorer(Scorer): refer to section four of (Schoch et al., 2022) 1. + !!! info "Surface plot for default values." + ![](img/classwise-shapley-discounted-utility-function.svg){:style="float:left"} + For $f(x)=x$ and $g(x)=e^x$ the surface plot looks as shown in the left plot + where the x-axis refers to in-class accuracy a_S(D_{y_i}) and the y-axis to + out-of-class accuracy $a_S(D_{-y_i})$ + Args: default: Score used when a model cannot be fit, e.g. when too little data is passed, or errors arise. From 6e2b0a41a380bde98583b0311716937b75c57d34 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Tue, 26 Sep 2023 00:08:50 +0200 Subject: [PATCH 27/43] Rework documentation. --- ...se-shapley-discounted-utility-function.svg | 89870 ++++++++-------- src/pydvl/value/shapley/classwise.py | 19 +- 2 files changed, 45157 insertions(+), 44732 deletions(-) diff --git a/docs/api/pydvl/value/shapley/classwise/img/classwise-shapley-discounted-utility-function.svg b/docs/api/pydvl/value/shapley/classwise/img/classwise-shapley-discounted-utility-function.svg index 8fc402f9d..c925f1e4a 100644 --- a/docs/api/pydvl/value/shapley/classwise/img/classwise-shapley-discounted-utility-function.svg +++ b/docs/api/pydvl/value/shapley/classwise/img/classwise-shapley-discounted-utility-function.svg @@ -1,12 +1,12 @@ - + - 2023-09-25T20:19:20.731288 + 2023-09-25T22:47:24.903053 image/svg+xml @@ -22,8 +22,8 @@ - - - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + - + - + - + - + - + - + - + - + - + - + - + - + - - + - + @@ -66920,12 +67250,12 @@ L -3.5 0 - + - + @@ -66935,12 +67265,12 @@ L -3.5 0 - + - + @@ -66950,12 +67280,12 @@ L -3.5 0 - + - + @@ -66965,12 +67295,12 @@ L -3.5 0 - + - + @@ -66980,12 +67310,12 @@ L -3.5 0 - + - + @@ -66994,7 +67324,7 @@ L -3.5 0 - + - + + + + + + + + + + + + + + + + + + + - - + - - + - - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +iVBORw0KGgoAAAANSUhEUgAAAC4AAAOcCAYAAAAxU4PUAAAFVElEQVR4nO3c263bQBAFQT42NIfg/EOxnIM+WjhAVQQNYTGzJH19/7n/fq5Bz68DviW8dq57s32z+hoOP/dz/7rhK7O/uPCacVibDTcOa8JrxmFNeO1c5nhrNvxct6OSEl4797PZvll9DYcbhzXhtXMZhy3hNXO8NhvuYbkmvHZuLz1bwmvmeG023O2wJrzmjNdmw70QqgmvGYc14TVnvDYb7qjUhNe8EKrNhhuHNeE1//C9Jrxm5ddmw638mvCaM16bDXdUasJrboc14TVzvDYbfj6OSkt47aymj2YPh9ucNeE1Z7wmvOaM12bDPQHVhNc8AdWE16z82my4o1ITXnOtrc2Gux3WhNes/JrwmpVfmw0/1+ZJ2f3Fhdes/Nps+PmMpo9mC+8ZhzXhtfPZPOK7v/hsuHFYE17zlF+bDffusCa85jtnTXjNHK/Nhrsd1oTXvBCqCa95k1WbDTcOa8JrrrW12XD/F1xNeM3KrwmvWfm12XDjsCa85oVQbTbcOKwJr7kd1oTXzPHabLjbYU14zTiszYa7HdaE1/x5e014zRyvzYa7HdaE1zzl12bDjcOa8JrbYU14zRyvzYYbhzXhNeOwJrxmjtdmw43DmvCacVibDff/ZNWE14zDmvCaa21tNtw4rAmvOeO12XCbsya8ZhzWhNfM8dpsuHFYE14zDmuz4cZhTXjNGa8Jr1n5tdlw47AmvHY+vy740uwvPhtuHNaE15zxmvCaJ6DabLhxWBNeMw5rwmvmeG023FGpCa9Z+bXZcOOwJrzmjNeE16z82my4cVgTXjMOa7PhxmFNeM0ZrwmvWfm12XDjsCa8Jrw2G25z1oTXrPya8JozXpsNt/JrwmvGYU14zRyvzYYbhzXhtXPdm//fx+wvPhtuc9aE16z8mvCaM16bDbfya8JrxmFtNtxRqQmveSFUE15zra0JrwmvzYa71tZmw23OmvCaJ6Ca8JqVX5sNd1RqwmvOeG023O2wJrxmHNaE17zJqs2GW/k14TUrvya8dm5nvDUbbuXXhNes/NpsuKNSE14712Xlp4TXzPHabPi5PQG1hNeMw9psuHFYE14zDmvCa74B1WbDfQOqCa+51tZmw23OmvCacVgTXvOUX5sNNw5rwmtndBru/uKz4cZhTXjNGa8Jr3mTVZsNP49x2BJes/JrwmvOeG023O2wJrzmWlubDbc5a8Jr5/Gnvy3hNdfa2my422FNeM21tjYbbhzWhNeMw5rwmjlemw33QqgmvGbl12bDbc6a8JozXhNec8Zrs+GOSk14zVN+TXjNHK/NhjsqNeE1Z7w2G+6o1ITXPAHVhNfM8dps+Hnuf79u+MrsLy68ZhzWZsPdDmvCa8ZhTXjNtbY2G35e47AlvOZaW5sNtzlrwmuegGrCa+e18luz4VZ+TXjNyq/NhnshVBNeO89l5aeE16z82mz4eY3DlvCacVgTXjuvN1mt2XCfxGvCa8ZhbTbcN6Ca8JpP4jXhNXO8NhtuHNaE14zD2my4b0A14TXfgGrCa+Z4bTbc+/Ga8JrvnLXZcOOwJrzmH77XhNf8AVNtNtzDck14zTeg2my4cVgTXnM7rAmvecqvzYZ76VkTXnOtrQmv+c5Zmw238mvCa/64ozYb7mG5Jrxm5deE16z82my4vwOqCa956VmbDbc5a8JrVn5NeM0cr82GOyo14bXz3L9O+M7sLz4bbhzWhNfOavlqt/Dcea381mz4ea/NszL7iwuvWfk14bXz3uZ4ajb8PFZ+S3jNtbY2G24c1oTX3A5rwmvnGW3frL6Gw638mvDaee/N9s3qazjc5qwJr/0HnPGSOmw0fgEAAAAASUVORK5CYII=" id="image3ea7e272ee" transform="scale(1 -1) translate(0 -221.76)" x="338.64" y="-34.56" width="11.04" height="221.76"/> - + - - + - + - + @@ -67474,14 +67904,14 @@ L 3.5 0 - + - + - + - + @@ -67489,14 +67919,14 @@ L 3.5 0 - + - + - + - + @@ -67504,14 +67934,14 @@ L 3.5 0 - + - + - + - + @@ -67519,14 +67949,14 @@ L 3.5 0 - + - + - + - + @@ -67534,14 +67964,14 @@ L 3.5 0 - + - + - + - + @@ -67550,22 +67980,22 @@ L 3.5 0 - - + - - + + diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 408bb50e4..1235febc5 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -24,15 +24,7 @@ $$ with $S^{(1)}, \dots, S^{(K)} \subseteq T_{-y_i}$ and -$\sigma^{(1)}, \dots, \sigma^{(L)} \in \Pi(T_{y_i}\setminus\{i\})$. Furthermore, the -estimator employs a specialized utility function - -$$ -u(S_{y_i}|S_{-y_i}) = a_S(D_{y_i}) \exp(a_S(D_{-y_i})), -$$ - -where $S=S_{y_i} \cup S_{-y_i}$ and $a_S(D)$ is the accuracy of the model trained on $S$ -and evaluated on $D$. +$\sigma^{(1)}, \dots, \sigma^{(L)} \in \Pi(T_{y_i}\setminus\{i\})$. !!! info "Notes for derivation of test cases." Let $D=\{(1,0),(2,0),(3,0),(4,1)\}$ be the test set and $T=\{(1,0),(2,0),(3,1),(4,1)\}$ @@ -113,10 +105,13 @@ class ClasswiseScorer(Scorer): 1. !!! info "Surface plot for default values." - ![](img/classwise-shapley-discounted-utility-function.svg){:style="float:left"} + ![Surface plot](img/classwise-shapley-discounted-utility-function.svg) + For $f(x)=x$ and $g(x)=e^x$ the surface plot looks as shown in the left plot - where the x-axis refers to in-class accuracy a_S(D_{y_i}) and the y-axis to - out-of-class accuracy $a_S(D_{-y_i})$ + where the x-axis refers to in-class accuracy $a_S(D_{y_i})$ and the y-axis to + out-of-class accuracy $a_S(D_{-y_i})$. The white lines represent the contour + lines annotated with the gradients. + Args: default: Score used when a model cannot be fit, e.g. when too little data is From cd01657f2e3ea1b772ee00c9249c142d02661475 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Fri, 6 Oct 2023 13:43:06 +0200 Subject: [PATCH 28/43] Finalize documentation. --- docs/value/classwise-shapley.md | 146 + ...se-shapley-discounted-utility-function.svg | 68001 ++++++++++++++++ .../classwise-shapley-example-densities.svg | 4714 ++ ...curacy-drop-logistic-regression-to-mlp.svg | 32051 ++++++++ mkdocs.yml | 1 + src/pydvl/value/shapley/classwise.py | 18 +- 6 files changed, 104921 insertions(+), 10 deletions(-) create mode 100644 docs/value/classwise-shapley.md create mode 100644 docs/value/img/classwise-shapley-discounted-utility-function.svg create mode 100644 docs/value/img/classwise-shapley-example-densities.svg create mode 100644 docs/value/img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg diff --git a/docs/value/classwise-shapley.md b/docs/value/classwise-shapley.md new file mode 100644 index 000000000..ac2bd4859 --- /dev/null +++ b/docs/value/classwise-shapley.md @@ -0,0 +1,146 @@ +--- +title: Classwise Shapley +--- + +# Classwise Shapley + +Classwise Shapley (CS) [@schoch_csshapley_2022] offers a distinct Shapley framework +tailored for classification problems. Let $D$ be the dataset, $D_{y_i}$ be the subset of +$D$ with labels $y_i$, and $D_{-y_i}$ be the complement of $D_{y_i}$ in $D$. The key +idea is that a sample $(x_i, y_i)$ might enhance the overall performance on $D$, while +being detrimental for the performance on $D_{y_i}$. To address this issue, the authors +introduced the estimator + +$$ +v_u(i) = \frac{1}{2^{|D_{-y_i}|}} \sum_{S_{-y_i}} +\left [ +\frac{1}{|D_{y_i}|}\sum_{S_{y_i}} \binom{|D_{y_i}|-1}{|S_{y_i}|}^{-1} +\delta(S_{y_i} | S_{-y_i}) +\right ], +$$ + +where $S_{y_i} \subseteq D_{y_i} \setminus \{i\}$ and $S_{-y_i} \subseteq D_{-y_i}$. In +other words, the summations are over the powerset of $D_{y_i} \setminus \{i\}$ and +$D_{-y_i}$ respectively. The function $\delta$ is called **set-conditional marginal +Shapley value** and is defined as + +$$ +\delta(S | C) = u( S \cup \{i\} | C ) − u(S | C), +$$ + +where $i \notin S, C$ and $S \bigcap C = \emptyset$. It makes sense to talk about +complexity at this point. In comparison to the original Shapley value, fewer samples are +used on average to fit the model in the score function. Although this speed-up depends +on the model, it should be at most linear in the number of data points. + +```python +from pydvl.utils import Dataset, Utility +from pydvl.value import HistoryDeviation, MaxChecks, MaxUpdates, RelativeTruncation +from pydvl.value.shapley.classwise import compute_classwise_shapley_values + +model = ... +data = Dataset(...) +scorer = ... +utility = Utility(model, data, scorer) +values = compute_classwise_shapley_values( + utility, + done=HistoryDeviation(n_steps=500, rtol=5e-2) | MaxUpdates(5000), + truncation=RelativeTruncation(utility, rtol=0.01), + done_sample_complements=MaxChecks(1), + normalize_values=True +) +``` + +where `ClasswiseScorer` is a special type of scorer, only applicable for classification +problems. In practical applications, the evaluation of this estimator leverages both +Monte Carlo sampling and permutation Monte Carlo sampling [@castro_polynomial_2009]. + +### Class-agnostic score + +Any estimator of the form $s(D)=\frac{1}{|D|}\sum_{i \in D} v_i$ (e.g. accuracy) can be +wrapped into a class-agnostic estimator using the provided `ClasswiseScorer` class. Each +valuation point $i$ belongs to a label set $y_i \in \mathbb{N}$. Hence, for each $i$ +one has two disjoint $D_{y_i}$ and $D_{-y_i}$ datasets. The class-agnostic estimator is +then defined as + +$$ +u(S) = f(a_S(D_{y_i}))) g(a_S(D_{-y_i}))), +$$ + +where $f$ and $g$ are monotonically increasing functions. The authors showed that it +holds that $f(x)=x$ and $g(x)=e^x$ have favorable properties. For further research, +e.g. in the direction of the f1-score, we leave the option to set different $f$ and $g$ +functions + +```python +import numpy as np +from pydvl.value.shapley.classwise import ClasswiseScorer + +identity = lambda x: x +scorer = ClasswiseScorer( + "accuracy", + in_class_discount_fn=identity, + out_of_class_discount_fn=np.exp +) +``` + +For $f(x)=x$ and $g(x)=e^x$, the surface plot is depicted below. The x-axis +represents in-class accuracy $a_S(D_{y_i})$, and the y-axis corresponds to out-of-class +accuracy $a_S(D_{-y_i})$. The white lines illustrate the contour lines, annotated with +their respective gradients. + +![Surface plot](img/classwise-shapley-discounted-utility-function.svg) + +## Evaluation + +Assessing the quality of an algorithm is crucial. Therefore, this section provides a +brief evaluation of the method using nine different datasets to evaluate the performance +of CS. The following table + +| Dataset | Data Type | Classes | Input Dims | OpenML ID | +|----------------|-----------|---------|------------|-----------| +| Diabetes | Tabular | 2 | 8 | 37 | +| Click | Tabular | 2 | 11 | 1216 | +| CPU | Tabular | 2 | 21 | 197 | +| Covertype | Tabular | 7 | 54 | 1596 | +| Phoneme | Tabular | 2 | 5 | 1489 | +| FMNIST | Image | 2 | 32 | 40996 | +| CIFAR10 | Image | 2 | 32 | 40927 | +| MNIST (binary) | Image | 2 | 32 | 554 | +| MNIST (multi) | Image | 10 | 32 | 554 | + +lists all used datasets with their number of features, classes and OpenML ID. The table +is taken from [@schoch_csshapley_2022] and the same pre-processing steps are applied. +For images Principal Component Analysis (PCA) is used to obtain 32 features after +`resnet18` extracts relevant features of the images. For more details on the +pre-processing steps, please refer to the paper. + +### Accuracy compared to other state-of-the-art methods + +Classwise Shapley (CS) is compared to other state-of-the-art valuation methods. These +include Truncated Monte Carlo Shapley (TMC), Beta Shapley (Beta) as well as +Leave-one-out (LOO). Random values serve as a baseline. Logistic regression is used for +both the valuation and a neural network for the measurement of accuracy. The following +plot shows + +![Accuracy](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg) + +the test set accuracy on the y-axis, and the number of samples removed on the x-axis. +Samples are removed high to low valuation order. A lower curve means that the first +removed values have a higher relevancy for the model. CS is competitive to the compared +methods. Especially in very unbalanced datasets, like `Click`, the performance of CS +seems superior. In other datasets, like `Covertype` and `Diabetes` and `MNIST (multi)` +the performance is on par with TMC. For `MNIST (binary)` and `Phoneme` the performance +is competitive. Is it noteworthy that for all valuation methods the same number of +marginal evaluations was used. + +### Value density + +This section takes a look at the density values of CS. The distribution is not uniform +or Gaussian, but is slightly biased to the right and skewed to the left. This is evident +in the following plot + +![Density](img/classwise-shapley-example-densities.svg) + +where the x-axis represents values and the y-axis reveals the count. It shows the +histogram as well an approximation to the density using KDE. \ No newline at end of file diff --git a/docs/value/img/classwise-shapley-discounted-utility-function.svg b/docs/value/img/classwise-shapley-discounted-utility-function.svg new file mode 100644 index 000000000..c925f1e4a --- /dev/null +++ b/docs/value/img/classwise-shapley-discounted-utility-function.svg @@ -0,0 +1,68001 @@ + + + + + + + + 2023-09-25T22:47:24.903053 + image/svg+xml + + + Matplotlib v3.7.0, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/value/img/classwise-shapley-example-densities.svg b/docs/value/img/classwise-shapley-example-densities.svg new file mode 100644 index 000000000..3b95747a0 --- /dev/null +++ b/docs/value/img/classwise-shapley-example-densities.svg @@ -0,0 +1,4714 @@ + + + + + + + + 2023-10-05T03:20:15.147632 + image/svg+xml + + + Matplotlib v3.7.0, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/value/img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg b/docs/value/img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg new file mode 100644 index 000000000..54570bbf0 --- /dev/null +++ b/docs/value/img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg @@ -0,0 +1,32051 @@ + + + + + + + + 2023-10-05T03:20:36.991079 + image/svg+xml + + + Matplotlib v3.7.0, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mkdocs.yml b/mkdocs.yml index 7ba274e51..1c4cbcfce 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -194,6 +194,7 @@ nav: - Shapley Values: value/shapley.md - Semi-values: value/semi-values.md - The core: value/the-core.md + - Classwise Shapley: value/classwise-shapley.md - Examples: - Shapley values: examples/shapley_basic_spotify.ipynb - KNN Shapley: examples/shapley_knn_flowers.ipynb diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 1235febc5..6bf8dab0a 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -26,7 +26,7 @@ with $S^{(1)}, \dots, S^{(K)} \subseteq T_{-y_i}$ and $\sigma^{(1)}, \dots, \sigma^{(L)} \in \Pi(T_{y_i}\setminus\{i\})$. -!!! info "Notes for derivation of test cases." +??? info "Notes for derivation of test cases." Let $D=\{(1,0),(2,0),(3,0),(4,1)\}$ be the test set and $T=\{(1,0),(2,0),(3,1),(4,1)\}$ the train set. This specific dataset is chosen as it allows to solve the model @@ -104,16 +104,14 @@ class ClasswiseScorer(Scorer): refer to section four of (Schoch et al., 2022) 1. - !!! info "Surface plot for default values." - ![Surface plot](img/classwise-shapley-discounted-utility-function.svg) - - For $f(x)=x$ and $g(x)=e^x$ the surface plot looks as shown in the left plot - where the x-axis refers to in-class accuracy $a_S(D_{y_i})$ and the y-axis to - out-of-class accuracy $a_S(D_{-y_i})$. The white lines represent the contour - lines annotated with the gradients. - + !!! warning Multi-class support + Metrics must support multiple class labels if you intend to apply them to a + multi-class problem. For instance, the metric 'accuracy' supports multiple + classes, but the metric 'f1' does not. For a two-class classification problem, + using 'f1_weighted' is essentially equivalent to using 'accuracy'. Args: + scoring: Name of the scoring function. See [Scorer][pydvl.utils.scorer.Scorer]. default: Score used when a model cannot be fit, e.g. when too little data is passed, or errors arise. range: Numerical range of the score function. Some Monte Carlo methods can @@ -138,7 +136,7 @@ def __init__( self, scoring: str = "accuracy", default: float = 0.0, - range: Tuple[float, float] = (0, np.inf), + range: Tuple[float, float] = (0, 1), in_class_discount_fn: Callable[[float], float] = lambda x: x, out_of_class_discount_fn: Callable[[float], float] = np.exp, initial_label: Optional[int] = None, From 67416a30f40035fcef0ed5dd087c3f0397de9e4e Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Sat, 7 Oct 2023 22:13:57 +0200 Subject: [PATCH 29/43] Improvements to CWS docs --- docs/value/classwise-shapley.md | 196 +++++++++++++-------------- docs_includes/abbreviations.md | 2 +- src/pydvl/value/shapley/classwise.py | 101 +++++++------- 3 files changed, 146 insertions(+), 153 deletions(-) diff --git a/docs/value/classwise-shapley.md b/docs/value/classwise-shapley.md index ac2bd4859..d2662e080 100644 --- a/docs/value/classwise-shapley.md +++ b/docs/value/classwise-shapley.md @@ -1,15 +1,15 @@ --- -title: Classwise Shapley +title: Class-wise Shapley --- -# Classwise Shapley +# Class-wise Shapley -Classwise Shapley (CS) [@schoch_csshapley_2022] offers a distinct Shapley framework -tailored for classification problems. Let $D$ be the dataset, $D_{y_i}$ be the subset of -$D$ with labels $y_i$, and $D_{-y_i}$ be the complement of $D_{y_i}$ in $D$. The key -idea is that a sample $(x_i, y_i)$ might enhance the overall performance on $D$, while -being detrimental for the performance on $D_{y_i}$. To address this issue, the authors -introduced the estimator +Class-wise Shapley (CWS) [@schoch_csshapley_2022] offers a Shapley framework +tailored for classification problems. Let $D$ be a dataset, $D_{y_i}$ be the +subset of $D$ with labels $y_i$, and $D_{-y_i}$ be the complement of $D_{y_i}$ +in $D$. The key idea is that a sample $(x_i, y_i)$ might enhance the overall +performance on $D$, while being detrimental for the performance on $D_{y_i}$. To +address this issue, the authors introduce the estimator $$ v_u(i) = \frac{1}{2^{|D_{-y_i}|}} \sum_{S_{-y_i}} @@ -19,83 +19,85 @@ v_u(i) = \frac{1}{2^{|D_{-y_i}|}} \sum_{S_{-y_i}} \right ], $$ -where $S_{y_i} \subseteq D_{y_i} \setminus \{i\}$ and $S_{-y_i} \subseteq D_{-y_i}$. In -other words, the summations are over the powerset of $D_{y_i} \setminus \{i\}$ and -$D_{-y_i}$ respectively. The function $\delta$ is called **set-conditional marginal -Shapley value** and is defined as +where $S_{y_i} \subseteq D_{y_i} \setminus \{i\}$ and $S_{-y_i} \subseteq +D_{-y_i}$, and the function $\delta$ is called **set-conditional marginal +Shapley value**. It is defined as $$ \delta(S | C) = u( S \cup \{i\} | C ) − u(S | C), $$ -where $i \notin S, C$ and $S \bigcap C = \emptyset$. It makes sense to talk about -complexity at this point. In comparison to the original Shapley value, fewer samples are -used on average to fit the model in the score function. Although this speed-up depends -on the model, it should be at most linear in the number of data points. - -```python -from pydvl.utils import Dataset, Utility -from pydvl.value import HistoryDeviation, MaxChecks, MaxUpdates, RelativeTruncation -from pydvl.value.shapley.classwise import compute_classwise_shapley_values - -model = ... -data = Dataset(...) -scorer = ... -utility = Utility(model, data, scorer) -values = compute_classwise_shapley_values( - utility, - done=HistoryDeviation(n_steps=500, rtol=5e-2) | MaxUpdates(5000), - truncation=RelativeTruncation(utility, rtol=0.01), - done_sample_complements=MaxChecks(1), - normalize_values=True -) -``` - -where `ClasswiseScorer` is a special type of scorer, only applicable for classification -problems. In practical applications, the evaluation of this estimator leverages both -Monte Carlo sampling and permutation Monte Carlo sampling [@castro_polynomial_2009]. - -### Class-agnostic score - -Any estimator of the form $s(D)=\frac{1}{|D|}\sum_{i \in D} v_i$ (e.g. accuracy) can be -wrapped into a class-agnostic estimator using the provided `ClasswiseScorer` class. Each -valuation point $i$ belongs to a label set $y_i \in \mathbb{N}$. Hence, for each $i$ -one has two disjoint $D_{y_i}$ and $D_{-y_i}$ datasets. The class-agnostic estimator is -then defined as +where $i \notin S, C$ and $S \bigcap C = \emptyset$. + +In practical applications, estimating this quantity is done both with Monte +Carlo sampling of the powerset, and the set of index permutations +[@castro_polynomial_2009]. Typically, this requires fewer samples than the +original Shapley value, although the actual speed-up depends on the model and +the dataset. -$$ -u(S) = f(a_S(D_{y_i}))) g(a_S(D_{-y_i}))), -$$ -where $f$ and $g$ are monotonically increasing functions. The authors showed that it -holds that $f(x)=x$ and $g(x)=e^x$ have favorable properties. For further research, -e.g. in the direction of the f1-score, we leave the option to set different $f$ and $g$ -functions +??? Example "Computing classwise Shapley values" + ```python + from pydvl.value import * + + model = ... + data = Dataset(...) + scorer = ClasswiseScorer(...) + utility = Utility(model, data, scorer) + values = compute_classwise_shapley_values( + utility, + done=HistoryDeviation(n_steps=500, rtol=5e-2) | MaxUpdates(5000), + truncation=RelativeTruncation(utility, rtol=0.01), + done_sample_complements=MaxChecks(1), + normalize_values=True + ) + ``` -```python -import numpy as np -from pydvl.value.shapley.classwise import ClasswiseScorer -identity = lambda x: x -scorer = ClasswiseScorer( - "accuracy", - in_class_discount_fn=identity, - out_of_class_discount_fn=np.exp -) -``` +## Class-wise scorer -For $f(x)=x$ and $g(x)=e^x$, the surface plot is depicted below. The x-axis -represents in-class accuracy $a_S(D_{y_i})$, and the y-axis corresponds to out-of-class -accuracy $a_S(D_{-y_i})$. The white lines illustrate the contour lines, annotated with -their respective gradients. +In order to use the classwise Shapley value, one needs to define a +[ClasswiseScorer][pydvl.value.shapley.classwise.ClasswiseScorer]. Given a sample +$x_i$ with label $y_i \in \mathbb{N}$, we define two disjoint sets $D_{y_i}$ and +$D_{-y_i}$ and define + +$$ +u(S) = f(a_S(D_{y_i}))) g(a_S(D_{-y_i}))), +$$ -![Surface plot](img/classwise-shapley-discounted-utility-function.svg) +where $f$ and $g$ are monotonically increasing functions, $a_S(D_{y_i})$ is the +**in-class accuracy**, and $a_S(D_{-y_i})$ is the **out-of-class accuracy** (the +names originate from a choice by the authors to use accuracy, but in principle +any other score, like $F_1$ can be used). + +The authors show that $f(x)=x$ and $g(x)=e^x$ have favorable properties and are +therefore the defaults, but we leave the option to set different functions $f$ +and $g$ for an exploration with different base scores. + +??? Example "The default class-wise scorer" + ```python + import numpy as np + from pydvl.value.shapley.classwise import ClasswiseScorer + + # These are the defaults + identity = lambda x: x + scorer = ClasswiseScorer( + "accuracy", + in_class_discount_fn=identity, + out_of_class_discount_fn=np.exp + ) + ``` + +The level curves for $f(x)=x$ and $g(x)=e^x$ are depicted below. The white lines +illustrate the contour lines, annotated with their respective gradients. + +![Level curves of the class-wise utility](img/classwise-shapley-discounted-utility-function.svg) ## Evaluation -Assessing the quality of an algorithm is crucial. Therefore, this section provides a -brief evaluation of the method using nine different datasets to evaluate the performance -of CS. The following table +Here is brief evaluation of the method using nine different datasets. The +following table lists all used datasets with their number of features, classes +and OpenML ID: | Dataset | Data Type | Classes | Input Dims | OpenML ID | |----------------|-----------|---------|------------|-----------| @@ -109,38 +111,36 @@ of CS. The following table | MNIST (binary) | Image | 2 | 32 | 554 | | MNIST (multi) | Image | 10 | 32 | 554 | -lists all used datasets with their number of features, classes and OpenML ID. The table -is taken from [@schoch_csshapley_2022] and the same pre-processing steps are applied. -For images Principal Component Analysis (PCA) is used to obtain 32 features after -`resnet18` extracts relevant features of the images. For more details on the -pre-processing steps, please refer to the paper. +The table is taken from [@schoch_csshapley_2022] and the same pre-processing +steps are applied. For images, Principal Component Analysis (PCA) is used to +obtain 32 features after `resnet18` extracts relevant features of the images. +For more details on the pre-processing steps, please refer to the paper. -### Accuracy compared to other state-of-the-art methods +### Comparison to the state-of-the-art -Classwise Shapley (CS) is compared to other state-of-the-art valuation methods. These -include Truncated Monte Carlo Shapley (TMC), Beta Shapley (Beta) as well as -Leave-one-out (LOO). Random values serve as a baseline. Logistic regression is used for -both the valuation and a neural network for the measurement of accuracy. The following -plot shows +We compare CWS to TMCS, Beta Shapley (Beta) and LOO on a sample removal task +with value transfer to another model. Random values serve as a baseline. +Logistic regression is used for the computation of values, and they are used to +prune the training set for a neural network. The following plot shows +valuation-set accuracy of the network on the y-axis, and the number of samples +removed on the x-axis. -![Accuracy](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg) +![Accuracy of transferred values from logistic regression to an MLP](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg) -the test set accuracy on the y-axis, and the number of samples removed on the x-axis. -Samples are removed high to low valuation order. A lower curve means that the first -removed values have a higher relevancy for the model. CS is competitive to the compared -methods. Especially in very unbalanced datasets, like `Click`, the performance of CS -seems superior. In other datasets, like `Covertype` and `Diabetes` and `MNIST (multi)` -the performance is on par with TMC. For `MNIST (binary)` and `Phoneme` the performance -is competitive. Is it noteworthy that for all valuation methods the same number of -marginal evaluations was used. +Samples are removed from high to low valuation order and hence we expect a steep +decrease in the curve. CWS is competitive with the compared methods. Especially +in very unbalanced datasets, like `Click`, the performance of CWS seems +superior. In other datasets, like `Covertype` and `Diabetes` and `MNIST (multi)` +the performance is on par with TMC. For `MNIST (binary)` and `Phoneme` the +performance is competitive. We remark that for all valuation methods the +same number of _evaluations of the marginal utility_ was used. -### Value density +### Value distribution -This section takes a look at the density values of CS. The distribution is not uniform -or Gaussian, but is slightly biased to the right and skewed to the left. This is evident -in the following plot +This section takes a look at the distribution of CWS values for the above +datasets. In the histogram plots one can see that it is slightly biased and +often skewed. -![Density](img/classwise-shapley-example-densities.svg) +![Distribution of values computed using logistic regression](img/classwise-shapley-example-densities.svg) -where the x-axis represents values and the y-axis reveals the count. It shows the -histogram as well an approximation to the density using KDE. \ No newline at end of file +The curves are an approximation of the density using KDE. diff --git a/docs_includes/abbreviations.md b/docs_includes/abbreviations.md index 6cc7cb258..622fa5a5d 100644 --- a/docs_includes/abbreviations.md +++ b/docs_includes/abbreviations.md @@ -1,5 +1,5 @@ *[CSP]: Constraint Satisfaction Problem -*[CS]: Class-wise Shapley +*[CWS]: Class-wise Shapley *[GT]: Group Testing *[LC]: Least Core *[LOO]: Leave-One-Out diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 6bf8dab0a..7bee8afaa 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -1,10 +1,10 @@ r""" -Class-wise Shapley (Schoch et al., 2022)[^1] offers a distinct Shapley framework tailored -for classification problems. Let $D$ be the dataset, $D_{y_i}$ be the subset of $D$ with -labels $y_i$, and $D_{-y_i}$ be the complement of $D_{y_i}$ in $D$. The key idea is that -a sample $(x_i, y_i)$, might enhance the overall performance on $D$, while being -detrimental for the performance on $D_{y_i}$. The valuation formula of the paper is -given by +Class-wise Shapley (Schoch et al., 2022)[^1] offers a Shapley framework tailored +for classification problems. Let $D$ be a dataset, $D_{y_i}$ be the subset of +$D$ with labels $y_i$, and $D_{-y_i}$ be the complement of $D_{y_i}$ in $D$. The +key idea is that a sample $(x_i, y_i)$, might enhance the overall performance on +$D$, while being detrimental for the performance on $D_{y_i}$. The Class-wise +value is defined as: $$ v_u(i) = \frac{1}{2^{|D_{-y_i}|}} \sum_{S_{-y_i}} \frac{1}{|D_{y_i}|!} @@ -12,21 +12,24 @@ [u( S_{y_i} \cup \{i\} | S_{-y_i} ) − u( S_{y_i} | S_{-y_i})], $$ -where $S_{y_i} \subseteq D_{y_i} \setminus \{i\}$ and $S_{-y_i} \subseteq D_{-y_i}$. In -other words, the summations are over the powerset of $D_{y_i} \setminus \{i\}$ and -$D_{-y_i}$ respectively. In practical applications, the implementation of this estimator -leverages both Monte Carlo sampling and permutation Monte Carlo sampling. Applying these -techniques results in the estimator +where $S_{y_i} \subseteq D_{y_i} \setminus \{i\}$ and $S_{-y_i} \subseteq +D_{-y_i}$. In practice, this quantity is estimated using Monte Carlo sampling of +the powerset and the set of index permutations. Applying these techniques +results in the estimator $$ v_u(i) = \frac{1}{K} \sum_k \frac{1}{L} \sum_l [u(\sigma^{(l)}_{:i} \cup \{i\} | S^{(k)} ) − u( \sigma^{(l)}_{:i} | S^{(k)})], $$ -with $S^{(1)}, \dots, S^{(K)} \subseteq T_{-y_i}$ and -$\sigma^{(1)}, \dots, \sigma^{(L)} \in \Pi(T_{y_i}\setminus\{i\})$. +with $S^{(1)}, \dots, S^{(K)} \subseteq T_{-y_i},$ $\sigma^{(1)}, \dots, +\sigma^{(L)} \in \Pi(T_{y_i}\setminus\{i\}),$ and $\sigma^{(l)}_{:i}$ denoting +the set of indices in permutation $\sigma^{(l)}$ before the position where $i$ +appears. The sets $T_{y_i}$ and $T_{-y_i}$ are the training sets for the labels +$y_i$ and $-y_i$, respectively. -??? info "Notes for derivation of test cases." +??? info "Notes for derivation of test cases" + The unit tests include the following manually constructed data: Let $D=\{(1,0),(2,0),(3,0),(4,1)\}$ be the test set and $T=\{(1,0),(2,0),(3,1),(4,1)\}$ the train set. This specific dataset is chosen as it allows to solve the model @@ -85,30 +88,29 @@ class ClasswiseScorer(Scorer): - r"""A Scorer designed for evaluation in classification problems. Its value is - derived from both in-class and out-of-class scores (Schoch et al., 2022) - 1. Let $S$ represent the training - set and $D$ be the test set. For each label $c$, the test set $D$ is factorized into - two disjoint sets: $D_c$ for in-class instances and $D_{-c}$ for out-of-class - instances. The score function itself than estimates the in-class metric, adjusted by - the discounted out-of-class metric. In essence, the score function for each element - of label $c$ is conditioned on the out-of-class instances (or a subset thereof). - Both the in-class and out-of-class metrics are determined by an inner score function - $a_S$ trained on the train set $S$. The expression for the outer score function is - - $${ - u(S_{y_i}) = f(a_S(D_{y_i}))) g(a_S(D_{-y_i}))), - }$$ - - where $f$ and $g$ are continuous, monotonic functions. For a detailed explanation, - refer to section four of (Schoch et al., 2022) - 1. + r"""A Scorer designed for evaluation in classification problems. Its value + is computed from an in-class and an out-of-class score (Schoch et al., 2022) + 1. Let $S$ represent the + training set and $D$ be the valuation set. For each label $c$, $D$ is + factorized into two disjoint sets: $D_c$ for in-class instances and $D_{-c}$ + for out-of-class instances. The score combines an in-class metric of + performance, adjusted by a discounted out-of-class metric. These "inner" + metrics must be provided or default to accuracy. They are combined into: + + $$ + u(S_{y_i}) = f(a_S(D_{y_i}))\ g(a_S(D_{-y_i})), + $$ + + where $f$ and $g$ are continuous, monotonic functions. For a detailed + explanation, refer to section four of (Schoch et al., 2022) 1. !!! warning Multi-class support - Metrics must support multiple class labels if you intend to apply them to a - multi-class problem. For instance, the metric 'accuracy' supports multiple - classes, but the metric 'f1' does not. For a two-class classification problem, - using 'f1_weighted' is essentially equivalent to using 'accuracy'. + Metrics must support multiple class labels if you intend to apply them + to a multi-class problem. For instance, the metric 'accuracy' supports + multiple classes, but the metric 'f1' does not. For a two-class + classification problem, using 'f1_weighted' is essentially equivalent to + using 'accuracy'. Args: scoring: Name of the scoring function. See [Scorer][pydvl.utils.scorer.Scorer]. @@ -180,12 +182,12 @@ def estimate_in_class_and_out_of_class_score( rescale_scores: bool = True, ) -> Tuple[float, float]: r""" - Computes in-class and out-of-class scores using the provided scoring function - $s$. The result can be expressed as + Computes in-class and out-of-class scores using the provided inner + scoring function. The result is - $${ + $$ a_S(D=\{(x_1, y_1), \dots, (x_K, y_K)\}) = \frac{1}{N} \sum_k s(y(x_k), y_k). - }$$ + $$ In this context, for label $c$ calculations are executed twice: once for $D_c$ and once for $D_{-c}$ to determine the in-class and out-of-class scores, @@ -312,9 +314,7 @@ def compute_classwise_shapley_values( pbar = tqdm(disable=not progress, position=0, total=100, unit="%") algorithm = "classwise_shapley" accumulated_result = ValuationResult.zeros( - algorithm=algorithm, - indices=u.data.indices, - data_names=u.data.data_names, + algorithm=algorithm, indices=u.data.indices, data_names=u.data.data_names ) terminate_exec = False seed_sequence = ensure_seed_sequence(seed) @@ -396,9 +396,7 @@ def _permutation_montecarlo_classwise_shapley_one_step( done_sample_complements = MaxChecks(1) result = ValuationResult.zeros( - algorithm=algorithm_name, - indices=u.data.indices, - data_names=u.data.data_names, + algorithm=algorithm_name, indices=u.data.indices, data_names=u.data.data_names ) rng = np.random.default_rng(seed) x_train, y_train = u.data.get_training_data(u.data.indices) @@ -409,9 +407,7 @@ def _permutation_montecarlo_classwise_shapley_one_step( for label in unique_labels: u.scorer.label = label class_indices_set, class_complement_indices_set = _split_indices_by_label( - u.data.indices, - y_train, - label, + u.data.indices, y_train, label ) _, complement_y_train = u.data.get_training_data(class_complement_indices_set) indices_permutation = rng.permutation(class_indices_set) @@ -440,8 +436,7 @@ def _permutation_montecarlo_classwise_shapley_one_step( def _normalize_classwise_shapley_values( - result: ValuationResult, - u: Utility, + result: ValuationResult, u: Utility ) -> ValuationResult: r""" Normalize a valuation result specific to classwise Shapley. @@ -525,9 +520,7 @@ def _permutation_montecarlo_shapley_rollout( ) result = ValuationResult.zeros( - algorithm=algorithm_name, - indices=u.data.indices, - data_names=u.data.data_names, + algorithm=algorithm_name, indices=u.data.indices, data_names=u.data.data_names ) prev_score = ( From 74ba6b2d9dbf7c10998ed19a27377f72204ee1e8 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Sat, 7 Oct 2023 22:39:11 +0200 Subject: [PATCH 30/43] Actually allow for other inner metrics in ClasswiseScorer --- src/pydvl/utils/score.py | 8 +++++- src/pydvl/value/shapley/classwise.py | 39 ++++++++++++++-------------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/pydvl/utils/score.py b/src/pydvl/utils/score.py index a5d1aceef..f077c9a53 100644 --- a/src/pydvl/utils/score.py +++ b/src/pydvl/utils/score.py @@ -26,7 +26,13 @@ from pydvl.utils.types import SupervisedModel -__all__ = ["Scorer", "compose_score", "squashed_r2", "squashed_variance"] +__all__ = [ + "Scorer", + "ScorerCallable", + "compose_score", + "squashed_r2", + "squashed_variance", +] class ScorerCallable(Protocol): diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index 7bee8afaa..d03257120 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -55,8 +55,8 @@ import logging import numbers from concurrent.futures import FIRST_COMPLETED, Future, wait -from copy import copy, deepcopy -from typing import Callable, Optional, Set, Tuple, cast +from copy import copy +from typing import Callable, Optional, Set, Tuple, Union, cast import numpy as np from numpy.random import SeedSequence @@ -72,6 +72,7 @@ from pydvl.utils import ( Dataset, Scorer, + ScorerCallable, Seed, SupervisedModel, Utility, @@ -89,13 +90,14 @@ class ClasswiseScorer(Scorer): r"""A Scorer designed for evaluation in classification problems. Its value - is computed from an in-class and an out-of-class score (Schoch et al., 2022) - 1. Let $S$ represent the + is computed from an in-class and an out-of-class "inner score" (Schoch et + al., 2022) 1. Let $S$ be the training set and $D$ be the valuation set. For each label $c$, $D$ is factorized into two disjoint sets: $D_c$ for in-class instances and $D_{-c}$ for out-of-class instances. The score combines an in-class metric of - performance, adjusted by a discounted out-of-class metric. These "inner" - metrics must be provided or default to accuracy. They are combined into: + performance, adjusted by a discounted out-of-class metric. These inner + scores must be provided upon construction or default to accuracy. They are + combined into: $$ u(S_{y_i}) = f(a_S(D_{y_i}))\ g(a_S(D_{-y_i})), @@ -113,9 +115,10 @@ class ClasswiseScorer(Scorer): using 'accuracy'. Args: - scoring: Name of the scoring function. See [Scorer][pydvl.utils.scorer.Scorer]. - default: Score used when a model cannot be fit, e.g. when too little data is - passed, or errors arise. + scoring: Name of the scoring function or a callable that can be passed + to [Scorer][pydvl.utils.score.Scorer]. + default: Score to use when a model fails to provide a number, e.g. when + too little was used to train it, or errors arise. range: Numerical range of the score function. Some Monte Carlo methods can use this to estimate the number of samples required for a certain quality of approximation. If not provided, it can be read from the `scoring` object @@ -125,10 +128,8 @@ class ClasswiseScorer(Scorer): discount the in-class score. out_of_class_discount_fn: Continuous, monotonic increasing function used to discount the out-of-class score. - initial_label: Set initial label (Doesn't require to set parameter `label` - on [ClasswiseScorer][pydvl.value.shapley.classwise.ClasswiseScorer] in first - iteration) - name: Name of the scorer. If not provided, the name of the passed + initial_label: Set initial label (for the first iteration) + name: Name of the scorer. If not provided, the name of the inner scoring function will be prefixed by 'classwise '. !!! tip "New in version 0.7.1" @@ -136,7 +137,7 @@ class ClasswiseScorer(Scorer): def __init__( self, - scoring: str = "accuracy", + scoring: Union[str, ScorerCallable] = "accuracy", default: float = 0.0, range: Tuple[float, float] = (0, 1), in_class_discount_fn: Callable[[float], float] = lambda x: x, @@ -148,10 +149,10 @@ def __init__( disc_score_out_of_class = out_of_class_discount_fn(range[1]) transformed_range = (0, disc_score_in_class * disc_score_out_of_class) super().__init__( - "accuracy", + scoring=scoring, range=transformed_range, default=default, - name=name or f"classwise {scoring}", + name=name or f"classwise {str(scoring)}", ) self._in_class_discount_fn = in_class_discount_fn self._out_of_class_discount_fn = out_of_class_discount_fn @@ -247,9 +248,9 @@ def compute_classwise_shapley_values( ) -> ValuationResult: r""" Computes an approximate Class-wise Shapley value by sampling independent - permutations of the index set for each label and unifying it with index sets sampled - from the complement (with respect to the currently evaluated label), approximating - the sum: + permutations of the index set for each label and index sets sampled from the + powerset of the complement (with respect to the currently evaluated label), + approximating the sum: $$ v_u(i) = \frac{1}{K} \sum_k \frac{1}{L} \sum_l From 6d21579b2612fe7b58de297845afd8955032eeb4 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Sun, 8 Oct 2023 10:51:37 +0200 Subject: [PATCH 31/43] [skip ci] More doc changes for CWS --- docs/value/classwise-shapley.md | 55 ++++++++++++++++----------------- docs_includes/abbreviations.md | 9 +++--- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/docs/value/classwise-shapley.md b/docs/value/classwise-shapley.md index d2662e080..a86b71597 100644 --- a/docs/value/classwise-shapley.md +++ b/docs/value/classwise-shapley.md @@ -95,35 +95,32 @@ illustrate the contour lines, annotated with their respective gradients. ## Evaluation -Here is brief evaluation of the method using nine different datasets. The -following table lists all used datasets with their number of features, classes -and OpenML ID: - -| Dataset | Data Type | Classes | Input Dims | OpenML ID | -|----------------|-----------|---------|------------|-----------| -| Diabetes | Tabular | 2 | 8 | 37 | -| Click | Tabular | 2 | 11 | 1216 | -| CPU | Tabular | 2 | 21 | 197 | -| Covertype | Tabular | 7 | 54 | 1596 | -| Phoneme | Tabular | 2 | 5 | 1489 | -| FMNIST | Image | 2 | 32 | 40996 | -| CIFAR10 | Image | 2 | 32 | 40927 | -| MNIST (binary) | Image | 2 | 32 | 554 | -| MNIST (multi) | Image | 10 | 32 | 554 | - -The table is taken from [@schoch_csshapley_2022] and the same pre-processing -steps are applied. For images, Principal Component Analysis (PCA) is used to -obtain 32 features after `resnet18` extracts relevant features of the images. -For more details on the pre-processing steps, please refer to the paper. - -### Comparison to the state-of-the-art - -We compare CWS to TMCS, Beta Shapley (Beta) and LOO on a sample removal task -with value transfer to another model. Random values serve as a baseline. -Logistic regression is used for the computation of values, and they are used to -prune the training set for a neural network. The following plot shows -valuation-set accuracy of the network on the y-axis, and the number of samples -removed on the x-axis. +We evaluate the method on the nine datasets used in [@schoch_csshapley_2022], +using the same pre-processing. For images, PCA is used to reduce down to 32 the +number of features found by a `Resnet18` model. For more details on the +pre-processing steps, please refer to the paper. + +??? info "Datasets used for evaluation" + | Dataset | Data Type | Classes | Input Dims | OpenML ID | + |----------------|-----------|---------|------------|-----------| + | Diabetes | Tabular | 2 | 8 | 37 | + | Click | Tabular | 2 | 11 | 1216 | + | CPU | Tabular | 2 | 21 | 197 | + | Covertype | Tabular | 7 | 54 | 1596 | + | Phoneme | Tabular | 2 | 5 | 1489 | + | FMNIST | Image | 2 | 32 | 40996 | + | CIFAR10 | Image | 2 | 32 | 40927 | + | MNIST (binary) | Image | 2 | 32 | 554 | + | MNIST (multi) | Image | 10 | 32 | 554 | + + +### Value transfer + +We compare CWS to TMCS, Beta Shapley and LOO on a sample removal task with value +transfer to another model. Values are computed using a logistic regression +model, and used to prune the training set for a neural network. Random values +serve as a baseline. The following plot shows valuation-set accuracy of the +network on the y-axis, and the number of samples removed on the x-axis. ![Accuracy of transferred values from logistic regression to an MLP](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg) diff --git a/docs_includes/abbreviations.md b/docs_includes/abbreviations.md index 622fa5a5d..aee5dba7f 100644 --- a/docs_includes/abbreviations.md +++ b/docs_includes/abbreviations.md @@ -1,16 +1,17 @@ *[CSP]: Constraint Satisfaction Problem *[CWS]: Class-wise Shapley +*[DUL]: Data Utility Learning *[GT]: Group Testing +*[IF]: Influence Function +*[iHVP]: inverse Hessian-vector product *[LC]: Least Core +*[LiSSA]: Linear-time Stochastic Second-order Algorithm *[LOO]: Leave-One-Out *[MCLC]: Monte Carlo Least Core *[MCS]: Monte Carlo Shapley *[ML]: Machine Learning *[MLRC]: Machine Learning Reproducibility Challenge *[MSE]: Mean Squared Error +*[PCA]: Principal Component Analysis *[SV]: Shapley Value *[TMCS]: Truncated Monte Carlo Shapley -*[IF]: Influence Function -*[iHVP]: inverse Hessian-vector product -*[LiSSA]: Linear-time Stochastic Second-order Algorithm -*[DUL]: Data Utility Learning From 09636152f22131e64a45ad9732f69bb74fc21f5b Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Sun, 8 Oct 2023 11:04:17 +0200 Subject: [PATCH 32/43] [skip ci] Minor --- docs/value/classwise-shapley.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/value/classwise-shapley.md b/docs/value/classwise-shapley.md index a86b71597..f1e522ffd 100644 --- a/docs/value/classwise-shapley.md +++ b/docs/value/classwise-shapley.md @@ -122,7 +122,8 @@ model, and used to prune the training set for a neural network. Random values serve as a baseline. The following plot shows valuation-set accuracy of the network on the y-axis, and the number of samples removed on the x-axis. -![Accuracy of transferred values from logistic regression to an MLP](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg) +![Accuracy after sample removal using values transferred from logistic regression +to an MLP](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg) Samples are removed from high to low valuation order and hence we expect a steep decrease in the curve. CWS is competitive with the compared methods. Especially @@ -138,6 +139,7 @@ This section takes a look at the distribution of CWS values for the above datasets. In the histogram plots one can see that it is slightly biased and often skewed. -![Distribution of values computed using logistic regression](img/classwise-shapley-example-densities.svg) +![Distribution of values computed using logistic +regression](img/classwise-shapley-example-densities.svg) The curves are an approximation of the density using KDE. From c1195ecf38d53bc51fb5babcb56cd8c3937c824a Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Sun, 8 Oct 2023 11:25:30 +0200 Subject: [PATCH 33/43] Delete duplicate section on CWS --- docs/value/shapley.md | 46 +++---------------------------------------- 1 file changed, 3 insertions(+), 43 deletions(-) diff --git a/docs/value/shapley.md b/docs/value/shapley.md index 4d1c78677..6de30f0ab 100644 --- a/docs/value/shapley.md +++ b/docs/value/shapley.md @@ -78,47 +78,6 @@ stop condition. This is an instance of a [MaxTime][pydvl.value.stopping.MaxTime] and [AbsoluteStandardError][pydvl.value.stopping.AbsoluteStandardError]. -### Class-wise Shapley - -Class-wise Shapley [@schoch_csshapley_2022] offers a distinct Shapley framework tailored -for classification problems. Let $D$ be the dataset, $D_{y_i}$ be the subset of $D$ with -labels $y_i$, and $D_{-y_i}$ be the complement of $D_{y_i}$ in $D$. The key idea is that -a sample $(x_i, y_i)$, might enhance the overall performance on $D$, while being -detrimental for the performance on $D_{y_i}$. To address this issue, the -authors introduced the estimator - -$$ -v_u(i) = \frac{1}{2^{|D_{-y_i}|}} \sum_{S_{-y_i}} \frac{1}{|D_{y_i}|!} -\sum_{S_{y_i}} \binom{|D_{y_i}|-1}{|S_{y_i}|}^{-1} -[u( S_{y_i} \cup \{i\} | S_{-y_i} ) − u( S_{y_i} | S_{-y_i})], -$$ - -where $S_{y_i} \subseteq D_{y_i} \setminus \{i\}$ and $S_{-y_i} \subseteq D_{-y_i}$. In -other words, the summations are over the powerset of $D_{y_i} \setminus \{i\}$ and -$D_{-y_i}$ respectively. The algorithm can be applied by using the snippet - -```python -from pydvl.utils import Dataset, Utility -from pydvl.value import HistoryDeviation, MaxChecks, RelativeTruncation -from pydvl.value.shapley.classwise import compute_classwise_shapley_values, \ - ClasswiseScorer - -model = ... -data = Dataset(...) -scoring = ("accuracy") -utility = Utility(model, data, scoring) -values = compute_classwise_shapley_values( - utility, - done=HistoryDeviation(n_steps=500, rtol=5e-2), - truncation=RelativeTruncation(utility, rtol=0.01), - done_sample_complements=MaxChecks(1), - normalize_values=True -) -``` - -where `ClasswiseScorer` is a special type of scorer only applicable for classification -problems. In practical applications, the evaluation of this estimator leverages both -Monte Carlo sampling and permutation Monte Carlo sampling [@castro_polynomial_2009]. ### Owen sampling @@ -216,8 +175,9 @@ values = compute_shapley_values(u=utility, mode="knn") ### Group testing -An alternative approach introduced in [@jia_efficient_2019a] first approximates -the differences of values with a Monte Carlo sum. With +An alternative method for the approximation of Shapley values introduced in +[@jia_efficient_2019a] first estimates the differences of values with a Monte +Carlo sum. With $$\hat{\Delta}_{i j} \approx v_i - v_j,$$ From 79e4e5a37838c0393f5ccb63bf97555038a76080 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Sun, 8 Oct 2023 11:25:50 +0200 Subject: [PATCH 34/43] [skip ci] More doc tweaks --- docs/value/notation.md | 11 +++++++++-- mkdocs.yml | 2 +- src/pydvl/value/shapley/gt.py | 6 ++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/value/notation.md b/docs/value/notation.md index f14ce6466..83054d5e6 100644 --- a/docs/value/notation.md +++ b/docs/value/notation.md @@ -4,17 +4,24 @@ title: Notation for valuation # Notation for valuation +!!! todo + Organize this page better and use its content consistently throughout the + documentation. + The following notation is used throughout the documentation: Let $D = \{x_1, \ldots, x_n\}$ be a training set of $n$ samples. The utility function $u:\mathcal{D} \rightarrow \mathbb{R}$ maps subsets of $D$ -to real numbers. +to real numbers. In pyDVL, we typically call this mappin a **score** for +consistency with sklearn, and reserve the term **utility** for the triple of +dataset $D$, model $f$ and score $u$, since they are used together to compute +the value. The value $v$ of the $i$-th sample in dataset $D$ wrt. utility $u$ is denoted as $v_u(x_i)$ or simply $v(i)$. -For any $S \subseteq D$, we donote by $S_{-i}$ the set of samples in $D$ +For any $S \subseteq D$, we denote by $S_{-i}$ the set of samples in $D$ excluding $x_i$, and $S_{+i}$ denotes the set $S$ with $x_i$ added. The marginal utility of adding sample $x_i$ to a subset $S$ is denoted as diff --git a/mkdocs.yml b/mkdocs.yml index 867379796..c4a80316a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -193,7 +193,7 @@ nav: - Data Valuation: - Introduction: value/index.md - Notation: value/notation.md - - Shapley Values: value/shapley.md + - Shapley values: value/shapley.md - Semi-values: value/semi-values.md - The core: value/the-core.md - Classwise Shapley: value/classwise-shapley.md diff --git a/src/pydvl/value/shapley/gt.py b/src/pydvl/value/shapley/gt.py index b193af6e5..2d3be7710 100644 --- a/src/pydvl/value/shapley/gt.py +++ b/src/pydvl/value/shapley/gt.py @@ -17,8 +17,10 @@ ## References [^1]: Jia, R. et al., 2019. - [Towards Efficient Data Valuation Based on the Shapley Value](https://proceedings.mlr.press/v89/jia19a.html). - In: Proceedings of the 22nd International Conference on Artificial Intelligence and Statistics, pp. 1167–1176. PMLR. + [Towards Efficient Data Valuation Based on the Shapley + Value](https://proceedings.mlr.press/v89/jia19a.html). + In: Proceedings of the 22nd International Conference on Artificial + Intelligence and Statistics, pp. 1167–1176. PMLR. """ import logging from collections import namedtuple From 6236caaa2caaf9c5b5561b92f707eeb1c205eff3 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Sun, 8 Oct 2023 11:34:33 +0200 Subject: [PATCH 35/43] [skip ci] Add CWS paper to README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index ae72b280c..80e5c1f2f 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,11 @@ methods from the following papers: Efficient Data Value](https://proceedings.mlr.press/v202/kwon23e.html). In Proceedings of the 40th International Conference on Machine Learning, 18135–52. PMLR, 2023. +- Schoch, Stephanie, Haifeng Xu, and Yangfeng Ji. [CS-Shapley: Class-Wise + Shapley Values for Data Valuation in + Classification](https://openreview.net/forum?id=KTOcrOR5mQ9). In Proc. of the + Thirty-Sixth Conference on Neural Information Processing Systems (NeurIPS). + New Orleans, Louisiana, USA, 2022. Influence Functions compute the effect that single points have on an estimator / model. We implement methods from the following papers: From f2a0bfc53d8f67343fe485c01946c18737ee2565 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Sun, 8 Oct 2023 11:35:54 +0200 Subject: [PATCH 36/43] [skip ci] Updated CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c385d364c..b8eabc3b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- New method: Class-wise Shapley values + [PR #338](https://github.com/aai-institute/pyDVL/pull/338) - No longer using docker within tests to start a memcached server [PR #444](https://github.com/aai-institute/pyDVL/pull/444) - Improvements and fixes to notebooks From 94d95db3b3e9dab1919b3cb9de16a847f74c3cc8 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Tue, 10 Oct 2023 18:38:31 +0200 Subject: [PATCH 37/43] Finalize documentation, update plots. --- docs/value/classwise-shapley.md | 84 +- docs/value/img/classwise-shapley-density.svg | 1 + ...se-shapley-discounted-utility-function.svg | 68002 +--------------- .../classwise-shapley-example-densities.svg | 4714 -- .../img/classwise-shapley-metric-mlp-cv.svg | 1 + .../img/classwise-shapley-metric-mlp-mean.svg | 1 + ...stic-regression-to-logistic-regression.svg | 1 + ...curacy-drop-logistic-regression-to-mlp.svg | 32052 +------- docs_includes/abbreviations.md | 3 +- 9 files changed, 76 insertions(+), 104783 deletions(-) create mode 100644 docs/value/img/classwise-shapley-density.svg delete mode 100644 docs/value/img/classwise-shapley-example-densities.svg create mode 100644 docs/value/img/classwise-shapley-metric-mlp-cv.svg create mode 100644 docs/value/img/classwise-shapley-metric-mlp-mean.svg create mode 100644 docs/value/img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-logistic-regression.svg diff --git a/docs/value/classwise-shapley.md b/docs/value/classwise-shapley.md index f1e522ffd..4b4fc1f0f 100644 --- a/docs/value/classwise-shapley.md +++ b/docs/value/classwise-shapley.md @@ -91,9 +91,9 @@ and $g$ for an exploration with different base scores. The level curves for $f(x)=x$ and $g(x)=e^x$ are depicted below. The white lines illustrate the contour lines, annotated with their respective gradients. -![Level curves of the class-wise utility](img/classwise-shapley-discounted-utility-function.svg) +![Level curves of the class-wise utility](img/classwise-shapley-discounted-utility-function.svg){ align=left width=33% class=invertible } -## Evaluation +# Evaluation We evaluate the method on the nine datasets used in [@schoch_csshapley_2022], using the same pre-processing. For images, PCA is used to reduce down to 32 the @@ -113,17 +113,59 @@ pre-processing steps, please refer to the paper. | MNIST (binary) | Image | 2 | 32 | 554 | | MNIST (multi) | Image | 10 | 32 | 554 | +## Performance for (direct) point removal -### Value transfer +We compare the mean and the coefficient of variation (CV) of the weighted accuracy drop +(WAD) as proposed in [@schoch_csshapley_2022]. The metric is defined by -We compare CWS to TMCS, Beta Shapley and LOO on a sample removal task with value -transfer to another model. Values are computed using a logistic regression -model, and used to prune the training set for a neural network. Random values -serve as a baseline. The following plot shows valuation-set accuracy of the -network on the y-axis, and the number of samples removed on the x-axis. +$$ +\text{WAD} = a_T(D) - \sum_{j=1}^{n} \frac{a_{T_{-\{1 \colon j \}}}(D)}{j}, +$$ + +where $a_T(D)$ is the accuracy of the model (trained on $T$) evaluated on $D$ and +$T_{-\{1 \colon j \}}$ is the set $T$ without elements from $\{1 \colon j \}$. The +metric was evaluated over five runs and is summarized by mean $\mu_\text{WAD}$ and +standard deviation $\sigma_\text{WAD}$. The valuation of the training samples and the +evaluation on the validation samples are both calculated based on a logistic regression +model. Let's have a look at the mean + +![WAD drop (Mean)](img/classwise-shapley-metric-mlp-mean.svg){ align=left width=50% class=invertible } + +of the metric WAD. The table shows that CWS is competitive with all three other methods. +In all problems except `MNIST (multi)` it is better than TMCS, whereas in that +case TMCS has a slight advantage. Another important quantity is the CV +$\frac{\sigma_\text{WAD}}{\mu_\text{WAD}}$. It normalizes the standard +deviation relative to the mean. The results are shown below. + +![WAD drop (CV)](img/classwise-shapley-metric-mlp-cv.svg){ align=left width=50% class=invertible } + +It is noteworthy that CWS is not the best method in terms of CV (Lower CV means better +performance). For `CIFAR10`, `Click`, `CPU` and `MNIST (binary)` Beta Shapley has the +lowest CV. For `Diabetes`, `MNIST (multi)` and `Phoneme` CWS is the winner and for +`FMNIST` and `Covertype` TMCS takes the lead. Without considering LOO, TMCS has the +highest relative standard deviation. + +The following plot shows valuation-set accuracy of logistic regression on the y-axis. +The x-axis shows the number of samples removed. Random values serve as a baseline. + +![Accuracy after sample removal using values from logistic regression](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-logistic-regression.svg){ class=invertible } + +Overall we conclude that in terms of mean WAD CWS and TMCS are the best methods. In +terms of the CV CWS and Beta Shapley are the clear winners. Hence, CWS is a competitive +method for valuation of data sets with a low relative standard deviation. We remark that +for all valuation methods the same number of _evaluations of the marginal utility_ was +used. + +## Performance in value transfer for point removal + +Practically more relevant is transfer of values for one model to another one. As before +the values are calculated using a logistic regression model. However, here they are +used to prune the training set for a neural network. The following plot shows +valuation-set accuracy of the network on the y-axis, and the number of samples removed +on the x-axis. ![Accuracy after sample removal using values transferred from logistic regression -to an MLP](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg) +to an MLP](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg){ class=invertible } Samples are removed from high to low valuation order and hence we expect a steep decrease in the curve. CWS is competitive with the compared methods. Especially @@ -133,13 +175,23 @@ the performance is on par with TMC. For `MNIST (binary)` and `Phoneme` the performance is competitive. We remark that for all valuation methods the same number of _evaluations of the marginal utility_ was used. -### Value distribution +## Density of values + +Last but not least let's compare the distribution of values for TMCS (green) and CWS +(red). Therefore, the following plots show a histogram the density estimated by kernel +density estimation (KDE). + -This section takes a look at the distribution of CWS values for the above -datasets. In the histogram plots one can see that it is slightly biased and -often skewed. +![Density of TMCS and CWS](img/classwise-shapley-density.svg){ class=invertible } -![Distribution of values computed using logistic -regression](img/classwise-shapley-example-densities.svg) +As the metrics already suggest TMCS has a higher variance as CWS. In mean, they +seem to approximate the same quantity, which is not obvious due to their different +nature of their utility functions. -The curves are an approximation of the density using KDE. +For `Click` TMCS has a multi-modal distribution of values. This is inferior to CWS which +has only one-mode and is more stable on that dataset. `Click` is a very unbalanced +dataset and hence CWS seems to be more robust on unbalanced datasets. It seems that +CWS is a good way to handle classification problems. Given the underlying similarities +in the architecture of TMCS, Beta Shapley, and CWS algorithms, there's a clear pathway +for improving convergence rates, sample efficiency, and stabilize variance in all of +these methods. \ No newline at end of file diff --git a/docs/value/img/classwise-shapley-density.svg b/docs/value/img/classwise-shapley-density.svg new file mode 100644 index 000000000..554e26cd8 --- /dev/null +++ b/docs/value/img/classwise-shapley-density.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/value/img/classwise-shapley-discounted-utility-function.svg b/docs/value/img/classwise-shapley-discounted-utility-function.svg index c925f1e4a..70ed7ab58 100644 --- a/docs/value/img/classwise-shapley-discounted-utility-function.svg +++ b/docs/value/img/classwise-shapley-discounted-utility-function.svg @@ -1,68001 +1 @@ - - - - - - - - 2023-09-25T22:47:24.903053 - image/svg+xml - - - Matplotlib v3.7.0, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/docs/value/img/classwise-shapley-example-densities.svg b/docs/value/img/classwise-shapley-example-densities.svg deleted file mode 100644 index 3b95747a0..000000000 --- a/docs/value/img/classwise-shapley-example-densities.svg +++ /dev/null @@ -1,4714 +0,0 @@ - - - - - - - - 2023-10-05T03:20:15.147632 - image/svg+xml - - - Matplotlib v3.7.0, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/value/img/classwise-shapley-metric-mlp-cv.svg b/docs/value/img/classwise-shapley-metric-mlp-cv.svg new file mode 100644 index 000000000..696226e83 --- /dev/null +++ b/docs/value/img/classwise-shapley-metric-mlp-cv.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/value/img/classwise-shapley-metric-mlp-mean.svg b/docs/value/img/classwise-shapley-metric-mlp-mean.svg new file mode 100644 index 000000000..6dc40687f --- /dev/null +++ b/docs/value/img/classwise-shapley-metric-mlp-mean.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/value/img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-logistic-regression.svg b/docs/value/img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-logistic-regression.svg new file mode 100644 index 000000000..1071d5f0b --- /dev/null +++ b/docs/value/img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-logistic-regression.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/value/img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg b/docs/value/img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg index 54570bbf0..85a3244d8 100644 --- a/docs/value/img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg +++ b/docs/value/img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg @@ -1,32051 +1 @@ - - - - - - - - 2023-10-05T03:20:36.991079 - image/svg+xml - - - Matplotlib v3.7.0, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/docs_includes/abbreviations.md b/docs_includes/abbreviations.md index aee5dba7f..efb73bb22 100644 --- a/docs_includes/abbreviations.md +++ b/docs_includes/abbreviations.md @@ -1,4 +1,5 @@ *[CSP]: Constraint Satisfaction Problem +*[CV]: Coefficient of Variation *[CWS]: Class-wise Shapley *[DUL]: Data Utility Learning *[GT]: Group Testing @@ -14,4 +15,4 @@ *[MSE]: Mean Squared Error *[PCA]: Principal Component Analysis *[SV]: Shapley Value -*[TMCS]: Truncated Monte Carlo Shapley +*[TMCS]: Truncated Monte Carlo Shapley \ No newline at end of file From 37efbb88cb872bb2933980e41cd26c94f261675b Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Tue, 10 Oct 2023 19:35:25 +0200 Subject: [PATCH 38/43] Reformat `stopping.py`. --- src/pydvl/value/stopping.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pydvl/value/stopping.py b/src/pydvl/value/stopping.py index 56311e839..4ce4b27e8 100644 --- a/src/pydvl/value/stopping.py +++ b/src/pydvl/value/stopping.py @@ -554,12 +554,11 @@ def completion(self) -> float: def reset(self): self.start = time() - + def __str__(self): return f"MaxTime(seconds={self.max_seconds})" - class HistoryDeviation(StoppingCriterion): r"""A simple check for relative distance to a previous step in the computation. From fdc4a0653a5ebc4907adf40a6399244a2ada337b Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Wed, 11 Oct 2023 00:51:57 +0200 Subject: [PATCH 39/43] Add noise removal to README.md. --- docs/value/classwise-shapley.md | 107 ++++++++++++------ .../img/classwise-shapley-metric-auc-cv.svg | 1 + .../img/classwise-shapley-metric-auc-mean.svg | 1 + ...vg => classwise-shapley-metric-wad-cv.svg} | 0 ... => classwise-shapley-metric-wad-mean.svg} | 2 +- ...se-shapley-roc-auc-logistic-regression.svg | 1 + 6 files changed, 78 insertions(+), 34 deletions(-) create mode 100644 docs/value/img/classwise-shapley-metric-auc-cv.svg create mode 100644 docs/value/img/classwise-shapley-metric-auc-mean.svg rename docs/value/img/{classwise-shapley-metric-mlp-cv.svg => classwise-shapley-metric-wad-cv.svg} (100%) rename docs/value/img/{classwise-shapley-metric-mlp-mean.svg => classwise-shapley-metric-wad-mean.svg} (99%) create mode 100644 docs/value/img/classwise-shapley-roc-auc-logistic-regression.svg diff --git a/docs/value/classwise-shapley.md b/docs/value/classwise-shapley.md index 4b4fc1f0f..42f6d2764 100644 --- a/docs/value/classwise-shapley.md +++ b/docs/value/classwise-shapley.md @@ -4,12 +4,14 @@ title: Class-wise Shapley # Class-wise Shapley +## AlgorIntroductionithm + Class-wise Shapley (CWS) [@schoch_csshapley_2022] offers a Shapley framework tailored for classification problems. Let $D$ be a dataset, $D_{y_i}$ be the subset of $D$ with labels $y_i$, and $D_{-y_i}$ be the complement of $D_{y_i}$ in $D$. The key idea is that a sample $(x_i, y_i)$ might enhance the overall performance on $D$, while being detrimental for the performance on $D_{y_i}$. To -address this issue, the authors introduce the estimator +address this issue, the authors introduced $$ v_u(i) = \frac{1}{2^{|D_{-y_i}|}} \sum_{S_{-y_i}} @@ -54,7 +56,7 @@ the dataset. ``` -## Class-wise scorer +### Class-wise scorer In order to use the classwise Shapley value, one needs to define a [ClasswiseScorer][pydvl.value.shapley.classwise.ClasswiseScorer]. Given a sample @@ -88,12 +90,12 @@ and $g$ for an exploration with different base scores. ) ``` -The level curves for $f(x)=x$ and $g(x)=e^x$ are depicted below. The white lines +The level curves for $f(x)=x$ and $g(x)=e^x$ are depicted below. The lines illustrate the contour lines, annotated with their respective gradients. ![Level curves of the class-wise utility](img/classwise-shapley-discounted-utility-function.svg){ align=left width=33% class=invertible } -# Evaluation +## Evaluation We evaluate the method on the nine datasets used in [@schoch_csshapley_2022], using the same pre-processing. For images, PCA is used to reduce down to 32 the @@ -113,13 +115,15 @@ pre-processing steps, please refer to the paper. | MNIST (binary) | Image | 2 | 32 | 554 | | MNIST (multi) | Image | 10 | 32 | 554 | -## Performance for (direct) point removal +### Performance for (direct) point removal We compare the mean and the coefficient of variation (CV) of the weighted accuracy drop (WAD) as proposed in [@schoch_csshapley_2022]. The metric is defined by $$ -\text{WAD} = a_T(D) - \sum_{j=1}^{n} \frac{a_{T_{-\{1 \colon j \}}}(D)}{j}, +\text{WAD} = \sum_{j=1}^{n} \left ( \frac{1}{j} \sum_{i=1}^{j} +a_{T_{-\{1 \colon i-1 \}}}(D) - a_{T_{-\{1 \colon i \}}}(D) \right) += a_T(D) - \sum_{j=1}^{n} \frac{a_{T_{-\{1 \colon j \}}}(D)}{j} , $$ where $a_T(D)$ is the accuracy of the model (trained on $T$) evaluated on $D$ and @@ -129,15 +133,15 @@ standard deviation $\sigma_\text{WAD}$. The valuation of the training samples an evaluation on the validation samples are both calculated based on a logistic regression model. Let's have a look at the mean -![WAD drop (Mean)](img/classwise-shapley-metric-mlp-mean.svg){ align=left width=50% class=invertible } +![Weighted accuracy drop (Mean)](img/classwise-shapley-metric-wad-mean.svg){ align=left width=50% class=invertible } of the metric WAD. The table shows that CWS is competitive with all three other methods. In all problems except `MNIST (multi)` it is better than TMCS, whereas in that case TMCS has a slight advantage. Another important quantity is the CV $\frac{\sigma_\text{WAD}}{\mu_\text{WAD}}$. It normalizes the standard -deviation relative to the mean. The results are shown below. +deviation by the mean. The results are shown below. -![WAD drop (CV)](img/classwise-shapley-metric-mlp-cv.svg){ align=left width=50% class=invertible } +![Weighted accuracy drop (CV)](img/classwise-shapley-metric-wad-cv.svg){ align=left width=50% class=invertible } It is noteworthy that CWS is not the best method in terms of CV (Lower CV means better performance). For `CIFAR10`, `Click`, `CPU` and `MNIST (binary)` Beta Shapley has the @@ -146,20 +150,24 @@ lowest CV. For `Diabetes`, `MNIST (multi)` and `Phoneme` CWS is the winner and f highest relative standard deviation. The following plot shows valuation-set accuracy of logistic regression on the y-axis. -The x-axis shows the number of samples removed. Random values serve as a baseline. +The x-axis shows the number of samples removed. Random values serve as a baseline. +Each line represents five runs, whereas bootstrapping was used to estimate the 95% +confidence intervals. + ![Accuracy after sample removal using values from logistic regression](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-logistic-regression.svg){ class=invertible } -Overall we conclude that in terms of mean WAD CWS and TMCS are the best methods. In -terms of the CV CWS and Beta Shapley are the clear winners. Hence, CWS is a competitive -method for valuation of data sets with a low relative standard deviation. We remark that -for all valuation methods the same number of _evaluations of the marginal utility_ was -used. +Samples are removed from high to low valuation order and hence we expect a steep +decrease in the curve. Overall we conclude that in terms of mean WAD CWS and TMCS are +the best methods. In terms of CV, CWS and Beta Shapley are the clear winners. Hence, CWS +is a competitive method for valuation of data sets with a low relative standard +deviation. We remark that for all valuation methods the same number of _evaluations of +the marginal utility_ was used. -## Performance in value transfer for point removal +### Performance in value transfer for point removal -Practically more relevant is transfer of values for one model to another one. As before -the values are calculated using a logistic regression model. However, here they are +Practically more relevant is the transfer of values from one model to another one. As +before the values are calculated using logistic regression. However, this time they are used to prune the training set for a neural network. The following plot shows valuation-set accuracy of the network on the y-axis, and the number of samples removed on the x-axis. @@ -167,31 +175,64 @@ on the x-axis. ![Accuracy after sample removal using values transferred from logistic regression to an MLP](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg){ class=invertible } -Samples are removed from high to low valuation order and hence we expect a steep +Again samples are removed from high to low valuation order and hence we expect a steep decrease in the curve. CWS is competitive with the compared methods. Especially in very unbalanced datasets, like `Click`, the performance of CWS seems superior. In other datasets, like `Covertype` and `Diabetes` and `MNIST (multi)` the performance is on par with TMC. For `MNIST (binary)` and `Phoneme` the -performance is competitive. We remark that for all valuation methods the -same number of _evaluations of the marginal utility_ was used. +performance is competitive. -## Density of values +### Density of values -Last but not least let's compare the distribution of values for TMCS (green) and CWS -(red). Therefore, the following plots show a histogram the density estimated by kernel -density estimation (KDE). +This experiment compares the distribution of values for TMCS (green) and CWS +(red). Both methods are chosen due to their competivieness. The following plots show a +histogram as well as the density estimated by kernel density estimation (KDE). ![Density of TMCS and CWS](img/classwise-shapley-density.svg){ class=invertible } -As the metrics already suggest TMCS has a higher variance as CWS. In mean, they -seem to approximate the same quantity, which is not obvious due to their different -nature of their utility functions. +As apparent in the metric CV from the previous section, the variance of CWS is lower +than for TCMS. They seem to approximate the same form of distribution, although their +utility functions are different. For `Click` TMCS has a multi-modal distribution of values. This is inferior to CWS which has only one-mode and is more stable on that dataset. `Click` is a very unbalanced -dataset and hence CWS seems to be more robust on unbalanced datasets. It seems that -CWS is a good way to handle classification problems. Given the underlying similarities -in the architecture of TMCS, Beta Shapley, and CWS algorithms, there's a clear pathway -for improving convergence rates, sample efficiency, and stabilize variance in all of -these methods. \ No newline at end of file +dataset, and we conclude that CWS seems to be more robust on unbalanced datasets. + +### Noise removal for 20% of the flipped data + +Another type of experiment uses the algorithms to explore mis-labelled data points. The +indices are chosen randomly. Multi-class datasets are discarded, because they do not +possess a unique flipping strategy. The following table shows the mean of the area under +the curve (AUC) of five runs. + +![Area under the Curve (Mean)](img/classwise-shapley-metric-auc-mean.svg){ align=left width=50% class=invertible } + +In the majority of the cases TMCS has a slight advantage over CWS on average. For +`Click` CWS has a slight edge, most probably due to the unbalanced nature of `Click`. +The following plot shows the CV for the AUC of the five runs. + +![Area under the Curve (CV)](img/classwise-shapley-metric-auc-cv.svg){ align=left width=50% class=invertible } + +In terms of CV, CWS has a clear edge over TMCS and Beta Shapley. The following plot +shows the receiving operator characteristic (ROC) for the mean of five runs. + +![Receiver Operating Characteristic](img/classwise-shapley-roc-auc-logistic-regression.svg){ align=left width=50% class=invertible } + +The ROC curve is a plot of the true positive rate (TPR) against the false positive rate +(FPR). The TPR is the ratio of correctly classified positive samples to all positive +samples. The FPR is the ratio of incorrectly classified negative samples to all negative +samples. This tuple is calculated for all prefixes of the training set with respect to +the values. Although it seems that TMCS is the winner, considering sample efficiency, +CWS stays competitive. For a perfectly balanced dataset, CWS needs fewer samples than +TCMS on average. CWS is competitive and almost on par with TCMS, while requring less +samples on average. + +## Conclusion + +CWS is a reasonable and effective way to handle classification problems. It reduces the +computing power and variance by splitting up the data set into classes. Given the +underlying similarities in the architecture of TMCS, Beta Shapley, and CWS, there's a +clear pathway for improving convergence rates, sample efficiency, and stabilize variance +for TMCS and Beta Shapley. + diff --git a/docs/value/img/classwise-shapley-metric-auc-cv.svg b/docs/value/img/classwise-shapley-metric-auc-cv.svg new file mode 100644 index 000000000..3ddc5f5a4 --- /dev/null +++ b/docs/value/img/classwise-shapley-metric-auc-cv.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/value/img/classwise-shapley-metric-auc-mean.svg b/docs/value/img/classwise-shapley-metric-auc-mean.svg new file mode 100644 index 000000000..197ada82b --- /dev/null +++ b/docs/value/img/classwise-shapley-metric-auc-mean.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/value/img/classwise-shapley-metric-mlp-cv.svg b/docs/value/img/classwise-shapley-metric-wad-cv.svg similarity index 100% rename from docs/value/img/classwise-shapley-metric-mlp-cv.svg rename to docs/value/img/classwise-shapley-metric-wad-cv.svg diff --git a/docs/value/img/classwise-shapley-metric-mlp-mean.svg b/docs/value/img/classwise-shapley-metric-wad-mean.svg similarity index 99% rename from docs/value/img/classwise-shapley-metric-mlp-mean.svg rename to docs/value/img/classwise-shapley-metric-wad-mean.svg index 6dc40687f..7f74a384a 100644 --- a/docs/value/img/classwise-shapley-metric-mlp-mean.svg +++ b/docs/value/img/classwise-shapley-metric-wad-mean.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/docs/value/img/classwise-shapley-roc-auc-logistic-regression.svg b/docs/value/img/classwise-shapley-roc-auc-logistic-regression.svg new file mode 100644 index 000000000..0ec200f83 --- /dev/null +++ b/docs/value/img/classwise-shapley-roc-auc-logistic-regression.svg @@ -0,0 +1 @@ + \ No newline at end of file From 3cee0f8bfa33e37da7c9697f5869672c44d15c62 Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Wed, 11 Oct 2023 07:36:30 +0200 Subject: [PATCH 40/43] Refactor some text and add legend to density plot. --- docs/value/classwise-shapley.md | 170 ++++++++++--------- docs/value/img/classwise-shapley-density.svg | 2 +- 2 files changed, 91 insertions(+), 81 deletions(-) diff --git a/docs/value/classwise-shapley.md b/docs/value/classwise-shapley.md index 42f6d2764..b84864580 100644 --- a/docs/value/classwise-shapley.md +++ b/docs/value/classwise-shapley.md @@ -4,8 +4,6 @@ title: Class-wise Shapley # Class-wise Shapley -## AlgorIntroductionithm - Class-wise Shapley (CWS) [@schoch_csshapley_2022] offers a Shapley framework tailored for classification problems. Let $D$ be a dataset, $D_{y_i}$ be the subset of $D$ with labels $y_i$, and $D_{-y_i}$ be the complement of $D_{y_i}$ @@ -90,35 +88,45 @@ and $g$ for an exploration with different base scores. ) ``` -The level curves for $f(x)=x$ and $g(x)=e^x$ are depicted below. The lines -illustrate the contour lines, annotated with their respective gradients. - -![Level curves of the class-wise utility](img/classwise-shapley-discounted-utility-function.svg){ align=left width=33% class=invertible } +??? Surface of the discounted utility function + The level curves for $f(x)=x$ and $g(x)=e^x$ are depicted below. The lines + illustrate the contour lines, annotated with their respective gradients. + ![Level curves of the class-wise + utility](img/classwise-shapley-discounted-utility-function.svg){ align=left width=33% class=invertible } ## Evaluation We evaluate the method on the nine datasets used in [@schoch_csshapley_2022], -using the same pre-processing. For images, PCA is used to reduce down to 32 the -number of features found by a `Resnet18` model. For more details on the -pre-processing steps, please refer to the paper. - -??? info "Datasets used for evaluation" - | Dataset | Data Type | Classes | Input Dims | OpenML ID | - |----------------|-----------|---------|------------|-----------| - | Diabetes | Tabular | 2 | 8 | 37 | - | Click | Tabular | 2 | 11 | 1216 | - | CPU | Tabular | 2 | 21 | 197 | - | Covertype | Tabular | 7 | 54 | 1596 | - | Phoneme | Tabular | 2 | 5 | 1489 | - | FMNIST | Image | 2 | 32 | 40996 | - | CIFAR10 | Image | 2 | 32 | 40927 | - | MNIST (binary) | Image | 2 | 32 | 554 | - | MNIST (multi) | Image | 10 | 32 | 554 | - -### Performance for (direct) point removal - -We compare the mean and the coefficient of variation (CV) of the weighted accuracy drop -(WAD) as proposed in [@schoch_csshapley_2022]. The metric is defined by +using the same pre-processing. For images, PCA is used to project the feature, found +by a pre-trained `Resnet18` model, to 32 principal components. A loc-scale normalization +is performed for all models, except gradient boosting. The latter is not sensitive to +the scale of the features. The following table shows the datasets used in the + +| Dataset | Data Type | Classes | Input Dims | OpenML ID | +|----------------|-----------|---------|------------|-----------| +| Diabetes | Tabular | 2 | 8 | 37 | +| Click | Tabular | 2 | 11 | 1216 | +| CPU | Tabular | 2 | 21 | 197 | +| Covertype | Tabular | 7 | 54 | 1596 | +| Phoneme | Tabular | 2 | 5 | 1489 | +| FMNIST | Image | 2 | 32 | 40996 | +| CIFAR10 | Image | 2 | 32 | 40927 | +| MNIST (binary) | Image | 2 | 32 | 554 | +| MNIST (multi) | Image | 10 | 32 | 554 | + +experiments. In general there are three different experiments: point removal, noise +removal and a distribution analysis. Metrics are evaluated as tables for mean and +coefficient of variation (CV) $\frac{\sigma}{\mu}$ of an inner metric. The former +displays the performance of the method, whereas the latter displays the repeatability of +the method. We assume the mean has to be maximized and the CV has to be minimized. +Furthermore, we remark that for all sampling-based valuation methods the same number of +_evaluations of the marginal utility_ was used. This is important, to make the +algorithms comparable. In practice one should consider using a more sophisticated +stopping criterion. + +### Dataset pruning for logistic regression + +Weighted accuracy drop (WAD) [@schoch_csshapley_2022] is defined as $$ \text{WAD} = \sum_{j=1}^{n} \left ( \frac{1}{j} \sum_{i=1}^{j} @@ -133,15 +141,16 @@ standard deviation $\sigma_\text{WAD}$. The valuation of the training samples an evaluation on the validation samples are both calculated based on a logistic regression model. Let's have a look at the mean -![Weighted accuracy drop (Mean)](img/classwise-shapley-metric-wad-mean.svg){ align=left width=50% class=invertible } +![Weighted accuracy drop +(Mean)](img/classwise-shapley-metric-wad-mean.svg){ align=left width=50% class=invertible } of the metric WAD. The table shows that CWS is competitive with all three other methods. In all problems except `MNIST (multi)` it is better than TMCS, whereas in that -case TMCS has a slight advantage. Another important quantity is the CV -$\frac{\sigma_\text{WAD}}{\mu_\text{WAD}}$. It normalizes the standard -deviation by the mean. The results are shown below. +case TMCS has a slight advantage. Another important quantity is the CV. The results are +shown below. -![Weighted accuracy drop (CV)](img/classwise-shapley-metric-wad-cv.svg){ align=left width=50% class=invertible } +![Weighted accuracy drop +(CV)](img/classwise-shapley-metric-wad-cv.svg){ align=left width=50% class=invertible } It is noteworthy that CWS is not the best method in terms of CV (Lower CV means better performance). For `CIFAR10`, `Click`, `CPU` and `MNIST (binary)` Beta Shapley has the @@ -155,16 +164,15 @@ Each line represents five runs, whereas bootstrapping was used to estimate the 9 confidence intervals. -![Accuracy after sample removal using values from logistic regression](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-logistic-regression.svg){ class=invertible } +![Accuracy after sample removal using values from logistic +regression](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-logistic-regression.svg){ class=invertible } Samples are removed from high to low valuation order and hence we expect a steep decrease in the curve. Overall we conclude that in terms of mean WAD CWS and TMCS are the best methods. In terms of CV, CWS and Beta Shapley are the clear winners. Hence, CWS -is a competitive method for valuation of data sets with a low relative standard -deviation. We remark that for all valuation methods the same number of _evaluations of -the marginal utility_ was used. +is a competitive CV. -### Performance in value transfer for point removal +### Dataset pruning for neural network by value transfer Practically more relevant is the transfer of values from one model to another one. As before the values are calculated using logistic regression. However, this time they are @@ -172,67 +180,69 @@ used to prune the training set for a neural network. The following plot shows valuation-set accuracy of the network on the y-axis, and the number of samples removed on the x-axis. -![Accuracy after sample removal using values transferred from logistic regression -to an MLP](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg){ class=invertible } - -Again samples are removed from high to low valuation order and hence we expect a steep -decrease in the curve. CWS is competitive with the compared methods. Especially -in very unbalanced datasets, like `Click`, the performance of CWS seems -superior. In other datasets, like `Covertype` and `Diabetes` and `MNIST (multi)` -the performance is on par with TMC. For `MNIST (binary)` and `Phoneme` the -performance is competitive. +![Accuracy after sample removal using values transferred from logistic regression to an +MLP](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg){ class=invertible } -### Density of values - -This experiment compares the distribution of values for TMCS (green) and CWS -(red). Both methods are chosen due to their competivieness. The following plots show a -histogram as well as the density estimated by kernel density estimation (KDE). +As in the previous experiment samples are removed from high to low valuation order and +hence we expect a steep decrease in the curve. CWS is competitive with the compared +methods. Especially in very unbalanced datasets, like `Click`, the performance of CWS +seems superior. In other datasets, like `Covertype` and `Diabetes` and `MNIST (multi)` +the performance is on par with TMC. For `MNIST (binary)` and `Phoneme` the performance +is competitive. +### Detection of mis-labelled data points -![Density of TMCS and CWS](img/classwise-shapley-density.svg){ class=invertible } - -As apparent in the metric CV from the previous section, the variance of CWS is lower -than for TCMS. They seem to approximate the same form of distribution, although their -utility functions are different. - -For `Click` TMCS has a multi-modal distribution of values. This is inferior to CWS which -has only one-mode and is more stable on that dataset. `Click` is a very unbalanced -dataset, and we conclude that CWS seems to be more robust on unbalanced datasets. - -### Noise removal for 20% of the flipped data - -Another type of experiment uses the algorithms to explore mis-labelled data points. The -indices are chosen randomly. Multi-class datasets are discarded, because they do not +The next experiment uses the algorithms to detect mis-labelled data points. 20% of the +indices is selected by choice. Multi-class datasets are discarded, because they do not possess a unique flipping strategy. The following table shows the mean of the area under -the curve (AUC) of five runs. +the curve (AUC) for five runs. -![Area under the Curve (Mean)](img/classwise-shapley-metric-auc-mean.svg){ align=left width=50% class=invertible } +![Area under the Curve +(Mean)](img/classwise-shapley-metric-auc-mean.svg){ align=left width=50% class=invertible } In the majority of the cases TMCS has a slight advantage over CWS on average. For `Click` CWS has a slight edge, most probably due to the unbalanced nature of `Click`. The following plot shows the CV for the AUC of the five runs. -![Area under the Curve (CV)](img/classwise-shapley-metric-auc-cv.svg){ align=left width=50% class=invertible } +![Area under the Curve +(CV)](img/classwise-shapley-metric-auc-cv.svg){ align=left width=50% class=invertible } -In terms of CV, CWS has a clear edge over TMCS and Beta Shapley. The following plot -shows the receiving operator characteristic (ROC) for the mean of five runs. +In terms of CV, CWS has a clear edge over TMCS and Beta Shapley. The receiving operator +characteristic (ROC) curve is a plot of the precision to the recall. The classifier +uses the $n$-smallest values +respect to the order of the valuation. The following plot shows thec (ROC) for the mean +of five runs. -![Receiver Operating Characteristic](img/classwise-shapley-roc-auc-logistic-regression.svg){ align=left width=50% class=invertible } +![Receiver Operating +Characteristic](img/classwise-shapley-roc-auc-logistic-regression.svg){ align=left width=50% class=invertible } -The ROC curve is a plot of the true positive rate (TPR) against the false positive rate -(FPR). The TPR is the ratio of correctly classified positive samples to all positive -samples. The FPR is the ratio of incorrectly classified negative samples to all negative -samples. This tuple is calculated for all prefixes of the training set with respect to -the values. Although it seems that TMCS is the winner, considering sample efficiency, +Although it seems that TMCS is the winner: If you consider sample efficiency, CWS stays competitive. For a perfectly balanced dataset, CWS needs fewer samples than -TCMS on average. CWS is competitive and almost on par with TCMS, while requring less -samples on average. +TCMS on average. Furthermore, CWS is almost on par with TCMS performance-wise. + +### Density of values + +This experiment compares the distribution of values for TMCS (green) and CWS +(red). Both methods are chosen due to their competitiveness. The plot shows a +histogram as well as the density estimated by kernel density estimation (KDE) for each +dataset. + +![Density of TMCS and +CWS](img/classwise-shapley-density.svg){ class=invertible } + +Similar to the behaviour of the CV from the previous section, the variance of CWS is +lower than for TCMS. They seem to approximate the same of distribution, although their +utility functions are quite different. + +For `Click` TMCS has a multi-modal distribution of values. This is inferior to CWS which +has only one-mode and is more stable on that dataset. `Click` is a very unbalanced +dataset, and we conclude that CWS seems to be more robust on unbalanced datasets. ## Conclusion CWS is a reasonable and effective way to handle classification problems. It reduces the computing power and variance by splitting up the data set into classes. Given the underlying similarities in the architecture of TMCS, Beta Shapley, and CWS, there's a -clear pathway for improving convergence rates, sample efficiency, and stabilize variance -for TMCS and Beta Shapley. +clear pathway for improving convergence rates, sample efficiency, and stabilizing +variance for TMCS and Beta Shapley. diff --git a/docs/value/img/classwise-shapley-density.svg b/docs/value/img/classwise-shapley-density.svg index 554e26cd8..44d954546 100644 --- a/docs/value/img/classwise-shapley-density.svg +++ b/docs/value/img/classwise-shapley-density.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file From 72d0105a3b4953ed2b4c452688ce9e787254486d Mon Sep 17 00:00:00 2001 From: Markus Semmler Date: Thu, 12 Oct 2023 20:42:26 +0200 Subject: [PATCH 41/43] Remove variance remark. --- docs/value/classwise-shapley.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/value/classwise-shapley.md b/docs/value/classwise-shapley.md index b84864580..0466d75fa 100644 --- a/docs/value/classwise-shapley.md +++ b/docs/value/classwise-shapley.md @@ -88,7 +88,7 @@ and $g$ for an exploration with different base scores. ) ``` -??? Surface of the discounted utility function +??? "Surface of the discounted utility function" The level curves for $f(x)=x$ and $g(x)=e^x$ are depicted below. The lines illustrate the contour lines, annotated with their respective gradients. ![Level curves of the class-wise @@ -117,7 +117,10 @@ the scale of the features. The following table shows the datasets used in the experiments. In general there are three different experiments: point removal, noise removal and a distribution analysis. Metrics are evaluated as tables for mean and coefficient of variation (CV) $\frac{\sigma}{\mu}$ of an inner metric. The former -displays the performance of the method, whereas the latter displays the repeatability of +displays the performance of the method, whereas the latter displays the stability of a +method. We normalize by the mean to make standard deviations for different runs +comparable. + the method. We assume the mean has to be maximized and the CV has to be minimized. Furthermore, we remark that for all sampling-based valuation methods the same number of _evaluations of the marginal utility_ was used. This is important, to make the @@ -231,8 +234,8 @@ dataset. CWS](img/classwise-shapley-density.svg){ class=invertible } Similar to the behaviour of the CV from the previous section, the variance of CWS is -lower than for TCMS. They seem to approximate the same of distribution, although their -utility functions are quite different. +lower than for TCMS. They seem to approximate the same mode although their utility +function is very different. For `Click` TMCS has a multi-modal distribution of values. This is inferior to CWS which has only one-mode and is more stable on that dataset. `Click` is a very unbalanced From 5ee24bdc94f1c62471f864c8b73e999a0a038161 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Sat, 14 Oct 2023 13:55:00 +0200 Subject: [PATCH 42/43] [skip ci] Bunch of fixes to the doc --- docs/value/classwise-shapley.md | 280 +++++++++++++++++--------------- docs_includes/abbreviations.md | 5 +- 2 files changed, 152 insertions(+), 133 deletions(-) diff --git a/docs/value/classwise-shapley.md b/docs/value/classwise-shapley.md index 0466d75fa..4ae2c3f9f 100644 --- a/docs/value/classwise-shapley.md +++ b/docs/value/classwise-shapley.md @@ -5,11 +5,12 @@ title: Class-wise Shapley # Class-wise Shapley Class-wise Shapley (CWS) [@schoch_csshapley_2022] offers a Shapley framework -tailored for classification problems. Let $D$ be a dataset, $D_{y_i}$ be the -subset of $D$ with labels $y_i$, and $D_{-y_i}$ be the complement of $D_{y_i}$ -in $D$. The key idea is that a sample $(x_i, y_i)$ might enhance the overall -performance on $D$, while being detrimental for the performance on $D_{y_i}$. To -address this issue, the authors introduced +tailored for classification problems. Given a sample $x_i$ with label $y_i \in +\mathbb{N}$, let $D_{y_i}$ be the subset of $D$ with labels $y_i$, and +$D_{-y_i}$ be the complement of $D_{y_i}$ in $D$. The key idea is that the +sample $(x_i, y_i)$ might improve the overall model performance on $D$, while +being detrimental for the performance on $D_{y_i},$ e.g. because of a wrong +label. To address this issue, the authors introduced $$ v_u(i) = \frac{1}{2^{|D_{-y_i}|}} \sum_{S_{-y_i}} @@ -20,14 +21,15 @@ v_u(i) = \frac{1}{2^{|D_{-y_i}|}} \sum_{S_{-y_i}} $$ where $S_{y_i} \subseteq D_{y_i} \setminus \{i\}$ and $S_{-y_i} \subseteq -D_{-y_i}$, and the function $\delta$ is called **set-conditional marginal -Shapley value**. It is defined as +D_{-y_i}$ is _arbitrary_ (in particular, not the complement of $S_{y_i}$). The +function $\delta$ is called **set-conditional marginal Shapley value** and is +defined as $$ -\delta(S | C) = u( S \cup \{i\} | C ) − u(S | C), +\delta(S | C) = u( S_{+i} | C ) − u(S | C), $$ -where $i \notin S, C$ and $S \bigcap C = \emptyset$. +for any set $S$ such that $i \notin S, C$ and $S \cap C = \emptyset$. In practical applications, estimating this quantity is done both with Monte Carlo sampling of the powerset, and the set of index permutations @@ -36,7 +38,12 @@ original Shapley value, although the actual speed-up depends on the model and the dataset. -??? Example "Computing classwise Shapley values" +!!! Example "Computing classwise Shapley values" + Like all other game-theoretic valuation methods, CWS requires a + [Utility][pydvl.utils.utility.Utility] object constructed with model and + dataset, with the peculiarity of requiring a specific + [ClasswiseScorer][pydvl.value.shapley.classwise.ClasswiseScorer]: + ```python from pydvl.value import * @@ -54,15 +61,14 @@ the dataset. ``` -### Class-wise scorer +### The class-wise scorer In order to use the classwise Shapley value, one needs to define a -[ClasswiseScorer][pydvl.value.shapley.classwise.ClasswiseScorer]. Given a sample -$x_i$ with label $y_i \in \mathbb{N}$, we define two disjoint sets $D_{y_i}$ and -$D_{-y_i}$ and define +[ClasswiseScorer][pydvl.value.shapley.classwise.ClasswiseScorer]. This scorer +is defined as $$ -u(S) = f(a_S(D_{y_i}))) g(a_S(D_{-y_i}))), +u(S) = f(a_S(D_{y_i})) g(a_S(D_{-y_i})), $$ where $f$ and $g$ are monotonically increasing functions, $a_S(D_{y_i})$ is the @@ -74,7 +80,10 @@ The authors show that $f(x)=x$ and $g(x)=e^x$ have favorable properties and are therefore the defaults, but we leave the option to set different functions $f$ and $g$ for an exploration with different base scores. -??? Example "The default class-wise scorer" +!!! Example "The default class-wise scorer" + Constructing the CWS scorer requires choosing a metric and the functions $f$ + and $g$: + ```python import numpy as np from pydvl.value.shapley.classwise import ClasswiseScorer @@ -96,40 +105,47 @@ and $g$ for an exploration with different base scores. ## Evaluation -We evaluate the method on the nine datasets used in [@schoch_csshapley_2022], -using the same pre-processing. For images, PCA is used to project the feature, found -by a pre-trained `Resnet18` model, to 32 principal components. A loc-scale normalization -is performed for all models, except gradient boosting. The latter is not sensitive to -the scale of the features. The following table shows the datasets used in the - -| Dataset | Data Type | Classes | Input Dims | OpenML ID | -|----------------|-----------|---------|------------|-----------| -| Diabetes | Tabular | 2 | 8 | 37 | -| Click | Tabular | 2 | 11 | 1216 | -| CPU | Tabular | 2 | 21 | 197 | -| Covertype | Tabular | 7 | 54 | 1596 | -| Phoneme | Tabular | 2 | 5 | 1489 | -| FMNIST | Image | 2 | 32 | 40996 | -| CIFAR10 | Image | 2 | 32 | 40927 | -| MNIST (binary) | Image | 2 | 32 | 554 | -| MNIST (multi) | Image | 10 | 32 | 554 | - -experiments. In general there are three different experiments: point removal, noise -removal and a distribution analysis. Metrics are evaluated as tables for mean and -coefficient of variation (CV) $\frac{\sigma}{\mu}$ of an inner metric. The former -displays the performance of the method, whereas the latter displays the stability of a -method. We normalize by the mean to make standard deviations for different runs -comparable. - -the method. We assume the mean has to be maximized and the CV has to be minimized. -Furthermore, we remark that for all sampling-based valuation methods the same number of -_evaluations of the marginal utility_ was used. This is important, to make the -algorithms comparable. In practice one should consider using a more sophisticated -stopping criterion. - -### Dataset pruning for logistic regression - -Weighted accuracy drop (WAD) [@schoch_csshapley_2022] is defined as +We illustrate the method with two experiments: point removal and noise removal, +as well as an analysis of the distribution of the values. For this we employ the +nine datasets used in [@schoch_csshapley_2022], using the same pre-processing. +For images, PCA is used to reduce down to 32 the features found by a pre-trained +`Resnet18` model. Standard loc-scale normalization is performed for all models +except gradient boosting, since the latter is not sensitive to the scale of the +features. + +??? info "Datasets used for evaluation" + | Dataset | Data Type | Classes | Input Dims | OpenML ID | + |----------------|-----------|---------|------------|-----------| + | Diabetes | Tabular | 2 | 8 | 37 | + | Click | Tabular | 2 | 11 | 1216 | + | CPU | Tabular | 2 | 21 | 197 | + | Covertype | Tabular | 7 | 54 | 1596 | + | Phoneme | Tabular | 2 | 5 | 1489 | + | FMNIST | Image | 2 | 32 | 40996 | + | CIFAR10 | Image | 2 | 32 | 40927 | + | MNIST (binary) | Image | 2 | 32 | 554 | + | MNIST (multi) | Image | 10 | 32 | 554 | + +We show mean and coefficient of variation (CV) $\frac{\sigma}{\mu}$ of an "inner +metric". The former shows the performance of the method, whereas the latter +displays its stability: we normalize by the mean to see the relative effect of +the standard deviation. Ideally the mean value is maximal and CV minimal. + +Finally, we note that for all sampling-based valuation methods the same number +of _evaluations of the marginal utility_ was used. This is important to make the +algorithms comparable, but in practice one should consider using a more +sophisticated stopping criterion. + +### Dataset pruning for logistic regression (point removal) + +In (best-)point removal, one first computes values for the training set and then +removes in sequence the points with the highest values. After each removal, the +remaining points are used to train the model from scratch and performance is +measured on a test set. This produces a curve of performance vs. number of +points removed which we show below. + +As a scalar summary of this curve, [@schoch_csshapley_2022] define **Weighted +Accuracy Drop** (WAD) as: $$ \text{WAD} = \sum_{j=1}^{n} \left ( \frac{1}{j} \sum_{i=1}^{j} @@ -137,115 +153,115 @@ a_{T_{-\{1 \colon i-1 \}}}(D) - a_{T_{-\{1 \colon i \}}}(D) \right) = a_T(D) - \sum_{j=1}^{n} \frac{a_{T_{-\{1 \colon j \}}}(D)}{j} , $$ -where $a_T(D)$ is the accuracy of the model (trained on $T$) evaluated on $D$ and -$T_{-\{1 \colon j \}}$ is the set $T$ without elements from $\{1 \colon j \}$. The -metric was evaluated over five runs and is summarized by mean $\mu_\text{WAD}$ and -standard deviation $\sigma_\text{WAD}$. The valuation of the training samples and the -evaluation on the validation samples are both calculated based on a logistic regression -model. Let's have a look at the mean +where $a_T(D)$ is the accuracy of the model (trained on $T$) evaluated on $D$ +and $T_{-\{1 \colon j \}}$ is the set $T$ without elements from $\{1, \dots , j +\}$. -![Weighted accuracy drop -(Mean)](img/classwise-shapley-metric-wad-mean.svg){ align=left width=50% class=invertible } +We run the point removal experiment for a logistic regression model five times +and compute WAD for each run, then report the mean $\mu_\text{WAD}$ and standard +deviation $\sigma_\text{WAD}$. -of the metric WAD. The table shows that CWS is competitive with all three other methods. -In all problems except `MNIST (multi)` it is better than TMCS, whereas in that -case TMCS has a slight advantage. Another important quantity is the CV. The results are -shown below. +![Mean WAD for best-point removal on logistic regression. Values +computed using LOO, CWS, Beta Shapley, and TMCS +](img/classwise-shapley-metric-wad-mean.svg){ class=invertible } -![Weighted accuracy drop -(CV)](img/classwise-shapley-metric-wad-cv.svg){ align=left width=50% class=invertible } +We see that CWS is competitive with all three other methods. In all problems +except `MNIST (multi)` it outperforms TMCS, while in that case TMCS has a slight +advantage. -It is noteworthy that CWS is not the best method in terms of CV (Lower CV means better -performance). For `CIFAR10`, `Click`, `CPU` and `MNIST (binary)` Beta Shapley has the -lowest CV. For `Diabetes`, `MNIST (multi)` and `Phoneme` CWS is the winner and for -`FMNIST` and `Covertype` TMCS takes the lead. Without considering LOO, TMCS has the -highest relative standard deviation. +In order to understand the variability of WAD we look at its coefficient of +variation (lower is better): -The following plot shows valuation-set accuracy of logistic regression on the y-axis. -The x-axis shows the number of samples removed. Random values serve as a baseline. -Each line represents five runs, whereas bootstrapping was used to estimate the 95% -confidence intervals. +![Coefficient of Variation of WAD for best-point removal on logistic regression. +Values computed using LOO, CWS, Beta Shapley, and TMCS +](img/classwise-shapley-metric-wad-cv.svg){ class=invertible } +CWS is not the best method in terms of CV. For `CIFAR10`, `Click`, `CPU` and +`MNIST (binary)` Beta Shapley has the lowest CV. For `Diabetes`, `MNIST (multi)` +and `Phoneme` CWS is the winner and for `FMNIST` and `Covertype` TMCS takes the +lead. Besides LOO, TMCS has the highest relative standard deviation. -![Accuracy after sample removal using values from logistic +The following plot shows accuracy vs number of samples removed. Random values +serve as a baseline. The shaded area represents the 95% bootstrap confidence +interval of the mean across 5 runs. + +![Accuracy after best-sample removal using values from logistic regression](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-logistic-regression.svg){ class=invertible } -Samples are removed from high to low valuation order and hence we expect a steep -decrease in the curve. Overall we conclude that in terms of mean WAD CWS and TMCS are -the best methods. In terms of CV, CWS and Beta Shapley are the clear winners. Hence, CWS -is a competitive CV. +Because samples are removed from high to low valuation order, we expect a steep +decrease in the curve. + +Overall we conclude that in terms of mean WAD, CWS and TMCS perform best, with +CWS's CV on par with Beta Shapley's, making CWS a competitive method. + + +### Dataset pruning for a neural network by value transfer + +Transfer of values from one model to another is probably of greater practical +relevance: values are computed using a cheap model and used to prune the dataset +before training a more expensive one. -### Dataset pruning for neural network by value transfer +The following plot shows accuracy vs number of samples removed for transfer from +logistic regression to a neural network. The shaded area represents the 95% +bootstrap confidence interval of the mean across 5 runs. -Practically more relevant is the transfer of values from one model to another one. As -before the values are calculated using logistic regression. However, this time they are -used to prune the training set for a neural network. The following plot shows -valuation-set accuracy of the network on the y-axis, and the number of samples removed -on the x-axis. +![Accuracy after sample removal using values transferred from logistic +regression to an MLP +](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg){ class=invertible } -![Accuracy after sample removal using values transferred from logistic regression to an -MLP](img/classwise-shapley-weighted-accuracy-drop-logistic-regression-to-mlp.svg){ class=invertible } +As in the previous experiment samples are removed from high to low valuation +order and hence we expect a steep decrease in the curve. CWS is competitive with +the other methods, especially in very unbalanced datasets like `Click`. In other +datasets, like `Covertype`, `Diabetes` and `MNIST (multi)` the performance is on +par with TMCS. -As in the previous experiment samples are removed from high to low valuation order and -hence we expect a steep decrease in the curve. CWS is competitive with the compared -methods. Especially in very unbalanced datasets, like `Click`, the performance of CWS -seems superior. In other datasets, like `Covertype` and `Diabetes` and `MNIST (multi)` -the performance is on par with TMC. For `MNIST (binary)` and `Phoneme` the performance -is competitive. -### Detection of mis-labelled data points +### Detection of mis-labeled data points -The next experiment uses the algorithms to detect mis-labelled data points. 20% of the -indices is selected by choice. Multi-class datasets are discarded, because they do not -possess a unique flipping strategy. The following table shows the mean of the area under -the curve (AUC) for five runs. +The next experiment tries to detect mis-labeled data points in binary +classification tasks. 20% of the indices is flipped at random (we don't consider +multi-class datasets because there isn't a unique flipping strategy). The +following table shows the mean of the area under the curve (AUC) for five runs. -![Area under the Curve -(Mean)](img/classwise-shapley-metric-auc-mean.svg){ align=left width=50% class=invertible } +![Mean AUC for mis-labeled data point detection. Values computed using LOO, CWS, +Beta Shapley, and +TMCS](img/classwise-shapley-metric-auc-mean.svg){ class=invertible } -In the majority of the cases TMCS has a slight advantage over CWS on average. For -`Click` CWS has a slight edge, most probably due to the unbalanced nature of `Click`. -The following plot shows the CV for the AUC of the five runs. +In the majority of cases TMCS has a slight advantage over CWS, except for +`Click`, where CWS has a slight edge, most probably due to the unbalanced nature +of the dataset. The following plot shows the CV for the AUC of the five runs. -![Area under the Curve -(CV)](img/classwise-shapley-metric-auc-cv.svg){ align=left width=50% class=invertible } +![Coefficient of variation of AUC for mis-labeled data point detection. Values +computed using LOO, CWS, Beta Shapley, and TMCS +](img/classwise-shapley-metric-auc-cv.svg){ class=invertible } -In terms of CV, CWS has a clear edge over TMCS and Beta Shapley. The receiving operator -characteristic (ROC) curve is a plot of the precision to the recall. The classifier -uses the $n$-smallest values -respect to the order of the valuation. The following plot shows thec (ROC) for the mean -of five runs. +In terms of CV, CWS has a clear edge over TMCS and Beta Shapley. -![Receiver Operating -Characteristic](img/classwise-shapley-roc-auc-logistic-regression.svg){ align=left width=50% class=invertible } +Finally, we look at the ROC curves training the classifier on the $n$ first +samples in _increasing_ order of valuation (i.e. starting with the worst): -Although it seems that TMCS is the winner: If you consider sample efficiency, -CWS stays competitive. For a perfectly balanced dataset, CWS needs fewer samples than -TCMS on average. Furthermore, CWS is almost on par with TCMS performance-wise. +![Mean ROC across 5 runs with 95% bootstrap +CI](img/classwise-shapley-roc-auc-logistic-regression.svg){ class=invertible } -### Density of values +Although at first sight TMCS seems to be the winner, CWS stays competitive after +factoring in running time. For a perfectly balanced dataset, CWS needs on +average fewer samples than TCMS. -This experiment compares the distribution of values for TMCS (green) and CWS -(red). Both methods are chosen due to their competitiveness. The plot shows a -histogram as well as the density estimated by kernel density estimation (KDE) for each -dataset. +### Value distribution -![Density of TMCS and -CWS](img/classwise-shapley-density.svg){ class=invertible } +For illustration, we compare the distribution of values computed by TMCS and +CWS. -Similar to the behaviour of the CV from the previous section, the variance of CWS is -lower than for TCMS. They seem to approximate the same mode although their utility -function is very different. +![Histogram and estimated density of the values computed by TMCS and +CWS on all nine datasets](img/classwise-shapley-density.svg){ class=invertible } -For `Click` TMCS has a multi-modal distribution of values. This is inferior to CWS which -has only one-mode and is more stable on that dataset. `Click` is a very unbalanced -dataset, and we conclude that CWS seems to be more robust on unbalanced datasets. +For `Click` TMCS has a multi-modal distribution of values. We hypothesize that +this is due to the highly unbalanced nature of the dataset, and notice that CWS +has a single mode, leading to its greater performance on this dataset. ## Conclusion -CWS is a reasonable and effective way to handle classification problems. It reduces the -computing power and variance by splitting up the data set into classes. Given the -underlying similarities in the architecture of TMCS, Beta Shapley, and CWS, there's a -clear pathway for improving convergence rates, sample efficiency, and stabilizing -variance for TMCS and Beta Shapley. +CWS is an effective way to handle classification problems, in particular for +unbalanced datasets. It reduces the computing requirements by considering +in-class and out-of-class points separately. diff --git a/docs_includes/abbreviations.md b/docs_includes/abbreviations.md index efb73bb22..a89425885 100644 --- a/docs_includes/abbreviations.md +++ b/docs_includes/abbreviations.md @@ -11,8 +11,11 @@ *[MCLC]: Monte Carlo Least Core *[MCS]: Monte Carlo Shapley *[ML]: Machine Learning +*[MLP]: Multi-Layer Perceptron *[MLRC]: Machine Learning Reproducibility Challenge *[MSE]: Mean Squared Error *[PCA]: Principal Component Analysis +*[ROC]: Receiver Operating Characteristic *[SV]: Shapley Value -*[TMCS]: Truncated Monte Carlo Shapley \ No newline at end of file +*[TMCS]: Truncated Monte Carlo Shapley +*[WAD]: Weighted Accuracy Drop From bebfd9cca1eac1bab0212329920d5ab993014711 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Sat, 14 Oct 2023 14:31:35 +0200 Subject: [PATCH 43/43] [skip ci] More fixes --- docs/value/classwise-shapley.md | 4 +++- src/pydvl/value/shapley/classwise.py | 33 +++++++++++++++++----------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/docs/value/classwise-shapley.md b/docs/value/classwise-shapley.md index 4ae2c3f9f..a6911812a 100644 --- a/docs/value/classwise-shapley.md +++ b/docs/value/classwise-shapley.md @@ -42,7 +42,9 @@ the dataset. Like all other game-theoretic valuation methods, CWS requires a [Utility][pydvl.utils.utility.Utility] object constructed with model and dataset, with the peculiarity of requiring a specific - [ClasswiseScorer][pydvl.value.shapley.classwise.ClasswiseScorer]: + [ClasswiseScorer][pydvl.value.shapley.classwise.ClasswiseScorer]. The entry + point is the function + [compute_classwise_shapley_values][pydvl.value.shapley.classwise.compute_classwise_shapley_values]: ```python from pydvl.value import * diff --git a/src/pydvl/value/shapley/classwise.py b/src/pydvl/value/shapley/classwise.py index d03257120..438d953c8 100644 --- a/src/pydvl/value/shapley/classwise.py +++ b/src/pydvl/value/shapley/classwise.py @@ -13,9 +13,15 @@ $$ where $S_{y_i} \subseteq D_{y_i} \setminus \{i\}$ and $S_{-y_i} \subseteq -D_{-y_i}$. In practice, this quantity is estimated using Monte Carlo sampling of -the powerset and the set of index permutations. Applying these techniques -results in the estimator +D_{-y_i}$. + +!!! tip "Analysis of Class-wise Shapley" + For a detailed analysis of the method, with comparison to other valuation + techniques, please refer to the [main + documentation](../../../../../value/classwise-shapley). + +In practice, the quantity above is estimated using Monte Carlo sampling of +the powerset and the set of index permutations. This results in the estimator $$ v_u(i) = \frac{1}{K} \sum_k \frac{1}{L} \sum_l @@ -110,27 +116,28 @@ class ClasswiseScorer(Scorer): !!! warning Multi-class support Metrics must support multiple class labels if you intend to apply them to a multi-class problem. For instance, the metric 'accuracy' supports - multiple classes, but the metric 'f1' does not. For a two-class - classification problem, using 'f1_weighted' is essentially equivalent to - using 'accuracy'. + multiple classes, but the metric `f1` does not. For a two-class + classification problem, using `f1_weighted` is essentially equivalent to + using `accuracy`. Args: scoring: Name of the scoring function or a callable that can be passed to [Scorer][pydvl.utils.score.Scorer]. default: Score to use when a model fails to provide a number, e.g. when too little was used to train it, or errors arise. - range: Numerical range of the score function. Some Monte Carlo methods can - use this to estimate the number of samples required for a certain quality of - approximation. If not provided, it can be read from the `scoring` object - if it provides it, for instance if it was constructed with + range: Numerical range of the score function. Some Monte Carlo methods + can use this to estimate the number of samples required for a + certain quality of approximation. If not provided, it can be read + from the `scoring` object if it provides it, for instance if it was + constructed with [compose_score][pydvl.utils.score.compose_score]. in_class_discount_fn: Continuous, monotonic increasing function used to discount the in-class score. - out_of_class_discount_fn: Continuous, monotonic increasing function used to - discount the out-of-class score. + out_of_class_discount_fn: Continuous, monotonic increasing function used + to discount the out-of-class score. initial_label: Set initial label (for the first iteration) name: Name of the scorer. If not provided, the name of the inner scoring - function will be prefixed by 'classwise '. + function will be prefixed by `classwise `. !!! tip "New in version 0.7.1" """