From e05520051018f0e6f0d4fad4a4cc6caa3dd3fb54 Mon Sep 17 00:00:00 2001 From: "lukyanov_kirya@bk.ru" Date: Thu, 5 Dec 2024 20:32:33 +0300 Subject: [PATCH] add def full_pipeline_model_metrics_only --- experiments/attack_defense_metric_test.py | 92 ++++++++++--- src/attacks/metattack/meta_gradient_attack.py | 4 +- src/models_builder/attack_defense_manager.py | 122 ++++++++++++++++++ src/models_builder/gnn_models.py | 70 +++++++++- tests/defense_test.py | 11 +- 5 files changed, 268 insertions(+), 31 deletions(-) diff --git a/experiments/attack_defense_metric_test.py b/experiments/attack_defense_metric_test.py index 9655d5f..02f80a6 100644 --- a/experiments/attack_defense_metric_test.py +++ b/experiments/attack_defense_metric_test.py @@ -14,6 +14,15 @@ from src.base.datasets_processing import DatasetManager from src.models_builder.models_zoo import model_configs_zoo +for pack in [ + 'defense.GNNGuard.gnnguard', + 'defense.JaccardDefense.jaccard_def', +]: + try: + __import__(pack) + except ImportError: + print(f"Couldn't import Explainer from {pack}") + def attack_defense_metrics(): my_device = device('cuda' if torch.cuda.is_available() else 'cpu') @@ -61,12 +70,22 @@ def attack_defense_metrics(): gnn_model_manager.gnn.to(my_device) - random_poison_attack_config = ConfigPattern( - _class_name="RandomPoisonAttack", + # random_poison_attack_config = ConfigPattern( + # _class_name="RandomPoisonAttack", + # _import_path=POISON_ATTACK_PARAMETERS_PATH, + # _config_class="PoisonAttackConfig", + # _config_kwargs={ + # "n_edges_percent": 1.0, + # } + # ) + + metafull_poison_attack_config = ConfigPattern( + _class_name="MetaAttackFull", _import_path=POISON_ATTACK_PARAMETERS_PATH, _config_class="PoisonAttackConfig", _config_kwargs={ - "n_edges_percent": 1.0, + "num_nodes": dataset.dataset.x.shape[0], + "lambda": 0, } ) @@ -86,7 +105,7 @@ def attack_defense_metrics(): _import_path=POISON_DEFENSE_PARAMETERS_PATH, _config_class="PoisonDefenseConfig", _config_kwargs={ - "threshold": 0.05, + "threshold": 0.4, } ) @@ -95,7 +114,7 @@ def attack_defense_metrics(): _import_path=EVASION_ATTACK_PARAMETERS_PATH, _config_class="EvasionAttackConfig", _config_kwargs={ - "epsilon": 0.001 * 12, + "epsilon": 0.001 * 5, } ) @@ -104,12 +123,30 @@ def attack_defense_metrics(): _import_path=EVASION_DEFENSE_PARAMETERS_PATH, _config_class="EvasionDefenseConfig", _config_kwargs={ - "regularization_strength": 0.1 * 1000 + "regularization_strength": 0.1 * 500 + } + ) + + fgsm_evasion_attack_config1 = ConfigPattern( + _class_name="FGSM", + _import_path=EVASION_ATTACK_PARAMETERS_PATH, + _config_class="EvasionAttackConfig", + _config_kwargs={ + "epsilon": 0.01, + } + ) + at_evasion_defense_config = ConfigPattern( + _class_name="AdvTraining", + _import_path=EVASION_DEFENSE_PARAMETERS_PATH, + _config_class="EvasionDefenseConfig", + _config_kwargs={ + "attack_name": None, + "attack_config": fgsm_evasion_attack_config1 } ) - gnn_model_manager.set_poison_attacker(poison_attack_config=random_poison_attack_config) - gnn_model_manager.set_poison_defender(poison_defense_config=jaccard_poison_defense_config) + # gnn_model_manager.set_poison_attacker(poison_attack_config=metafull_poison_attack_config) + gnn_model_manager.set_poison_defender(poison_defense_config=gnnguard_poison_defense_config) gnn_model_manager.set_evasion_attacker(evasion_attack_config=fgsm_evasion_attack_config) gnn_model_manager.set_evasion_defender(evasion_defense_config=gradientregularization_evasion_defense_config) @@ -117,14 +154,14 @@ def attack_defense_metrics(): dataset.train_test_split() # try: - # # raise FileNotFoundError() - # gnn_model_manager.load_model_executor() - # dataset = gnn_model_manager.load_train_test_split(dataset) + # raise FileNotFoundError() + # # gnn_model_manager.load_model_executor() # except FileNotFoundError: # gnn_model_manager.epochs = gnn_model_manager.modification.epochs = 0 # train_test_split_path = gnn_model_manager.train_model(gen_dataset=dataset, steps=steps_epochs, # save_model_flag=save_model_flag, - # metrics=[Metric("F1", mask='train', average=None)]) + # metrics=[Metric("F1", mask='train', average=None), + # Metric("Accuracy", mask="train")]) # # if train_test_split_path is not None: # dataset.save_train_test_mask(train_test_split_path) @@ -135,10 +172,19 @@ def attack_defense_metrics(): # # warnings.warn("Training was successful") # + # # mask_loc = Metric.create_mask_by_target_list(y_true=dataset.labels, target_list=node_idxs) + # + # metric_loc = gnn_model_manager.evaluate_model( + # gen_dataset=dataset, metrics=[Metric("F1", mask='train', average='macro'), + # Metric("Accuracy", mask='train')], + # save_flag=True + # ) + # print("TRAIN", metric_loc) + # # metric_loc = gnn_model_manager.evaluate_model( # gen_dataset=dataset, metrics=[Metric("F1", mask='test', average='macro'), # Metric("Accuracy", mask='test')]) - # print(metric_loc) + # print("TEST", metric_loc) adm = FrameworkAttackDefenseManager( gen_dataset=copy.deepcopy(dataset), @@ -156,17 +202,25 @@ def attack_defense_metrics(): # metrics_attack=[AttackMetric("ASR")], # mask='test' # ) - adm.evasion_defense_pipeline( + # adm.evasion_defense_pipeline( + # steps=steps_epochs, + # save_model_flag=save_model_flag, + # metrics_attack=[AttackMetric("ASR"), AttackMetric("AuccAttackDiff"),], + # metrics_defense=[DefenseMetric("AuccDefenseCleanDiff"), DefenseMetric("AuccDefenseAttackDiff"), ], + # mask='test' + # ) + + adm.full_pipeline_model_metrics_only( + # steps=1, steps=steps_epochs, save_model_flag=save_model_flag, - metrics_attack=[AttackMetric("ASR"), AttackMetric("AuccAttackDiff"),], - metrics_defense=[DefenseMetric("AuccDefenseCleanDiff"), DefenseMetric("AuccDefenseAttackDiff"), ], - mask='test' + model_metrics=[Metric("Accuracy", mask="test")], + task="tttttt", ) if __name__ == '__main__': import random - random.seed(10) - attack_defense_metrics() \ No newline at end of file + # random.seed(10) + attack_defense_metrics() diff --git a/src/attacks/metattack/meta_gradient_attack.py b/src/attacks/metattack/meta_gradient_attack.py index e144a14..94c1e2f 100644 --- a/src/attacks/metattack/meta_gradient_attack.py +++ b/src/attacks/metattack/meta_gradient_attack.py @@ -216,7 +216,7 @@ def __init__(self, num_nodes=None, feature_shape=None, lambda_=0.5, train_iters= self.b_velocities = [] self.momentum = momentum - def attack(self, gen_dataset, attack_budget=10, ll_constraint=True, ll_cutoff=0.004): + def attack(self, gen_dataset, attack_budget=0.05, ll_constraint=True, ll_cutoff=0.004): super().attack(gen_dataset=gen_dataset) self.hidden_sizes = [16] # FIXME get from model architecture @@ -264,7 +264,7 @@ def attack(self, gen_dataset, attack_budget=10, ll_constraint=True, ll_cutoff=0. modified_adj = ori_adj modified_features = ori_features - for i in tqdm(range(attack_budget), desc="Perturbing graph"): + for i in tqdm(range(int(attack_budget*gen_dataset.dataset.data.edge_index.shape[1])), desc="Perturbing graph"): if self.attack_structure: modified_adj = self.get_modified_adj(ori_adj) diff --git a/src/models_builder/attack_defense_manager.py b/src/models_builder/attack_defense_manager.py index 2193623..8b37a20 100644 --- a/src/models_builder/attack_defense_manager.py +++ b/src/models_builder/attack_defense_manager.py @@ -77,6 +77,128 @@ def return_attack_defense_flags( self.gnn_manager.evasion_defense_flag = self.start_attack_defense_flag_state["evasion_defense"] self.gnn_manager.mi_defense_flag = self.start_attack_defense_flag_state["mi_defense"] + def full_pipeline_model_metrics_only( + self, + steps: int, + save_model_flag: bool = True, + model_metrics=None, + task: str = "tttttt", + ): + if model_metrics is None: + from models_builder.gnn_models import Metric + model_metrics = [Metric("F1", mask='train', average=None), + Metric("Accuracy", mask="train")] + + task = list(task) + if not self.available_attacks["poison"]: + task[0] = 'f' + if not self.available_defense["poison"]: + task[1] = 'f' + if not self.available_attacks["evasion"]: + task[2] = 'f' + if not self.available_attacks["mi"]: + task[3] = 'f' + if not self.available_defense["evasion"]: + task[4] = 'f' + if not self.available_defense["mi"]: + task[5] = 'f' + task = "".join(task) + self.run_experiments( + task=task, + steps=steps, + save_model_flag=save_model_flag, + model_metrics=model_metrics, + ) + + def run_experiments( + self, + steps: int, + save_model_flag: bool = True, + model_metrics=None, + task: str = 'ffffff', + flags=None, + position: int = 0, + ): + if flags is None: + flags = [] + + if position == len(task): + self.start( + flags=flags, + steps=steps, + save_model_flag=save_model_flag, + model_metrics=model_metrics, + ) + return + + if task[position] == 'f': + self.run_experiments( + steps=steps, + save_model_flag=save_model_flag, + model_metrics=model_metrics, + task=task, + flags=flags + [False], + position=position + 1 + ) + elif task[position] == 't': + self.run_experiments( + steps=steps, + save_model_flag=save_model_flag, + model_metrics=model_metrics, + task=task, + flags=flags + [False], + position=position + 1 + ) + self.run_experiments( + steps=steps, + save_model_flag=save_model_flag, + model_metrics=model_metrics, + task=task, + flags=flags + [True], + position=position + 1 + ) + + def start( + self, + steps: int, + save_model_flag: bool = True, + model_metrics=None, + flags: list = None, + ): + if flags is None: + flags = [False] * 6 + self.set_clear_model() + if flags[0]: + self.gnn_manager.poison_attack_flag = True + if flags[1]: + self.gnn_manager.poison_defense_flag = True + if flags[2]: + self.gnn_manager.evasion_attack_flag = True + if flags[3]: + self.gnn_manager.mi_attack_flag = True + if flags[4]: + self.gnn_manager.evasion_defense_flag = True + if flags[5]: + self.gnn_manager.mi_defense_flag = True + self.gnn_manager.epochs = self.gnn_manager.modification.epochs = 0 + from models_builder.gnn_models import Metric + train_test_split_path = self.gnn_manager.train_model( + gen_dataset=self.gen_dataset, steps=steps, + save_model_flag=save_model_flag, + metrics=[Metric("F1", mask='train', average=None), + Metric("Accuracy", mask="train")] + ) + + if train_test_split_path is not None: + self.gen_dataset.save_train_test_mask(train_test_split_path) + metric_loc = self.gnn_manager.evaluate_model( + gen_dataset=self.gen_dataset, + metrics=model_metrics, + save_flag=save_model_flag, + ) + if not save_model_flag: + print(f"Model metrics in a pipeline with task sequence {flags}: {metric_loc}") + def evasion_attack_pipeline( self, metrics_attack: List, diff --git a/src/models_builder/gnn_models.py b/src/models_builder/gnn_models.py index 9674224..e4c642f 100644 --- a/src/models_builder/gnn_models.py +++ b/src/models_builder/gnn_models.py @@ -1,5 +1,6 @@ import importlib.util import json +import os import random from math import ceil from pathlib import Path @@ -409,7 +410,7 @@ def set_poison_attacker( # device=device("cpu"), **poison_attack_kwargs ) - self.poison_attack_flag = True + self.poison_attack_flag = True if poison_attack_name != "EmptyPoisonAttacker" else False def set_evasion_attacker( self, @@ -449,7 +450,7 @@ def set_evasion_attacker( # device=device("cpu"), **evasion_attack_kwargs ) - self.evasion_attack_flag = True + self.evasion_attack_flag = True if evasion_attack_name != "EmptyEvasionAttacker" else False def set_mi_attacker( self, @@ -489,7 +490,7 @@ def set_mi_attacker( # device=device("cpu"), **mi_attack_kwargs ) - self.mi_attack_flag = True + self.mi_attack_flag = True if mi_attack_name != "EmptyMIAttacker" else False def set_poison_defender( self, @@ -529,7 +530,7 @@ def set_poison_defender( # device=device("cpu"), **poison_defense_kwargs ) - self.poison_defense_flag = True + self.poison_defense_flag = True if poison_defense_name != "EmptyPoisonDefender" else False def set_evasion_defender( self, @@ -569,7 +570,7 @@ def set_evasion_defender( # device=device("cpu"), **evasion_defense_kwargs ) - self.evasion_defense_flag = True + self.evasion_defense_flag = True if evasion_defense_name != "EmptyEvasionDefender" else False def set_mi_defender( self, @@ -612,7 +613,7 @@ def set_mi_defender( # device=device("cpu"), **mi_defense_kwargs ) - self.mi_defense_flag = True + self.mi_defense_flag = True if mi_defense_name != "EmptyMIDefender" else False @staticmethod def available_attacker( @@ -1228,7 +1229,8 @@ def run_model( def evaluate_model( self, gen_dataset: GeneralDataset, - metrics: Union[List[Metric], Metric] + metrics: Union[List[Metric], Metric], + save_flag: bool = False, ) -> dict: """ Compute metrics for a model result on a part of dataset specified by the metric mask. @@ -1270,8 +1272,62 @@ def evaluate_model( # metrics_values[mask][metric.name] = MetricManager.compute(metric, y_pred, y_true) if self.mi_attacker and self.mi_attack_flag: self.call_mi_attack() + if save_flag: + self.save_metrics(metrics_values=metrics_values) return metrics_values + def save_metrics( + self, + metrics_values: dict, + ) -> None: + dir_path = self.model_path_info() + path = dir_path / "model_metrics.txt" + + if not os.path.exists(path): + self.save_model_executor() + + char1 = 't' if self.poison_attack_flag else 'f' + char2 = 't' if self.poison_defense_flag else 'f' + char3 = 't' if self.evasion_defense_flag else 'f' + char4 = 't' if self.mi_defense_flag else 'f' + char5 = 't' if self.evasion_attack_flag else 'f' + char6 = 't' if self.mi_attack_flag else 'f' + task_info = "" + char1 + char2 + char3 + char4 + char5 + char6 + + def tensor_to_str(key): + if isinstance(key, torch.Tensor): + return str(key.tolist()) + return key + + def str_to_tensor(key): + if isinstance(key, list): + return torch.tensor(key, dtype=torch.bool) + return key + + def prepare_dict_for_json(d): + return {tensor_to_str(k): v for k, v in d.items()} + + def restore_dict_from_json(d): + return {str_to_tensor(k): v for k, v in d.items()} + + if os.path.exists(path): + with open(path, "r") as f: + file_dict = restore_dict_from_json(json.load(f)) + else: + file_dict = {} + + if task_info not in file_dict: + file_dict[task_info] = metrics_values + else: + for mask, metrics in metrics_values.items(): + for metric, value in metrics.items(): + if mask not in file_dict[task_info]: + file_dict[task_info][mask] = {} + file_dict[task_info][mask][metric] = value + + with open(path, "w") as f: + json.dump(prepare_dict_for_json(file_dict), f, indent=2) + def call_evasion_attack( self, gen_dataset: GeneralDataset, diff --git a/tests/defense_test.py b/tests/defense_test.py index 796befc..7868a02 100644 --- a/tests/defense_test.py +++ b/tests/defense_test.py @@ -1,3 +1,7 @@ +import collections.abc + +collections.Callable = collections.abc.Callable + import unittest import torch @@ -13,8 +17,7 @@ class DefenseTest(unittest.TestCase): def setUp(self): print('setup') - - from defense.poison_defense.GNNGuard_defense import GNNGuard_defense + # Init datasets # Single-Graph - Example self.dataset_sg_example, _, results_dataset_path_sg_example = DatasetManager.get_by_full_name( @@ -60,7 +63,9 @@ def test_gnnguard(self): _import_path=POISON_DEFENSE_PARAMETERS_PATH, _config_class="PoisonDefenseConfig", _config_kwargs={ - # "num_nodes": self.gen_dataset_sg_example.dataset.x.shape[0] # is there more fancy way? + "lr": 0.01, + "train_iters": 100, + # "model": gnn_model_manager.gnn } )