Merge remote-tracking branch 'origin/QAttack_fix' into sazonov_experi…

…mental

# Conflicts:
#	experiments/attack_defense_test.py
#	metainfo/evasion_attack_parameters.json
#	metainfo/evasion_defense_parameters.json

.github/workflows/linter.yaml

@@ -0,0 +1,42 @@
+name: Pylint
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          python -m pip install --upgrade --no-cache-dir pip setuptools
+          python -m pip install --exists-action=w --no-cache-dir -r requirements1.txt
+          python -m pip install --exists-action=w --no-cache-dir -r requirements2.txt
+          python -m pip install --exists-action=w --no-cache-dir -r docs/requirements.txt
+          python -m pip install --upgrade --upgrade-strategy eager --no-cache-dir .
+      - name: Set PYTHONPATH
+        run: |
+          echo "PYTHONPATH=$PYTHONPATH:$(pwd)/src" >> $GITHUB_ENV
+
+      - name: Run Pylint Models
+        run: pylint models_builder
+      - name: Run Pylint defense
+        run: pylint defense
+      - name: Run Pylint attacks
+        run: pylint attacks
+      - name: Run Pylint aux
+        run: pylint aux
+      - name: Run Pylint explainers
+        run: pylint explainers
+

experiments/attack_defense_test.py

@@ -10,11 +10,12 @@
 from src.aux.configs import ModelModificationConfig, ConfigPattern
 from src.base.datasets_processing import DatasetManager
 from src.models_builder.models_zoo import model_configs_zoo
+from attacks.QAttack import qattack
 
 
 def test_attack_defense():
-    # my_device = device('cuda' if is_available() else 'cpu')
-    my_device = device('cpu')
+
+    my_device = device('cuda' if torch.cuda.is_available() else 'cpu')
 
     full_name = None
 
@@ -103,18 +104,18 @@ def test_attack_defense():
     data = data.to(my_device)
 
     poison_attack_config = ConfigPattern(
-        _class_name="RandomPoisonAttack",
+        _class_name="MetaAttackFull",
         _import_path=POISON_ATTACK_PARAMETERS_PATH,
         _config_class="PoisonAttackConfig",
         _config_kwargs={
-            "n_edges_percent": 0.1,
+            "num_nodes": dataset.dataset.x.shape[0]
         }
     )
 
-    # poison_defense_config = ConfigPattern(
-    #     _class_name="BadRandomPoisonDefender",
-    #     _import_path=POISON_DEFENSE_PARAMETERS_PATH,
-    #     _config_class="PoisonDefenseConfig",
+    # poison_attack_config = ConfigPattern(
+    #     _class_name="RandomPoisonAttack",
+    #     _import_path=POISON_ATTACK_PARAMETERS_PATH,
+    #     _config_class="PoisonAttackConfig",
     #     _config_kwargs={
     #         "n_edges_percent": 0.1,
     #     }
@@ -136,17 +137,18 @@ def test_attack_defense():
         }
     )
     evasion_defense_config = ConfigPattern(
-        _class_name="GradientRegularizationDefender",
+        _class_name="AdvTraining",
         _import_path=EVASION_DEFENSE_PARAMETERS_PATH,
         _config_class="EvasionDefenseConfig",
         _config_kwargs={
-            "regularization_strength": 0.1 * 10
+            "attack_name": None,
+            "attack_config": evasion_attack_config # evasion_attack_config
         }
     )
 
-    gnn_model_manager.set_poison_attacker(poison_attack_config=poison_attack_config)
+    # gnn_model_manager.set_poison_attacker(poison_attack_config=poison_attack_config)
     # gnn_model_manager.set_poison_defender(poison_defense_config=poison_defense_config)
-    # gnn_model_manager.set_evasion_attacker(evasion_attack_config=evasion_attack_config)
+    gnn_model_manager.set_evasion_attacker(evasion_attack_config=evasion_attack_config)
     # gnn_model_manager.set_evasion_defender(evasion_defense_config=evasion_defense_config)
 
     warnings.warn("Start training")
@@ -175,7 +177,7 @@ def test_attack_defense():
     print(metric_loc)
 
 def test_meta():
-    from attacks.poison_attacks_collection.metattack import meta_gradient_attack
+    from attacks.metattack import meta_gradient_attack
     my_device = device('cpu')
     full_name = ("single-graph", "Planetoid", 'Cora')
 
@@ -459,10 +461,237 @@ def test_qattack():
     # print(f"info_before_evasion_attack: {info_before_evasion_attack}")
     # print(f"info_after_evasion_attack: {info_after_evasion_attack}")
 
+def test_jaccard():
+    from defense.JaccardDefense import jaccard_def
+    # my_device = device('cuda' if is_available() else 'cpu')
+    my_device = device('cpu')
+
+    full_name = None
+
+    # full_name = ("multiple-graphs", "TUDataset", 'MUTAG')
+    # full_name = ("single-graph", "custom", 'karate')
+    full_name = ("single-graph", "Planetoid", 'Cora')
+    # full_name = ("multiple-graphs", "TUDataset", 'PROTEINS')
+
+    dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
+        full_name=full_name,
+        dataset_ver_ind=0
+    )
+
+    # dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
+    #     full_name=("single-graph", "custom", "example",),
+    #     features={'attr': {'a': 'as_is', 'b': 'as_is'}},
+    #     labeling='threeClasses',
+    #     dataset_ver_ind=0
+    # )
+
+    # dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
+    #     # full_name=("single-graph", "vk_samples", "vk2-ff40-N100000-A.1612175945",),
+    #     full_name=("single-graph", "vk_samples", "vk2-ff20-N10000-A.1611943634",),
+    #     # full_name=("single-graph", "vk_samples", "vk2-ff20-N1000-U.1612273925",),
+    #     # features=('sex',),
+    #     features={'str_f': tuple(), 'str_g': None, 'attr': {
+    #         # "('personal', 'political')": 'one_hot',
+    #         # "('occupation', 'type')": 'one_hot', # Don't work now
+    #         # "('relation',)": 'one_hot',
+    #         # "('age',)": 'one_hot',
+    #         "('sex',)": 'one_hot',
+    #     }},
+    #     # features={'str_f': tuple(), 'str_g': None, 'attr': {'sex': 'one_hot', }},
+    #     labeling='sex1',
+    #     dataset_ver_ind=0
+    # )
+
+    # print(data.train_mask)
+
+    gnn = model_configs_zoo(dataset=dataset, model_name='gcn_gcn')
+    # gnn = model_configs_zoo(dataset=dataset, model_name='gcn_gcn_lin')
+    # gnn = model_configs_zoo(dataset=dataset, model_name='test_gnn')
+    # gnn = model_configs_zoo(dataset=dataset, model_name='gin_gin_gin_lin_lin')
+    # gnn = model_configs_zoo(dataset=dataset, model_name='gin_gin_gin_lin_lin_prot')
+
+    manager_config = ConfigPattern(
+        _config_class="ModelManagerConfig",
+        _config_kwargs={
+            "mask_features": [],
+            "optimizer": {
+                # "_config_class": "Config",
+                "_class_name": "Adam",
+                # "_import_path": OPTIMIZERS_PARAMETERS_PATH,
+                # "_class_import_info": ["torch.optim"],
+                "_config_kwargs": {},
+            }
+        }
+    )
+    # manager_config = ModelManagerConfig(**{
+    #         "mask_features": [],
+    #         "optimizer": {
+    #             # "_config_class": "Config",
+    #             "_class_name": "Adam",
+    #             # "_import_path": OPTIMIZERS_PARAMETERS_PATH,
+    #             # "_class_import_info": ["torch.optim"],
+    #             "_config_kwargs": {},
+    #         }
+    #     }
+    # )
+
+    # train_test_split = [0.8, 0.2]
+    # train_test_split = [0.6, 0.4]
+    steps_epochs = 200
+    gnn_model_manager = FrameworkGNNModelManager(
+        gnn=gnn,
+        dataset_path=results_dataset_path,
+        manager_config=manager_config,
+        modification=ModelModificationConfig(model_ver_ind=0, epochs=steps_epochs)
+    )
+
+    save_model_flag = False
+    # save_model_flag = True
+
+    # data.x = data.x.float()
+    gnn_model_manager.gnn.to(my_device)
+    data = data.to(my_device)
+
+    evasion_attack_config = ConfigPattern(
+        _class_name="FGSM",
+        _import_path=EVASION_ATTACK_PARAMETERS_PATH,
+        _config_class="EvasionAttackConfig",
+        _config_kwargs={
+            "epsilon": 0.007 * 1,
+        }
+    )
+    # evasion_defense_config = ConfigPattern(
+    #     _class_name="JaccardDefender",
+    #     _import_path=EVASION_DEFENSE_PARAMETERS_PATH,
+    #     _config_class="EvasionDefenseConfig",
+    #     _config_kwargs={
+    #     }
+    # )
+    poison_defense_config = ConfigPattern(
+        _class_name="JaccardDefender",
+        _import_path=POISON_DEFENSE_PARAMETERS_PATH,
+        _config_class="PoisonDefenseConfig",
+        _config_kwargs={
+        }
+    )
+
+    # gnn_model_manager.set_poison_attacker(poison_attack_config=poison_attack_config)
+    gnn_model_manager.set_poison_defender(poison_defense_config=poison_defense_config)
+    gnn_model_manager.set_evasion_attacker(evasion_attack_config=evasion_attack_config)
+    # gnn_model_manager.set_evasion_defender(evasion_defense_config=evasion_defense_config)
+
+    warnings.warn("Start training")
+    dataset.train_test_split()
+
+    try:
+        raise FileNotFoundError()
+        # gnn_model_manager.load_model_executor()
+    except FileNotFoundError:
+        gnn_model_manager.epochs = gnn_model_manager.modification.epochs = 0
+        train_test_split_path = gnn_model_manager.train_model(gen_dataset=dataset, steps=steps_epochs,
+                                                              save_model_flag=save_model_flag,
+                                                              metrics=[Metric("F1", mask='train', average=None),
+                                                                       Metric("Accuracy", mask="train")])
+
+        if train_test_split_path is not None:
+            dataset.save_train_test_mask(train_test_split_path)
+            train_mask, val_mask, test_mask, train_test_sizes = torch.load(train_test_split_path / 'train_test_split')[
+                                                                :]
+            dataset.train_mask, dataset.val_mask, dataset.test_mask = train_mask, val_mask, test_mask
+            data.percent_train_class, data.percent_test_class = train_test_sizes
+
+    warnings.warn("Training was successful")
+
+    metric_loc = gnn_model_manager.evaluate_model(
+        gen_dataset=dataset, metrics=[Metric("F1", mask='train', average='macro'),
+                                      Metric("Accuracy", mask='train')])
+    print("TRAIN", metric_loc)
+
+    metric_loc = gnn_model_manager.evaluate_model(
+        gen_dataset=dataset, metrics=[Metric("F1", mask='test', average='macro'),
+                                      Metric("Accuracy", mask='test')])
+    print("TEST", metric_loc)
+
+
+def test_adv_training():
+    from defense.evasion_defense import AdvTraining
+
+    my_device = device('cpu')
+    full_name = ("single-graph", "Planetoid", 'Cora')
+
+    dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
+        full_name=full_name,
+        dataset_ver_ind=0
+    )
+    gnn = model_configs_zoo(dataset=dataset, model_name='gcn_gcn')
+    manager_config = ConfigPattern(
+        _config_class="ModelManagerConfig",
+        _config_kwargs={
+            "mask_features": [],
+            "optimizer": {
+                # "_config_class": "Config",
+                "_class_name": "Adam",
+                # "_import_path": OPTIMIZERS_PARAMETERS_PATH,
+                # "_class_import_info": ["torch.optim"],
+                "_config_kwargs": {},
+            }
+        }
+    )
+    steps_epochs = 200
+    gnn_model_manager = FrameworkGNNModelManager(
+        gnn=gnn,
+        dataset_path=results_dataset_path,
+        manager_config=manager_config,
+        modification=ModelModificationConfig(model_ver_ind=0, epochs=steps_epochs)
+    )
+    save_model_flag = False
+    gnn_model_manager.gnn.to(my_device)
+    data = data.to(my_device)
+
+    evasion_defense_config = ConfigPattern(
+        _class_name="AdvTraining",
+        _import_path=EVASION_DEFENSE_PARAMETERS_PATH,
+        _config_class="EvasionDefenseConfig",
+        _config_kwargs={
+            # "num_nodes": dataset.dataset.x.shape[0]
+        }
+    )
+    from defense.evasion_defense import EvasionDefender
+    from src.aux.utils import all_subclasses
+    print([e.name for e in all_subclasses(EvasionDefender)])
+    gnn_model_manager.set_evasion_defender(evasion_defense_config=evasion_defense_config)
+
+    warnings.warn("Start training")
+    dataset.train_test_split(percent_train_class=0.1)
+
+    try:
+        raise FileNotFoundError()
+        # gnn_model_manager.load_model_executor()
+    except FileNotFoundError:
+        gnn_model_manager.epochs = gnn_model_manager.modification.epochs = 0
+        train_test_split_path = gnn_model_manager.train_model(gen_dataset=dataset, steps=steps_epochs,
+                                                              save_model_flag=save_model_flag,
+                                                              metrics=[Metric("F1", mask='train', average=None)])
+
+        if train_test_split_path is not None:
+            dataset.save_train_test_mask(train_test_split_path)
+            train_mask, val_mask, test_mask, train_test_sizes = torch.load(train_test_split_path / 'train_test_split')[
+                                                                :]
+            dataset.train_mask, dataset.val_mask, dataset.test_mask = train_mask, val_mask, test_mask
+            data.percent_train_class, data.percent_test_class = train_test_sizes
+
+    warnings.warn("Training was successful")
+
+    metric_loc = gnn_model_manager.evaluate_model(
+        gen_dataset=dataset, metrics=[Metric("F1", mask='test', average='macro'),
+                                      Metric("Accuracy", mask='test')])
+    print(metric_loc)
 
 if __name__ == '__main__':
-    #test_attack_defense()
-    # torch.manual_seed(5000)
-    # test_meta()
-    test_qattack()
-    # test_nettack_evasion()
+    import random
+    random.seed(10)
+    test_attack_defense()
+    torch.manual_seed(5000)
+    # test_adv_training()
+    # test_gnnguard()
+    # test_jaccard()

metainfo/evasion_attack_parameters.json

@@ -24,5 +24,5 @@
   "generations" : ["Generations", "int", 50, {"min": 0, "step": 1}, "Number of generations for genetic algorithm"],
   "prob_cross": ["Probability for crossover", "float", 0.5, {"min": 0, "max": 1, "step": 0.01}, "Probability of crossover between two genes"],
   "prob_mutate": ["Probability for mutation", "float", 0.02, {"min": 0, "max": 1, "step": 0.01}, "Probability of gene mutation"]
-}
+ }
 }

metainfo/evasion_defense_parameters.json

@@ -8,7 +8,7 @@
   "qbit": ["qbit", "int", 8, {"min": 1, "step": 1}, "?"]
  },
  "AdvTraining": {
-  "epsilon": ["epsilon", "float", 0.1, {"min": 0.001, "step": 0.005}]
+  "attack_name": ["attack_name", "str", "FGSM", {}, "?"]
  }
 }
 

metainfo/poison_defense_parameters.json

@@ -3,7 +3,14 @@
  },
  "BadRandomPoisonDefender": {
   "n_edges_percent": ["n_edges_percent", "float", 0.1, {"min": 0.0001, "step": 0.01}, "?"]
+ },
+ "GNNGuard": {
+    "lr": ["lr", "float", 0.01, {"min": 0.0001, "step": 0.005}, "?"],
+    "attention": ["attention", "bool", true, {}, "?"],
+    "drop": ["drop", "bool", true, {}, "?"]
+ },
+ "JaccardDefender": {
+  "threshold": ["Edge Threshold", "float", 0.35, {"min": 0, "max": 1, "step":  0.01}, "Jaccard index threshold for dropping edges"]
  }
-
 }
 

requirements2.txt

@@ -21,5 +21,6 @@ patsy
 packaging
 opt-einsum
 pandas==2.2.0
+pylint
 
 torch_geometric==2.3.1