bug fixes, download models script

Models/download_models.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+wget "https://bitbucket.org/EvgeniaAR/game-of-noise-pretrained-models/downloads/Speckle_Model.pth"
+wget "https://bitbucket.org/EvgeniaAR/game-of-noise-pretrained-models/downloads/ANT_SIN_Model.pth"
+wget "https://bitbucket.org/EvgeniaAR/game-of-noise-pretrained-models/downloads/Gauss_mult_Model.pth"
+wget "https://bitbucket.org/EvgeniaAR/game-of-noise-pretrained-models/downloads/Gauss_sigma_0.5_Model.pth"
+wget "https://bitbucket.org/EvgeniaAR/game-of-noise-pretrained-models/downloads/ANT_Model.pth"

Results/ANT_results.txt

@@ -0,0 +1,33 @@
+ImageNet val: Top1 accuracy: 76.07, Top5 accuracy: 92.93
+Performance on ImageNet-C:
+
+Brightness: Top1 accuracy 69.30, Top5 accuracy: 69.30, CE: 54.38
+
+Contrast: Top1 accuracy 40.31, Top5 accuracy: 40.31, CE: 69.96
+
+Defocus Blur: Top1 accuracy 47.50, Top5 accuracy: 47.50, CE: 64.03
+
+Elastic Transform: Top1 accuracy 49.30, Top5 accuracy: 49.30, CE: 78.48
+
+Fog: Top1 accuracy 42.97, Top5 accuracy: 42.97, CE: 69.61
+
+Frost: Top1 accuracy 46.95, Top5 accuracy: 46.95, CE: 64.18
+
+Gaussian Noise: Top1 accuracy 65.47, Top5 accuracy: 65.47, CE: 38.96
+
+Glass Blur: Top1 accuracy 38.98, Top5 accuracy: 38.98, CE: 73.84
+
+Impulse Noise: Top1 accuracy 65.08, Top5 accuracy: 65.08, CE: 37.85
+
+JPEG Compression: Top1 accuracy 62.11, Top5 accuracy: 62.11, CE: 62.47
+
+Motion Blur: Top1 accuracy 42.34, Top5 accuracy: 42.34, CE: 73.36
+
+Pixelate: Top1 accuracy 56.48, Top5 accuracy: 56.48, CE: 60.62
+
+Shot Noise: Top1 accuracy 66.80, Top5 accuracy: 66.80, CE: 37.12
+
+Snow: Top1 accuracy 35.86, Top5 accuracy: 35.86, CE: 74.00
+
+Zoom Blur: Top1 accuracy 38.98, Top5 accuracy: 38.98, CE: 76.43
+

Results/Gauss_mult_results.txt

@@ -0,0 +1,37 @@
+ImageNet val: Top1 accuracy: 76.13, Top5 accuracy: 92.93
+Performance on ImageNet-C:
+
+Brightness: Top1 accuracy 26.83, Top5 accuracy: 34.63, CE: 1.30
+
+Contrast: Top1 accuracy 16.05, Top5 accuracy: 23.81, CE: 0.98
+
+Defocus Blur: Top1 accuracy 16.89, Top5 accuracy: 25.81, CE: 1.01
+
+Elastic Transform: Top1 accuracy 18.80, Top5 accuracy: 26.89, CE: 1.26
+
+Fog: Top1 accuracy 17.79, Top5 accuracy: 26.77, CE: 1.00
+
+Frost: Top1 accuracy 16.48, Top5 accuracy: 24.86, CE: 1.01
+
+Gaussian Noise: Top1 accuracy 26.24, Top5 accuracy: 34.14, CE: 0.83
+
+Glass Blur: Top1 accuracy 12.84, Top5 accuracy: 20.46, CE: 1.05
+
+Impulse Noise: Top1 accuracy 24.93, Top5 accuracy: 33.24, CE: 0.81
+
+JPEG Compression: Top1 accuracy 23.47, Top5 accuracy: 31.99, CE: 1.26
+
+Motion Blur: Top1 accuracy 15.88, Top5 accuracy: 23.68, CE: 1.07
+
+Pixelate: Top1 accuracy 19.79, Top5 accuracy: 28.26, CE: 1.12
+
+Shot Noise: Top1 accuracy 25.65, Top5 accuracy: 33.76, CE: 0.83
+
+Snow: Top1 accuracy 13.22, Top5 accuracy: 21.20, CE: 1.00
+
+Zoom Blur: Top1 accuracy 14.58, Top5 accuracy: 23.04, CE: 1.07
+
+Full ImageNet-C: Top1 accuracy 19.30, Top5 accuracy: 27.50, mCE: 1.04
+
+ImageNet-C w/o Noises: : Top1 accuracy: Top1 accuracy 17.72, Top5 accuracy: 25.95
+

Results/Gauss_sigma_0.5_results.txt

@@ -0,0 +1,37 @@
+ImageNet val: Top1 accuracy: 75.87, Top5 accuracy: 92.69
+Performance on ImageNet-C:
+
+Brightness: Top1 accuracy 26.60, Top5 accuracy: 34.47, CE: 1.30
+
+Contrast: Top1 accuracy 15.41, Top5 accuracy: 23.03, CE: 0.99
+
+Defocus Blur: Top1 accuracy 18.30, Top5 accuracy: 27.17, CE: 1.00
+
+Elastic Transform: Top1 accuracy 19.44, Top5 accuracy: 27.54, CE: 1.25
+
+Fog: Top1 accuracy 17.09, Top5 accuracy: 26.07, CE: 1.01
+
+Frost: Top1 accuracy 17.15, Top5 accuracy: 25.58, CE: 1.00
+
+Gaussian Noise: Top1 accuracy 23.21, Top5 accuracy: 31.70, CE: 0.87
+
+Glass Blur: Top1 accuracy 14.82, Top5 accuracy: 22.83, CE: 1.03
+
+Impulse Noise: Top1 accuracy 22.25, Top5 accuracy: 30.88, CE: 0.84
+
+JPEG Compression: Top1 accuracy 24.26, Top5 accuracy: 32.59, CE: 1.25
+
+Motion Blur: Top1 accuracy 17.09, Top5 accuracy: 25.09, CE: 1.05
+
+Pixelate: Top1 accuracy 21.49, Top5 accuracy: 30.06, CE: 1.09
+
+Shot Noise: Top1 accuracy 23.24, Top5 accuracy: 31.73, CE: 0.86
+
+Snow: Top1 accuracy 13.84, Top5 accuracy: 21.85, CE: 0.99
+
+Zoom Blur: Top1 accuracy 16.16, Top5 accuracy: 24.78, CE: 1.05
+
+Full ImageNet-C: Top1 accuracy 19.36, Top5 accuracy: 27.69, mCE: 1.04
+
+ImageNet-C w/o Noises: : Top1 accuracy: Top1 accuracy 18.47, Top5 accuracy: 26.76
+

Results/Speckle_results.txt

@@ -0,0 +1,2 @@
+ImageNet val: Top1 accuracy: 75.83, Top5 accuracy: 92.78
+

Results/clean_results.txt

@@ -0,0 +1,36 @@
+Performance on ImageNet-C:
+
+Brightness: Top1 accuracy 67.66, Top5 accuracy: 67.66, CE: 57.29
+
+Contrast: Top1 accuracy 38.12, Top5 accuracy: 38.12, CE: 72.52
+
+Defocus Blur: Top1 accuracy 41.17, Top5 accuracy: 41.17, CE: 71.75
+
+Elastic Transform: Top1 accuracy 44.06, Top5 accuracy: 44.06, CE: 86.58
+
+Fog: Top1 accuracy 44.14, Top5 accuracy: 44.14, CE: 68.18
+
+Frost: Top1 accuracy 39.06, Top5 accuracy: 39.06, CE: 73.72
+
+Gaussian Noise: Top1 accuracy 27.42, Top5 accuracy: 27.42, CE: 81.88
+
+Glass Blur: Top1 accuracy 24.38, Top5 accuracy: 24.38, CE: 91.53
+
+Impulse Noise: Top1 accuracy 23.05, Top5 accuracy: 23.05, CE: 83.41
+
+JPEG Compression: Top1 accuracy 54.45, Top5 accuracy: 54.45, CE: 75.10
+
+Motion Blur: Top1 accuracy 39.77, Top5 accuracy: 39.77, CE: 76.64
+
+Pixelate: Top1 accuracy 45.70, Top5 accuracy: 45.70, CE: 75.64
+
+Shot Noise: Top1 accuracy 27.89, Top5 accuracy: 27.89, CE: 80.62
+
+Snow: Top1 accuracy 33.44, Top5 accuracy: 33.44, CE: 76.79
+
+Zoom Blur: Top1 accuracy 38.05, Top5 accuracy: 38.05, CE: 77.60
+
+Full ImageNet-C: Top1 accuracy 39.22, Top5 accuracy: 59.11, mCE: 76.62
+
+ImageNet-C w/o Noises: : Top1 accuracy: Top1 accuracy 42.50, Top5 accuracy: 63.04
+

main.py

@@ -3,25 +3,25 @@
 from utils import *
 
 parser = argparse.ArgumentParser(description='Evaluation of Models')
-parser.add_argument('--model_name', default='ANT-SIN', type=str,
+parser.add_argument('--model_name', default='clean', type=str,
                     help='which model should be evaluated')
-parser.add_argument('--imagenetc-path', type=str,
+parser.add_argument('--imagenetc-path', metavar='DIR',
                     default='./data/ImageNet-C/imagenet-c/')
 parser.add_argument('--datadir-clean', metavar='DIR',
-                    default='./data/ImageNet/' , help='path to dataset')
+                    default='./data/IN/raw-data/', help='path to dataset')
 parser.add_argument('-j', '--workers', default=30, type=int, metavar='N',
                     help='number of data loading workers (default: 30)')
 parser.add_argument('-tb', '--test-batch-size', default=256, type=int,
                     metavar='N',
                     help='mini-batch size (default: 256)')
 
-def main():
 
+def main():
     args = parser.parse_args()
 
     model = load_model(args.model_name)
-    
-    valdir   = osp.join(args.datadir_clean, 'val')
+
+    valdir = osp.join(args.datadir_clean, 'val')
     normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                      std=[0.229, 0.224, 0.225])
     val_loader = torch.utils.data.DataLoader(
@@ -33,33 +33,25 @@ def main():
         ])),
         batch_size=args.test_batch_size, shuffle=False,
         num_workers=args.workers, pin_memory=True)
-
-
-    in_c_data_loaders = get_IN_C_data_loaders(args)
-    args.IN_C_Results = dict()
-    for name, data_loader in in_c_data_loaders.items():
-        args.IN_C_Results[name] = [[0]] * 7
-
-
+
+    in_c_data_loaders = get_in_c_data_loaders(args)
+
     # evaluate
     print("Start evaluation for model {}".format(args.model_name))
     outfile_name = './Results/' + args.model_name + '_results.txt'
     file = open(outfile_name, 'w')
     args.file = file
-
-    acc1, acc5 = validate(val_loader, model, args)
-    print(acc1)
-    file.write("Top1 accuracy on ImageNet val: {0:.2f}\n".format(acc1.item()))
-    file.write("Top5 accuracy on ImageNet val: {0:.2f}\n".format(acc5.item()))
-    print("Top1 accuracy on ImageNet val: {0:.2f}".format(acc1.item()))
-    print("Top5 accuracy on ImageNet val: {0:.2f}".format(acc5.item()))
-
+
+    acc1, acc5 = validate(val_loader, model)
+    print("ImageNet val: Top1 accuracy: {0:.2f}, Top5 accuracy: {1:.2f}\n".format(acc1.item(), acc5.item()), file=file)
+
     accuracy_on_imagenet_c(in_c_data_loaders, model, args)
-    
+
     file.close()
-    
+
     return
-
+
+
 if __name__ == '__main__':
     main()
 

run.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+python3 main.py --model_name clean
+python3 main.py --model_name ANT-SIN
+python3 main.py --model_name ANT
+python3 main.py --model_name Speckle
+python3 main.py --model_name Gauss_mult
+python3 main.py --model_name Gauss_sigma_0.5