diff --git a/Models/download_models.sh b/Models/download_models.sh new file mode 100755 index 0000000..3fd6754 --- /dev/null +++ b/Models/download_models.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +wget "https://bitbucket.org/EvgeniaAR/game-of-noise-pretrained-models/downloads/Speckle_Model.pth" +wget "https://bitbucket.org/EvgeniaAR/game-of-noise-pretrained-models/downloads/ANT_SIN_Model.pth" +wget "https://bitbucket.org/EvgeniaAR/game-of-noise-pretrained-models/downloads/Gauss_mult_Model.pth" +wget "https://bitbucket.org/EvgeniaAR/game-of-noise-pretrained-models/downloads/Gauss_sigma_0.5_Model.pth" +wget "https://bitbucket.org/EvgeniaAR/game-of-noise-pretrained-models/downloads/ANT_Model.pth" \ No newline at end of file diff --git a/Results/ANT-SIN_results.txt b/Results/ANT-SIN_results.txt new file mode 100644 index 0000000..e69de29 diff --git a/Results/ANT_results.txt b/Results/ANT_results.txt new file mode 100644 index 0000000..a4f37f1 --- /dev/null +++ b/Results/ANT_results.txt @@ -0,0 +1,33 @@ +ImageNet val: Top1 accuracy: 76.07, Top5 accuracy: 92.93 +Performance on ImageNet-C: + +Brightness: Top1 accuracy 69.30, Top5 accuracy: 69.30, CE: 54.38 + +Contrast: Top1 accuracy 40.31, Top5 accuracy: 40.31, CE: 69.96 + +Defocus Blur: Top1 accuracy 47.50, Top5 accuracy: 47.50, CE: 64.03 + +Elastic Transform: Top1 accuracy 49.30, Top5 accuracy: 49.30, CE: 78.48 + +Fog: Top1 accuracy 42.97, Top5 accuracy: 42.97, CE: 69.61 + +Frost: Top1 accuracy 46.95, Top5 accuracy: 46.95, CE: 64.18 + +Gaussian Noise: Top1 accuracy 65.47, Top5 accuracy: 65.47, CE: 38.96 + +Glass Blur: Top1 accuracy 38.98, Top5 accuracy: 38.98, CE: 73.84 + +Impulse Noise: Top1 accuracy 65.08, Top5 accuracy: 65.08, CE: 37.85 + +JPEG Compression: Top1 accuracy 62.11, Top5 accuracy: 62.11, CE: 62.47 + +Motion Blur: Top1 accuracy 42.34, Top5 accuracy: 42.34, CE: 73.36 + +Pixelate: Top1 accuracy 56.48, Top5 accuracy: 56.48, CE: 60.62 + +Shot Noise: Top1 accuracy 66.80, Top5 accuracy: 66.80, CE: 37.12 + +Snow: Top1 accuracy 35.86, Top5 accuracy: 35.86, CE: 74.00 + +Zoom Blur: Top1 accuracy 38.98, Top5 accuracy: 38.98, CE: 76.43 + diff --git a/Results/Gauss_mult_results.txt b/Results/Gauss_mult_results.txt new file mode 100644 index 0000000..1491da4 --- /dev/null +++ b/Results/Gauss_mult_results.txt @@ -0,0 +1,37 @@ +ImageNet val: Top1 accuracy: 76.13, Top5 accuracy: 92.93 +Performance on ImageNet-C: + +Brightness: Top1 accuracy 26.83, Top5 accuracy: 34.63, CE: 1.30 + +Contrast: Top1 accuracy 16.05, Top5 accuracy: 23.81, CE: 0.98 + +Defocus Blur: Top1 accuracy 16.89, Top5 accuracy: 25.81, CE: 1.01 + +Elastic Transform: Top1 accuracy 18.80, Top5 accuracy: 26.89, CE: 1.26 + +Fog: Top1 accuracy 17.79, Top5 accuracy: 26.77, CE: 1.00 + +Frost: Top1 accuracy 16.48, Top5 accuracy: 24.86, CE: 1.01 + +Gaussian Noise: Top1 accuracy 26.24, Top5 accuracy: 34.14, CE: 0.83 + +Glass Blur: Top1 accuracy 12.84, Top5 accuracy: 20.46, CE: 1.05 + +Impulse Noise: Top1 accuracy 24.93, Top5 accuracy: 33.24, CE: 0.81 + +JPEG Compression: Top1 accuracy 23.47, Top5 accuracy: 31.99, CE: 1.26 + +Motion Blur: Top1 accuracy 15.88, Top5 accuracy: 23.68, CE: 1.07 + +Pixelate: Top1 accuracy 19.79, Top5 accuracy: 28.26, CE: 1.12 + +Shot Noise: Top1 accuracy 25.65, Top5 accuracy: 33.76, CE: 0.83 + +Snow: Top1 accuracy 13.22, Top5 accuracy: 21.20, CE: 1.00 + +Zoom Blur: Top1 accuracy 14.58, Top5 accuracy: 23.04, CE: 1.07 + +Full ImageNet-C: Top1 accuracy 19.30, Top5 accuracy: 27.50, mCE: 1.04 + +ImageNet-C w/o Noises: : Top1 accuracy: Top1 accuracy 17.72, Top5 accuracy: 25.95 + diff --git a/Results/Gauss_sigma_0.5_results.txt b/Results/Gauss_sigma_0.5_results.txt new file mode 100644 index 0000000..ee295bb --- /dev/null +++ b/Results/Gauss_sigma_0.5_results.txt @@ -0,0 +1,37 @@ +ImageNet val: Top1 accuracy: 75.87, Top5 accuracy: 92.69 +Performance on ImageNet-C: + +Brightness: Top1 accuracy 26.60, Top5 accuracy: 34.47, CE: 1.30 + +Contrast: Top1 accuracy 15.41, Top5 accuracy: 23.03, CE: 0.99 + +Defocus Blur: Top1 accuracy 18.30, Top5 accuracy: 27.17, CE: 1.00 + +Elastic Transform: Top1 accuracy 19.44, Top5 accuracy: 27.54, CE: 1.25 + +Fog: Top1 accuracy 17.09, Top5 accuracy: 26.07, CE: 1.01 + +Frost: Top1 accuracy 17.15, Top5 accuracy: 25.58, CE: 1.00 + +Gaussian Noise: Top1 accuracy 23.21, Top5 accuracy: 31.70, CE: 0.87 + +Glass Blur: Top1 accuracy 14.82, Top5 accuracy: 22.83, CE: 1.03 + +Impulse Noise: Top1 accuracy 22.25, Top5 accuracy: 30.88, CE: 0.84 + +JPEG Compression: Top1 accuracy 24.26, Top5 accuracy: 32.59, CE: 1.25 + +Motion Blur: Top1 accuracy 17.09, Top5 accuracy: 25.09, CE: 1.05 + +Pixelate: Top1 accuracy 21.49, Top5 accuracy: 30.06, CE: 1.09 + +Shot Noise: Top1 accuracy 23.24, Top5 accuracy: 31.73, CE: 0.86 + +Snow: Top1 accuracy 13.84, Top5 accuracy: 21.85, CE: 0.99 + +Zoom Blur: Top1 accuracy 16.16, Top5 accuracy: 24.78, CE: 1.05 + +Full ImageNet-C: Top1 accuracy 19.36, Top5 accuracy: 27.69, mCE: 1.04 + +ImageNet-C w/o Noises: : Top1 accuracy: Top1 accuracy 18.47, Top5 accuracy: 26.76 + diff --git a/Results/Speckle_results.txt b/Results/Speckle_results.txt new file mode 100644 index 0000000..fcf382e --- /dev/null +++ b/Results/Speckle_results.txt @@ -0,0 +1,2 @@ +ImageNet val: Top1 accuracy: 75.83, Top5 accuracy: 92.78 + diff --git a/Results/clean_results.txt b/Results/clean_results.txt new file mode 100644 index 0000000..f2f410c --- /dev/null +++ b/Results/clean_results.txt @@ -0,0 +1,36 @@ +Performance on ImageNet-C: + +Brightness: Top1 accuracy 67.66, Top5 accuracy: 67.66, CE: 57.29 + +Contrast: Top1 accuracy 38.12, Top5 accuracy: 38.12, CE: 72.52 + +Defocus Blur: Top1 accuracy 41.17, Top5 accuracy: 41.17, CE: 71.75 + +Elastic Transform: Top1 accuracy 44.06, Top5 accuracy: 44.06, CE: 86.58 + +Fog: Top1 accuracy 44.14, Top5 accuracy: 44.14, CE: 68.18 + +Frost: Top1 accuracy 39.06, Top5 accuracy: 39.06, CE: 73.72 + +Gaussian Noise: Top1 accuracy 27.42, Top5 accuracy: 27.42, CE: 81.88 + +Glass Blur: Top1 accuracy 24.38, Top5 accuracy: 24.38, CE: 91.53 + +Impulse Noise: Top1 accuracy 23.05, Top5 accuracy: 23.05, CE: 83.41 + +JPEG Compression: Top1 accuracy 54.45, Top5 accuracy: 54.45, CE: 75.10 + +Motion Blur: Top1 accuracy 39.77, Top5 accuracy: 39.77, CE: 76.64 + +Pixelate: Top1 accuracy 45.70, Top5 accuracy: 45.70, CE: 75.64 + +Shot Noise: Top1 accuracy 27.89, Top5 accuracy: 27.89, CE: 80.62 + +Snow: Top1 accuracy 33.44, Top5 accuracy: 33.44, CE: 76.79 + +Zoom Blur: Top1 accuracy 38.05, Top5 accuracy: 38.05, CE: 77.60 + +Full ImageNet-C: Top1 accuracy 39.22, Top5 accuracy: 59.11, mCE: 76.62 + +ImageNet-C w/o Noises: : Top1 accuracy: Top1 accuracy 42.50, Top5 accuracy: 63.04 + diff --git a/main.py b/main.py index 092499b..e020555 100644 --- a/main.py +++ b/main.py @@ -3,25 +3,25 @@ from utils import * parser = argparse.ArgumentParser(description='Evaluation of Models') -parser.add_argument('--model_name', default='ANT-SIN', type=str, +parser.add_argument('--model_name', default='clean', type=str, help='which model should be evaluated') -parser.add_argument('--imagenetc-path', type=str, +parser.add_argument('--imagenetc-path', metavar='DIR', default='./data/ImageNet-C/imagenet-c/') parser.add_argument('--datadir-clean', metavar='DIR', - default='./data/ImageNet/' , help='path to dataset') + default='./data/IN/raw-data/', help='path to dataset') parser.add_argument('-j', '--workers', default=30, type=int, metavar='N', help='number of data loading workers (default: 30)') parser.add_argument('-tb', '--test-batch-size', default=256, type=int, metavar='N', help='mini-batch size (default: 256)') -def main(): +def main(): args = parser.parse_args() model = load_model(args.model_name) - - valdir = osp.join(args.datadir_clean, 'val') + + valdir = osp.join(args.datadir_clean, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) val_loader = torch.utils.data.DataLoader( @@ -33,33 +33,25 @@ def main(): ])), batch_size=args.test_batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) - - - in_c_data_loaders = get_IN_C_data_loaders(args) - args.IN_C_Results = dict() - for name, data_loader in in_c_data_loaders.items(): - args.IN_C_Results[name] = [[0]] * 7 - - + + in_c_data_loaders = get_in_c_data_loaders(args) + # evaluate print("Start evaluation for model {}".format(args.model_name)) outfile_name = './Results/' + args.model_name + '_results.txt' file = open(outfile_name, 'w') args.file = file - - acc1, acc5 = validate(val_loader, model, args) - print(acc1) - file.write("Top1 accuracy on ImageNet val: {0:.2f}\n".format(acc1.item())) - file.write("Top5 accuracy on ImageNet val: {0:.2f}\n".format(acc5.item())) - print("Top1 accuracy on ImageNet val: {0:.2f}".format(acc1.item())) - print("Top5 accuracy on ImageNet val: {0:.2f}".format(acc5.item())) - + + acc1, acc5 = validate(val_loader, model) + print("ImageNet val: Top1 accuracy: {0:.2f}, Top5 accuracy: {1:.2f}\n".format(acc1.item(), acc5.item()), file=file) + accuracy_on_imagenet_c(in_c_data_loaders, model, args) - + file.close() - + return - + + if __name__ == '__main__': main() diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..8a3f714 --- /dev/null +++ b/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +python3 main.py --model_name clean +python3 main.py --model_name ANT-SIN +python3 main.py --model_name ANT +python3 main.py --model_name Speckle +python3 main.py --model_name Gauss_mult +python3 main.py --model_name Gauss_sigma_0.5 diff --git a/utils.py b/utils.py index 551376a..523e3a7 100644 --- a/utils.py +++ b/utils.py @@ -1,16 +1,16 @@ import numpy as np import torch -import torch.nn as nn from torchvision import datasets, transforms import torchvision.models as models import os.path as osp +from torch.utils import model_zoo -def get_IN_C_data_loaders(args): +def get_in_c_data_loaders(args): """Returns data loaders for all ImageNet-C corruptions""" - n_worker=30 + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225]) + std=[0.229, 0.224, 0.225]) data_loaders_names = \ {'Brightness': 'brightness', 'Contrast': 'contrast', @@ -27,23 +27,21 @@ def get_IN_C_data_loaders(args): 'Shot Noise': 'shot_noise', 'Snow': 'snow', 'Zoom Blur': 'zoom_blur'} - data_loaders = {} + data_loaders = {} for name, path in data_loaders_names.items(): data_loaders[name] = {} for severity in range(1, 6): - dset = datasets.ImageFolder(osp.join(args.imagenetc_path, path, - str(severity)), transforms.Compose([ - transforms.CenterCrop(224), - transforms.ToTensor(), - normalize, - ])) + dset = datasets.ImageFolder(osp.join(args.imagenetc_path, path, str(severity)), + transforms.Compose([transforms.CenterCrop(224), + transforms.ToTensor(), normalize, ])) data_loaders[name][str(severity)] = torch.utils.data.DataLoader( - dset, batch_size=args.test_batch_size, shuffle=True, num_workers=n_worker) + dset, batch_size=args.test_batch_size, shuffle=True, num_workers=args.workers) return data_loaders def get_accuracy(output, target, topk=(1,)): """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): maxk = max(topk) batch_size = target.size(0) @@ -60,99 +58,156 @@ def get_accuracy(output, target, topk=(1,)): def accuracy_on_imagenet_c(data_loaders, model, args): - """Computes the accuracy on ImageNet-C""" + """Computes model accuracy and mCE on ImageNet-C""" + + print("Performance on ImageNet-C:\n", file=args.file) model.eval() - args.file.write("\n Performance on ImageNet-C:\n") + ce_alexnet = get_ce_alexnet() + with torch.no_grad(): - - all_accuracies = [] - all_accuracies_wo_noises = [] - all_accuracies_top5 = [] - all_accuracies_wo_noises_top5 = [] - + + top1_in_c = AverageMeter('Acc_IN_C@1', ':6.2f') + top5_in_c = AverageMeter('Acc_IN_C@5', ':6.2f') + top1_in_c_wo_noises = AverageMeter('Acc_IN_C_wo_Noises@1', ':6.2f') + top5_in_c_wo_noises = AverageMeter('Acc_IN_C_wo_Noises@5', ':6.2f') + mce, counter = 0, 0 + for name, data_loader in data_loaders.items(): - n_correct, n_correct_top5, n_total = 0, 0, 0 - n_correct_wo_noises, n_correct_wo_noises_top5, n_total_wo_noises = 0, 0, 0 - - for severity, loader in data_loader.items(): - n_correct_sev, n_total_sev = 0, 0 + top1_tmp = AverageMeter('Acc_tmp@1', ':6.2f') + for severity, loader in data_loader.items(): + top1_sev_tmp = AverageMeter('Acc_sev_tmp@1', ':6.2f') + top5_sev_tmp = AverageMeter('Acc_sev_tmp@5', ':6.2f') + for data, labels in loader: data, labels = data.cuda(), labels.cuda() logits = model(data) - acc_tmp = get_accuracy(logits, labels, (1, 5)) - n_correct += acc_tmp[0] - n_correct_sev += acc_tmp[0] - n_correct_top5 += acc_tmp[1] - n_total += float(data.shape[0]) - n_total_sev += float(data.shape[0]) + acc1, acc5 = get_accuracy(logits, labels, (1, 5)) + + top1_in_c.update(acc1[0], data.size(0)) + top5_in_c.update(acc5[0], data.size(0)) + top1_sev_tmp.update(acc1[0], data.size(0)) + top5_sev_tmp.update(acc5[0], data.size(0)) + top1_tmp.update(acc1[0], data.size(0)) + if name not in ['Gaussian Noise', 'Shot Noise', 'Impulse Noise']: - n_correct_wo_noises += acc_tmp[0] - n_correct_wo_noises_top5 += acc_tmp[1] - n_total_wo_noises += float(data.shape[0]) - break - - args.IN_C_Results[name][int(severity)+1] = n_correct_sev / n_total_sev - all_accuracies.append(100 * n_correct / n_total) - all_accuracies_top5.append(100*n_correct_top5 / n_total) - if name not in ['Gaussian Noise', 'Shot Noise', 'Impulse Noise']: - all_accuracies_wo_noises.append(100 * n_correct_wo_noises / n_total_wo_noises) - all_accuracies_wo_noises_top5.append(100*n_correct_wo_noises_top5 / n_total_wo_noises) - accuracy = 100 * n_correct / n_total - accuracy_top5 = 100 * n_correct_top5 / n_total - args.IN_C_Results[name][0] = accuracy.item() - args.IN_C_Results[name][1] = accuracy_top5.item() - + top1_in_c_wo_noises.update(acc1[0], data.size(0)) + top5_in_c_wo_noises.update(acc5[0], data.size(0)) + + # get Corruption Error CE: + CE = get_mce_from_accuracy(top1_tmp.avg.item(), ce_alexnet[name]) + mce += CE + counter += 1 + # Logging: - print("Top1 accuracy on {0}: {1:.2f}".format(name, accuracy.item())) - print("Top5 accuracy on {0}: {1:.2f}".format(name, accuracy_top5.item())) - args.file.write("Top1 accuracy on {0}: {1:.2f}, ".format(name, accuracy.item())) - args.file.write("Top5 accuracy on {0}: {1:.2f}".format(name, accuracy_top5.item())) - print("Top1 accuracy on full ImageNet-C: {0:.2f}, ".format(np.mean(all_accuracies))) - print("Top5 accuracy on full ImageNet-C: {0:.2f}".format(np.mean(all_accuracies_top5))) - print("Top1 accuracy on ImageNet-C w/o Noises: {0:.2f}, ".format(np.mean(all_accuracies_wo_noises))) - print("Top5 accuracy on ImageNet-C w/o Noises: {0:.2f}, ".format(np.mean(all_accuracies_wo_noises_top5))) - args.file.write("Top1 accuracy on full ImageNet-C: {0:.2f}, ".format(np.mean(all_accuracies))) - args.file.write("Top5 accuracy on full ImageNet-C: {0:.2f}\n".format(np.mean(all_accuracies_top5))) - args.file.write("Top1 accuracy on ImageNet-C w/o Noises: {0:.2f}, ".format(np.mean(all_accuracies_wo_noises))) - args.file.write("Top5 accuracy on ImageNet-C w/o Noises: {0:.2f}\n, ".format(np.mean(all_accuracies_wo_noises_top5))) - - outfile_name_IN_C_accuracy = './Results/' + args.model_name + '_IN_C_Results_resnet_50.npy' - np.save(outfile_name_IN_C_accuracy, args.IN_C_Results) - + print("{0}: Top1 accuracy {1:.2f}, Top5 accuracy: {2:.2f}, CE: {3:.2f}\n".format( + name, top1_tmp.avg.item(), top1_tmp.avg.item(), 100. * CE), file=args.file) + + mce /= counter + print("Full ImageNet-C: Top1 accuracy {0:.2f}, Top5 accuracy: {1:.2f}, mCE: {2:.2f}\n".format( + top1_in_c.avg.item(), + top5_in_c.avg.item(), + mce * 100.), file=args.file) + print("ImageNet-C w/o Noises: : Top1 accuracy: Top1 accuracy {0:.2f}, Top5 accuracy: {1:.2f}\n".format( + top1_in_c_wo_noises.avg.item(), + top5_in_c_wo_noises.avg.item()), file=args.file) + return - - -def validate(val_loader, model, args): + + +class AverageMeter(object): + """Computes and stores the average and current value""" + + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + +def validate(val_loader, model): """Computes accuracy on ImageNet val""" + top1 = AverageMeter('Acc@1', ':6.2f') + top5 = AverageMeter('Acc@5', ':6.2f') # switch to evaluate mode model.eval() - criterion = nn.CrossEntropyLoss().cuda() with torch.no_grad(): - for i, (images, target) in enumerate(val_loader): + for images, target in val_loader: images, target = images.cuda(), target.cuda() output = model(images) - loss = criterion(output, target) # measure accuracy and record loss acc1, acc5 = get_accuracy(output, target, topk=(1, 5)) - break + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) - return acc1, acc5 + return top1.avg, top5.avg def load_model(model_name): """loads robust model specified by modelname""" - + model = models.resnet50(pretrained='True') - - model_urls = { - 'ANT-SIN': './Models/ANT_SIN_Model_new.pth', + + model_paths = { + 'ANT-SIN': './Models/ANT_SIN_Model.pth', + 'ANT': './Models/ANT_Model.pth', + 'Speckle': './Models/Speckle_Model.pth', + 'Gauss_mult': './Models/Gauss_mult_Model.pth', + 'Gauss_sigma_0.5': './Models/Gauss_sigma_0.5_Model.pth', } - model_dict = torch.load(model_urls[model_name]) - model.load_state_dict(model_dict['model_state_dict']) + if not model_name == 'clean': + checkpoint = torch.load(model_paths[model_name]) + model.load_state_dict(checkpoint['model_state_dict']) model = model.eval().cuda() - + return model + + +def get_mce_from_accuracy(accuracy, error_AlexNet): + """Computes mean Corruption Error from accuracy""" + + error = 100. - accuracy + ce = error / (error_AlexNet * 100.) + + return ce + + +def get_ce_alexnet(): + """Returns Corruption Error values for AlexNet""" + + ce_alexnet = dict() + ce_alexnet['Gaussian Noise'] = 0.886428 + ce_alexnet['Shot Noise'] = 0.894468 + ce_alexnet['Impulse Noise'] = 0.922640 + ce_alexnet['Defocus Blur'] = 0.819880 + ce_alexnet['Glass Blur'] = 0.826268 + ce_alexnet['Motion Blur'] = 0.785948 + ce_alexnet['Zoom Blur'] = 0.798360 + ce_alexnet['Snow'] = 0.866816 + ce_alexnet['Frost'] = 0.826572 + ce_alexnet['Fog'] = 0.819324 + ce_alexnet['Brightness'] = 0.564592 + ce_alexnet['Contrast'] = 0.853204 + ce_alexnet['Elastic Transform'] = 0.646056 + ce_alexnet['Pixelate'] = 0.717840 + ce_alexnet['JPEG Compression'] = 0.606500 + + return ce_alexnet \ No newline at end of file