-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcalib_summary.py
123 lines (93 loc) · 6.04 KB
/
calib_summary.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
import torch
from matplotlib import pyplot as plt
import numpy as np
from torch.nn.functional import softmax
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
from metrics import expected_calibration_error, maximum_calibration_error, AdaptiveECELoss
from utils import reliability_curve
import pandas as pd
#import seaborn as sns
from sklearn.preprocessing import MinMaxScaler
from tqdm import tqdm
from sklearn.metrics import accuracy_score
import os
import sys
def post_process(dataset_type, adapter_type, nshots):
target_path = '/export/livia/home/vision/Bmurugesan/CLIPCalib/output/FINAL/{}/ZS/vitb16/target.pt'.format(dataset_type)
#zs_path = '/export/livia/home/vision/Bmurugesan/CLIPCalib/output/FINAL/{}/ZS/vitb16/logits.pt'.format(dataset_type)
#adapt_path = '/export/livia/home/vision/Bmurugesan/CLIPCalib/output/FINAL/{}/ADAPTER/vitb16/SGD_lr1e-1_B256_ep300_{}Init_noneConstraint_{}shots/seed1/logits.pt'.format(dataset_type, adapter_type, nshots)
#adaptpre_path ='/export/livia/home/vision/Bmurugesan/CLIPCalib/output/FINAL/{}/ADAPTER_ZSNORM/vitb16/SGD_lr1e-1_B256_ep300_{}Init_noneConstraint_{}shots/seed1/logits.pt'.format(dataset_type, adapter_type, nshots)
#adaptpen_path = '/export/livia/home/vision/Bmurugesan/CLIPCalib/output/FINAL/{}/ADAPTER_ZSPEN/vitb16/SGD_lr1e-1_B256_ep300_{}Init_noneConstraint_{}shots/seed1/logits.pt'.format(dataset_type, adapter_type, nshots)
zs_path = '/export/livia/home/vision/Bmurugesan/CLIPCalib/output/FINAL/{}/ZS/rn50/logits.pt'.format(dataset_type)
#adapt_path = '/export/livia/home/vision/Bmurugesan/CLIPCalib/output/FINAL/{}/ADAPTER/rn50/SGD_lr1e-1_B256_ep300_{}Init_noneConstraint_{}shots/seed1/logits.pt'.format(dataset_type, adapter_type, nshots)
adapt_path = '/export/livia/home/vision/Bmurugesan/CLIPCalib/output/FINAL/{}/ADAPTER/rn50/SGD_lr1e-1_B256_ep300_{}Init_noneConstraint_{}shots_ls/seed1/logits.pt'.format(dataset_type, adapter_type, nshots)
adaptpre_path ='/export/livia/home/vision/Bmurugesan/CLIPCalib/output/FINAL/{}/ADAPTER_ZSNORM/rn50/SGD_lr1e-1_B256_ep300_{}Init_noneConstraint_{}shots/seed1/logits.pt'.format(dataset_type, adapter_type, nshots)
adaptpen_path = '/export/livia/home/vision/Bmurugesan/CLIPCalib/output/FINAL/{}/ADAPTER_ZSPEN/rn50/SGD_lr1e-1_B256_ep300_{}Init_noneConstraint_{}shots/seed1/logits.pt'.format(dataset_type, adapter_type, nshots)
print (zs_path, os.path.isfile(zs_path))
print (adapt_path, os.path.isfile(adapt_path))
print (adaptpre_path, os.path.isfile(adaptpre_path))
print (adaptpen_path, os.path.isfile(adaptpen_path))
print (target_path, os.path.isfile(target_path))
logits_zs = torch.load(zs_path, map_location='cpu').float()
logits_adapt = torch.load(adapt_path, map_location='cpu').float()
logits_adapt_pren = torch.load(adaptpre_path,map_location='cpu').float()
logits_adapt_pen = torch.load(adaptpen_path,map_location='cpu').float()
target = torch.load(target_path, map_location='cpu')
min_logits_zs, max_logits_zs = torch.min(logits_zs,1)[0].unsqueeze(1), torch.max(logits_zs,1)[0].unsqueeze(1)
min_logits_adapt, max_logits_adapt = torch.min(logits_adapt,1)[0].unsqueeze(1), torch.max(logits_adapt,1)[0].unsqueeze(1)
logits_adapt_norm = (logits_adapt - min_logits_adapt)/ (max_logits_adapt - min_logits_adapt)
logits_adapt_postn = logits_adapt_norm * (max_logits_zs - min_logits_zs) + min_logits_zs
#zs
sftmx_zs = softmax(logits_zs,1)
conf_zs, pred_zs = torch.max(sftmx_zs,1)
acc_zs = round(accuracy_score(target,pred_zs) * 100,2)
ece_zs = round(expected_calibration_error(conf_zs,pred_zs,target).item()*100,2)
#adapt
sftmx_adapt = softmax(logits_adapt,1)
conf_adapt, pred_adapt = torch.max(sftmx_adapt,1)
acc_adapt = round(accuracy_score(target, pred_adapt) * 100,2)
ece_adapt = round(expected_calibration_error(conf_adapt,pred_adapt,target).item()*100,2)
#adaptpost
sftmx_adapt_postn = softmax(logits_adapt_postn,1)
conf_adapt_postn, pred_adapt_postn = torch.max(sftmx_adapt_postn,1)
acc_adapt_postn = round(accuracy_score(target, pred_adapt_postn) * 100,2)
ece_adapt_postn = round(expected_calibration_error(conf_adapt_postn,pred_adapt_postn,target).item()*100,2)
##adaptpre
sftmx_adapt_pren = softmax(logits_adapt_pren,1)
conf_adapt_pren, pred_adapt_pren = torch.max(sftmx_adapt_pren,1)
acc_adapt_pren = round(accuracy_score(target, pred_adapt_pren) * 100,2)
ece_adapt_pren = round(expected_calibration_error(conf_adapt_pren,pred_adapt_pren,target).item()*100,2)
## adaptpen
sftmx_adapt_pen = softmax(logits_adapt_pen,1)
conf_adapt_pen, pred_adapt_pen = torch.max(sftmx_adapt_pen,1)
acc_adapt_pen = round(accuracy_score(target, pred_adapt_pen) * 100,2)
ece_adapt_pen = round(expected_calibration_error(conf_adapt_pen,pred_adapt_pen,target).item()*100,2)
return acc_zs, acc_adapt, acc_adapt_pren, acc_adapt_postn, acc_adapt_pen, ece_zs, ece_adapt, ece_adapt_pren, ece_adapt_postn, ece_adapt_pen
info = {'dset':[],'method':[],'acc':[], 'ece':[]}
adapter_type = 'TR'
nshots = 16 #sys.argv[1]
for dset in tqdm(['imagenetv2','imagenet_sketch', 'imagenet_a', 'imagenet_r']):
try:
acc_zs, acc_adapt, acc_adapt_pren, acc_adapt_postn, acc_adapt_pen, ece_zs, ece_adapt, ece_adapt_pren, ece_adapt_postn, ece_adapt_pen = post_process(dset, adapter_type, nshots)
except Exception as error:
print (dset, adapter_type, nshots, error)
continue
info['dset'] += [dset]*5
info['acc'].append(acc_zs)
info['ece'].append(ece_zs)
info['method'].append('zs')
info['acc'].append(acc_adapt)
info['ece'].append(ece_adapt)
info['method'].append('adapt')
info['acc'].append(acc_adapt_postn)
info['ece'].append(ece_adapt_postn)
info['method'].append('adapt_posn')
info['acc'].append(acc_adapt_pren)
info['ece'].append(ece_adapt_pren)
info['method'].append('adapt_pren')
info['acc'].append(acc_adapt_pen)
info['ece'].append(ece_adapt_pen)
info['method'].append('adapt_pen')
df = pd.DataFrame(info)
#break
print (df)