forked from cgnorthcutt/benchmarking-keras-pytorch
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathimagenet_benchmarking.py
executable file
·201 lines (160 loc) · 7.01 KB
/
imagenet_benchmarking.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
#!/usr/bin/env python
# coding: utf-8
# In[ ]:
# These imports enhance Python2/3 compatibility.
from __future__ import print_function, absolute_import, division, unicode_literals, with_statement
# In[ ]:
import numpy as np
import os
import argparse
from torchvision import datasets
VAL_SIZE = 50000
# In[ ]:
models = {
"keras" : [
"densenet121",
"densenet169",
"densenet201",
"mobilenet",
"mobilenetV2",
"nasnetmobile",
"resnet50",
"vgg16",
"vgg19",
"xception",
"inceptionresnetv2",
"inceptionv3",
"nasnetlarge",
],
"pytorch" : [
"inception_v3",
"vgg11",
"vgg11_bn",
"vgg13",
"vgg13_bn",
"vgg16",
"vgg16_bn",
"vgg19",
"vgg19_bn",
"densenet121",
"densenet161",
"densenet169",
"densenet201",
"alexnet",
"squeezenet1_0",
"squeezenet1_1",
"resnet18",
"resnet34",
"resnet50",
"resnet101",
"resnet152",
],
}
# In[ ]:
# Set up argument parser
parser = argparse.ArgumentParser(description='PyTorch and Keras ImageNet Benchmarking')
parser.add_argument('val_dir', metavar='DIR',
help='path to imagenet val dataset folder')
parser.add_argument('-k', '--keras-dir', metavar='MODEL', default= "keras_imagenet/",
help='directory where Keras model outputs are stored.')
parser.add_argument('-p', '--pytorch-dir', metavar='MODEL', default= "pytorch_imagenet/",
help='directory where PyTorch model outputs are stored.')
parser.add_argument('-i', '--indices_to_omit', metavar='INDICES_TO_OMIT', default= None,
help='directory of .npy file of storing val indices to omit.')
# In[ ]:
def compute_val_acc(top5preds, top5probs, labels, indices_to_ignore = None):
# Create a mask of having True for each example we want to include in scoring.
bool_mask = np.ones(VAL_SIZE).astype(bool)
if indices_to_ignore is not None:
bool_mask[indices_to_ignore] = False
pred = top5preds[range(len(top5preds)), np.argmax(top5probs, axis = 1)]
pred = pred[bool_mask]
true = labels[bool_mask]
acc1 = sum(pred == true) / float(len(true))
acc5 = sum([true[i] in row for i, row in enumerate(top5preds[bool_mask])]) / float(len(true))
return acc1, acc5
# In[ ]:
def main(args = parser.parse_args()):
# Grab imagenet data
val_dataset = datasets.ImageFolder(args.val_dir)
img_paths, labels = (list(t) for t in zip(*val_dataset.imgs))
labels = np.asarray(labels)
if args.indices_to_omit:
indices_to_omit = np.load(args.indices_to_omit)
dirs = {
"keras" : args.keras_dir,
"pytorch" : args.pytorch_dir,
}
data = []
for platform_name in ["Keras 2.2.4", "PyTorch 1.0"]:
platform = platform_name.split()[0].lower()
# Read in data and compute accuracies
pred_suffix = "_{}_imagenet_top5preds.npy".format(platform)
prob_suffix = "_{}_imagenet_top5probs.npy".format(platform)
for model in models[platform]:
top5preds = np.load(os.path.join(dirs[platform], model + pred_suffix))
top5probs = np.load(os.path.join(dirs[platform], model + prob_suffix))
acc1, acc5 = compute_val_acc(top5preds, top5probs, labels)
if args.indices_to_omit:
acc1c, acc5c = compute_val_acc(top5preds, top5probs, labels, indices_to_omit)
data.append((platform_name, model, acc1, acc1c, acc5, acc5c))
else:
data.append((platform_name, model, acc1, acc5))
# Order final accuracies
data.sort(key=lambda x: x[2], reverse = True)
# Compute ranking for each column in data
ranks = [list(np.argsort(z)[::-1] + 1) for z in [list(z) for z in zip(*data)][2:]]
data = list(zip(*[list(z) for z in zip(*data)] + ranks))
#### Print results
if args.indices_to_omit is None:
header_row = ("{:<13}{:<19}{:<8}{:<8}{:<8}{:<8}")
print(header_row.format("Platform", "Model", "Acc@1", "Acc@5", "Rank@1", "Rank@5"))
print(header_row.format("-----", "-----", "-----", "------", "------", "-------"))
data_row = ("{:<13}{:<19}{:<8.2%}{:<8.2%}{:<8}{:<8}")
else:
header_row = ("{:<13}{:<19}{:<8}{:<8}{:<8}{:<8}{:<8}{:<8}{:<8}{:<8}")
print(header_row.format("Platform", "Model", "Acc@1", "cAcc@1", "Acc@5", "cAcc@5", "Rank@1", "cRank@5", "Rank@5", "cRank@5"))
print(header_row.format("-----", "-----", "-----", "------", "-----", "------", "------", "-------", "------", "-------"))
data_row = ("{:<13}{:<19}{:<8.2%}{:<8.2%}{:<8.2%}{:<8.2%}{:<8}{:<8}{:<8}{:<8}")
for d in data:
print(data_row.format(*d))
# In[ ]:
if __name__ == '__main__':
main()
# In[ ]:
# Platform Model Acc@1 Acc@5 Rank@1 Rank@5
# ----- ----- ----- ------ ------ -------
# Keras 2.2.4 nasnetlarge 80.83% 95.27% 1 1
# Keras 2.2.4 inceptionresnetv2 78.93% 94.45% 2 2
# PyTorch 1.0 resnet152 77.62% 93.81% 3 3
# Keras 2.2.4 xception 77.18% 93.49% 4 4
# PyTorch 1.0 densenet161 76.92% 93.49% 5 5
# PyTorch 1.0 resnet101 76.64% 93.30% 6 6
# PyTorch 1.0 densenet201 76.41% 93.18% 7 7
# Keras 2.2.4 inceptionv3 76.02% 92.90% 8 8
# PyTorch 1.0 densenet169 75.59% 92.69% 9 9
# PyTorch 1.0 resnet50 75.06% 92.48% 10 10
# Keras 2.2.4 densenet201 74.77% 92.32% 11 11
# PyTorch 1.0 densenet121 74.07% 91.94% 12 12
# Keras 2.2.4 densenet169 73.92% 91.76% 13 13
# PyTorch 1.0 vgg19_bn 72.90% 91.35% 14 14
# PyTorch 1.0 resnet34 72.34% 90.84% 15 16
# PyTorch 1.0 vgg16_bn 72.29% 91.01% 16 15
# Keras 2.2.4 densenet121 72.09% 90.70% 17 17
# Keras 2.2.4 nasnetmobile 71.59% 90.19% 18 19
# PyTorch 1.0 vgg19 71.19% 90.40% 19 18
# PyTorch 1.0 vgg16 70.66% 89.93% 20 20
# Keras 2.2.4 resnet50 70.35% 89.55% 21 22
# PyTorch 1.0 vgg13_bn 70.12% 89.56% 22 21
# Keras 2.2.4 mobilenetV2 69.98% 89.49% 23 23
# PyTorch 1.0 vgg11_bn 69.36% 89.06% 24 24
# PyTorch 1.0 inception_v3 69.25% 88.69% 25 25
# Keras 2.2.4 mobilenet 69.02% 88.48% 26 27
# PyTorch 1.0 vgg13 68.69% 88.65% 27 28
# PyTorch 1.0 resnet18 68.37% 88.56% 28 26
# PyTorch 1.0 vgg11 67.85% 88.11% 29 29
# Keras 2.2.4 vgg19 65.58% 86.54% 30 30
# Keras 2.2.4 vgg16 65.24% 86.20% 31 31
# PyTorch 1.0 squeezenet1_0 56.49% 79.06% 32 33
# PyTorch 1.0 squeezenet1_1 56.42% 79.21% 33 32
# PyTorch 1.0 alexnet 54.50% 77.64% 34 34