-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathEvaluate.py
118 lines (98 loc) · 4.24 KB
/
Evaluate.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
import Loader
import Model_handling
from pickle import load
from pickle import dump
from PIL import Image
from keras.preprocessing.text import Tokenizer
from keras.models import load_model
from nltk.translate.bleu_score import corpus_bleu
#convert descriptions into a list of descriptions
def to_lines(descriptions):
all_desc = list()
for key in descriptions.keys():
[all_desc.append(des) for des in descriptions[key]]
return all_desc
#Encode using tokenizer
#eg - tokenize --> Boy on horse --> startseq Boy on horse endSeq
# X1 X2 y (word)
# photo startseq Boy
# photo startseq, Boy on
# photo startseq, Boy,on horse
# photo startseq,Boy,on,horse endseq
def create_tokenizer(descriptions):
lines = to_lines(descriptions)
tokenizer = Tokenizer()
tokenizer.fit_on_texts(lines)
return tokenizer
# calculate the length of the description with the most words
def max_length(descriptions):
lines = to_lines(descriptions)
return max(len(d.split()) for d in lines)
#mapping of integer to word
def word_for_id(integer, tokenizer):
for word, index in tokenizer.word_index.items():
if index == integer:
return word
return None
def evaluate_model(model, descriptions, photos, tokenizer, max_length):
actual, predicted = list(), list()
for key, desc_list in descriptions.items():
#generate description
yhat = Model_handling.generate_desc(model, tokenizer, photos[key], max_length)
# store actual and predicted captions
references = [d.split() for d in desc_list]
actual.append(references)
predicted.append(yhat.split())
# calculate BLEU score
print('BLEU-1: %f' % corpus_bleu(actual, predicted, weights=(1.0, 0, 0, 0)))
print('BLEU-2: %f' % corpus_bleu(actual, predicted, weights=(0.5, 0.5, 0, 0)))
print('BLEU-3: %f' % corpus_bleu(actual, predicted, weights=(0.3, 0.3, 0.3, 0)))
print('BLEU-4: %f' % corpus_bleu(actual, predicted, weights=(0.25, 0.25, 0.25, 0.25)))
#load trainset
trainFile = './DataSets/Flickr_8k.trainImages.txt'
train = Loader.load_set(trainFile)
print('Dataset: %d' % len(train))
# descriptions
train_descriptions = Loader.load_clean_descriptions('./DataSets/descriptions.txt', train)
print('Descriptions: train=%d' % len(train_descriptions))
# photo features
train_features = Loader.load_photo_features('./DataSets/features.pkl', train)
print('Photos: train=%d' % len(train_features))
tokenizer = create_tokenizer(train_descriptions)
vocab_size = len(tokenizer.word_index) + 1
print('Vocabulary Size: %d' % vocab_size)
# determine the maximum sequence length
max_length = max_length(train_descriptions)
print('Description Length: %d' % max_length)
# prepare sequences
dataset_root_dir = './DataSets/'
code_root_dir = './'
# define the model
model = Model_handling.define_model(vocab_size, max_length)
# train the model, run epochs manually and save after each epoch
epochs = 20
steps = len(train_descriptions)
for i in range(epochs):
# create the data generator
generator = Model_handling.data_generator(train_descriptions, train_features, tokenizer, max_length)
# fit for one epoch
model.fit_generator(generator, epochs=1, steps_per_epoch=steps, verbose=1)
# save model
model.save(code_root_dir + 'ResNet50/model_' + str(i) + '.h5')
# load test set
testImgList = dataset_root_dir + 'Flickr_8k.testImages.txt'
testImg = Loader.load_set(testImgList)
print('Dataset: %d' % len(testImg))
# descriptions
test_descriptions = Loader.load_clean_descriptions(dataset_root_dir + 'descriptions.txt', testImg)
print('Descriptions: test=%d' % len(test_descriptions))
# photo features
test_features = Loader.load_photo_features('./DataSets/features.pkl', testImg)
print('Photos: test=%d' % len(test_features))
weights = code_root_dir + 'ResNet50/model_19.h5'
model = load_model(weights)
evaluate_model(model, test_descriptions, test_features, tokenizer, max_length)
# prepare tokenizer
tokenizer = create_tokenizer(train_descriptions)
# save the tokenizer
dump(tokenizer, open(code_root_dir + 'tokenizer.pkl', 'wb'))