CHanges evaluation multi-label

keras_wrapper/cnn_model.py

@@ -387,13 +387,13 @@ def __init__(self, nOutput=1000, type='basic_model', silence=False, input_shape=
                     logging.info("<<< Loading weights from file " + weights_path + " >>>")
                 self.model.load_weights(weights_path, seq_to_functional=seq_to_functional)
 
-    def updateLogger(self):
+    def updateLogger(self, force=False):
         """
             Checks if the model contains an updated logger.
             If it doesn't then it updates it, which will store evaluation results.
         """
         compulsory_data_types = ['iteration', 'loss', 'accuracy', 'accuracy top-5']
-        if '_Model_Wrapper__logger' not in self.__dict__:
+        if '_Model_Wrapper__logger' not in self.__dict__ or force:
             self.__logger = dict()
         if '_Model_Wrapper__data_types' not in self.__dict__:
             self.__data_types = compulsory_data_types
@@ -424,14 +424,15 @@ def setOutputsMapping(self, outputsMapping, acc_output=None):
         self.outputsMapping = outputsMapping
         self.acc_output = acc_output
 
-    def setOptimizer(self, lr=None, momentum=None, loss=None, metrics=None,
+    def setOptimizer(self, lr=None, momentum=None, loss=None, loss_weights=None, metrics=None,
                      decay=0.0, clipnorm=10., clipvalue=0., optimizer=None, sample_weight_mode=None):
         """
             Sets a new optimizer for the CNN model.
 
             :param lr: learning rate of the network
             :param momentum: momentum of the network (if None, then momentum = 1-lr)
             :param loss: loss function applied for optimization
+	    :param loss_weights: weights given to multi-loss models
             :param metrics: list of Keras' metrics used for evaluating the model. To specify different metrics for different outputs of a multi-output model, you could also pass a dictionary, such as `metrics={'output_a': 'accuracy'}`.
             :param decay: lr decay
             :param clipnorm: gradients' clip norm
@@ -476,7 +477,7 @@ def setOptimizer(self, lr=None, momentum=None, loss=None, metrics=None,
 
         # compile differently depending if our model is 'Sequential', 'Model' or 'Graph'
         if isinstance(self.model, Sequential) or isinstance(self.model, Model):
-            self.model.compile(optimizer=optimizer, metrics=metrics, loss=loss,
+            self.model.compile(optimizer=optimizer, metrics=metrics, loss=loss, loss_weights=loss_weights,
                                sample_weight_mode=sample_weight_mode)
         else:
             raise NotImplementedError()

keras_wrapper/extra/callbacks.py

@@ -59,6 +59,7 @@ def __init__(self,
                  extra_vars=None,
                  is_text=False,
                  is_multilabel=False,
+		 multilabel_idx=None,
                  min_pred_multilabel=0.5,
                  index2word_y=None,
                  input_text_id=None,
@@ -90,6 +91,7 @@ def __init__(self,
         :param extra_vars: dictionary of extra variables
         :param is_text: defines if the predicted info is of type text (in that case the data will be converted from values into a textual representation)
         :param is_multilabel: are we applying multi-label prediction?
+        :param multilabel_idx: output index where to apply the evaluation (set to None if the model has a single output)
         :param min_pred_multilabel: minimum prediction value considered for positive prediction
         :param index2word_y: mapping from the indices to words (only needed if is_text==True)
         :param input_text_id:
@@ -117,6 +119,7 @@ def __init__(self,
         self.index2word_y = index2word_y
         self.is_text = is_text
         self.is_multilabel = is_multilabel
+        self.multilabel_idx = multilabel_idx
         self.min_pred_multilabel = min_pred_multilabel
         self.is_3DLabel = is_3DLabel
         self.sampling = sampling
@@ -246,6 +249,8 @@ def evaluate(self, epoch, counter_name='epoch'):
                                                      verbose=self.verbose)
 
             elif self.is_multilabel:
+                if self.multilabel_idx is not None:
+                    predictions = predictions[self.multilabel_idx]
                 predictions = decode_multilabel(predictions, 
                                                 self.index2word_y, 
                                                 min_val=self.min_pred_multilabel, 

keras_wrapper/extra/evaluation.py

@@ -124,22 +124,45 @@ def multilabel_metrics(pred_list, verbose, extra_vars, split):
                 extra_vars['references'] - list of GT labels
     '''
     word2idx = extra_vars[split]['word2idx']
-    n_classes = len(word2idx)
+
+    # check if an additional dictionary matching raw to basic and general labels is provided
+    # in that case a more general evaluation will be considered
+    raw2basic = extra_vars[split].get('raw2basic', None)
+    if raw2basic is not None:
+        logging.info('Applying general evaluation with raw2basic dictionary.')
+
+    if raw2basic is None:
+        n_classes = len(word2idx)
+    else:
+        basic_values = set(raw2basic.values())
+        n_classes = len(basic_values)
     n_samples = len(pred_list)
 
-    """
-    with open('/media/HDD_3TB/marc/food_ingredients_recognition/pred_out', 'w') as file:
-        for pred in pred_list:
-            file.write(str(pred)+'\n')
-    """    
+
     # Create prediction matrix
     y_pred = np.zeros((n_samples, n_classes))
     for i_s, sample in enumerate(pred_list):
         for word in sample:
-            y_pred[i_s, word2idx[word]] = 1
+            if raw2basic is None:
+                y_pred[i_s, word2idx[word]] = 1
+            else:
+                word = word.strip()
+                y_pred[i_s, raw2basic[word]] = 1
 
+    # Prepare GT
     gt_list = extra_vars[split]['references']
-    y_gt = np.array(gt_list)
+
+    if raw2basic is None:
+        y_gt = np.array(gt_list)
+    else:
+        idx2word = {v:k for k,v in word2idx.iteritems()}
+        y_gt = np.zeros((n_samples, n_classes))
+        for i_s, sample in enumerate(gt_list):
+            for raw_idx, is_active in enumerate(sample):
+                if is_active:
+                    word = idx2word[raw_idx].strip()
+                    y_gt[i_s, raw2basic[word]] = 1
+
 
     # Compute Coverage Error
     coverr = sklearn_metrics.coverage_error(y_gt, y_pred)