working F1 and SMD metrics

angel_system/global_step_prediction/global_step_predictor.py

@@ -951,26 +951,33 @@ def save_TP_FP_FN_per_class(self, step_gts):
         Output form: TP, FP, FN, where each is an N-dimensional vector, where
         N = number of step classes.
         """
-        step_gts = [float(i) for i in step_gts][24:]
         assert len(self.trackers) == 1
         for i, tracker in enumerate(self.trackers):
             #import ipdb; ipdb.set_trace()
             step_predictions = tracker["granular_step_prediction_history"]
-            if len(step_predictions) * 1.8 < len(step_gts):
-                # This must be a 30Hz video, not 15Hz. Take every other GT frame.
-                step_gts = [a for ind, a in enumerate(step_gts) if i%2==0]
         TP = np.zeros(len(self.avg_probs))
         FP = np.zeros(len(self.avg_probs))
         FN = np.zeros(len(self.avg_probs))
         for ind in range(len(self.avg_probs)):
             # i = the index we'll get TP, FN, and FP for.
-            _TP = len([a for j, a in enumerate(step_predictions) if a == ind and step_gts[j] == ind])
+            _TP = len([a for j, a in enumerate(step_predictions[:len(step_gts)]) if a == ind and step_gts[j] == ind])
             TP[ind] = _TP
-            _FP = len([a for j, a in enumerate(step_predictions) if a == ind and step_gts[j] != ind])
+            _FP = len([a for j, a in enumerate(step_predictions[:len(step_gts)]) if a == ind and step_gts[j] != ind])
             FP[ind] = _FP
-            _FN = len([a for j, a in enumerate(step_predictions) if a != ind and step_gts[j] == ind])
+            _FN = len([a for j, a in enumerate(step_predictions[:len(step_gts)]) if a != ind and step_gts[j] == ind])
             FN[ind] = _FN
         return TP, FP, FN
+    def get_single_tracker_pred_history(self):
+        """
+        Get a single tracker's prediction history.
+        For now, this is only functional in the case of a single tracker instance
+        being initialized.
+
+        Output:
+        - list: prediction history. Length = number of frames processed.
+        """
+        assert len(self.trackers) == 1
+        return self.trackers[0]["granular_step_prediction_history"]
 
     def sanitize_str(self, str_: str):
         """

angel_system/global_step_prediction/run_expirement.py

@@ -32,11 +32,18 @@ def run_inference_all_vids(
     all_vid_ids = np.unique(np.asarray(coco_test.images().lookup("video_id")))
     avg_probs = None
     preds, gt = [], []
+    avg_smd, avg_smd_normd = np.array([]), np.array([])
+    mean_F1s = np.array([])
     for vid_id in all_vid_ids:
         print(f"vid_id {vid_id}===========================")
 
         act_path = code_dir / "config/activity_labels/medical" / f"{medical_task}.yaml"
 
+        # Get framerate
+        framerate = round(coco_test.videos(video_ids=[vid_id]).peek()['framerate'])
+        # Assuming FR = 15 or 30
+        assert framerate in [15, 30], f"framerate rounded to {framerate}"
+
         step_predictor = GlobalStepPredictor(
             recipe_types=[f"{medical_task}"],
             activity_config_fpath=act_path.as_posix(),
@@ -46,8 +53,8 @@ def run_inference_all_vids(
                 / "config/tasks/medical"
                 / f"{medical_task}.yaml"
             },
-            # threshold_multiplier=0.3,
-            # threshold_frame_count=2
+            threshold_multiplier=0.6,
+            threshold_frame_count=5
         )
 
         if avg_probs is not None:
@@ -76,6 +83,15 @@ def run_inference_all_vids(
         # All N activity confs x each video frame
         activity_confs = test_video_dset.annots().get("prob")
         activity_gts = truth_video_dset.annots().get("category_id")
+        # If framerate is 30, take every other frame.
+        if framerate == 30:
+            # This must be a 30Hz video, not 15Hz. Take every other GT frame.
+            print(f"halve gt for {vid_id}. Len = {len(activity_gts)}")
+            activity_gts = [a for ind, a in enumerate(activity_gts) if ind%2==0]
+            print(f"new len = {len(activity_gts)}")
+        # ...and cut out the first 25 frames.
+        activity_gts = activity_gts[25:]
+
 
         def get_unique(activity_ids):
             """
@@ -106,7 +122,12 @@ def get_unique(activity_ids):
 
         print(f"unique broad steps: {get_unique(broad_step_gts)}")
 
-        _TP, _FP, _FN = step_predictor.save_TP_FP_FN_per_class(granular_step_gts)
+        # activity_gt_maxes[i] set to max step so far at index i.
+        activity_gt_maxes = activity_gts.copy()
+        for ind in range(1, len(activity_gts)):
+            activity_gt_maxes[ind] = max(activity_gts[:ind])
+
+        _TP, _FP, _FN = step_predictor.save_TP_FP_FN_per_class(activity_gt_maxes)
         if 'TP' in locals():
             TP += _TP
             FP += _FP
@@ -118,23 +139,61 @@ def get_unique(activity_ids):
         class_F1s, mean_F1 = compute_class_f1s_and_mean_f1(TP,FP,FN)
         print(f"class-wise F1s: {class_F1s}\nmean F1: {mean_F1}")
 
+        mean_F1s = np.append(mean_F1s, mean_F1)
+
+        pred_history = step_predictor.get_single_tracker_pred_history()
+        smds, smds_normd = get_start_moment_distances(pred_history, activity_gt_maxes)
+
+        avg_smd = np.append(avg_smd, np.mean(smds))
+        avg_smd_normd = np.append(avg_smd_normd, np.mean(smds_normd))
+
+        print(f"smds (# frames): {smds}, normalized:{smds_normd}")
+        try:
+            print(f"avg frame-wise smd:{avg_smd[-1]}, normalized: {avg_smd_normd[-1]}")
+        except:
+            import ipdb; ipdb.set_trace()
+
         _ = step_predictor.plot_gt_vs_predicted_one_recipe(
-            granular_step_gts,
+            activity_gts,
             recipe_type,
             fname_suffix=f"{str(vid_id)}_granular_{extra_output_suffix}",
             granular_or_broad="granular",
         )
+    print("########## OVERALL")
+    print(f"Overall average smd: {np.mean(avg_smd)}. Normalized: {np.mean(avg_smd_normd)}")
+    print(f"overall mean F1: {np.mean(mean_F1s)}")
+
+def get_start_moment_distances(pred_history, activity_gt_maxes):
+    """
+    Get the distance between ground truth & predictions of the starting frame of
+    each step.
+
+    Outputs:
+    - smds: list of length equal to number 
+    """
+    num_classes = max(activity_gt_maxes)
+    vid_length = len(activity_gt_maxes)
+    smds = np.zeros(num_classes)
+    smds_normd = np.zeros(num_classes)
+    for i in range(num_classes):
+        if i+1 in pred_history:
+            smds[i] = abs(np.where(pred_history == i+1)[0][0] - activity_gt_maxes.index(i+1))
+            smds_normd[i] = smds[i] / vid_length
+        else:
+            smds[i] = vid_length
+            smds_normd[i] = 1
+    return smds, smds_normd
+
 
 def compute_class_f1s_and_mean_f1(TP,FP,FN):
     F1s = np.zeros(len(TP))
     # class-wise F1s:
     for i in range(len(TP)):
         F1s[i] = 2*TP[i] / (2*TP[i] + FP[i] + FN[i])
     # mean F1
-    mean_F1 = 2*np.sum(TP) / (2*np.sum(TP) + np.sum(FP) + np.sum(FN))
+    mean_F1 = 2*np.sum(TP[:-1]) / (2*np.sum(TP[:-1]) + np.sum(FP[:-1]) + np.sum(FN[:-1]))
     return F1s, mean_F1
 
-
 @click.command(context_settings={"help_option_names": ["-h", "--help"]})
 @click.argument(
     "medical_task",