From 02026c77145a51a643b19b7710007ab3311f887f Mon Sep 17 00:00:00 2001 From: zdy023 Date: Wed, 6 Mar 2019 23:56:22 +0800 Subject: [PATCH] Mar 6th Revision for README and the codes in tc-ssn. --- README.md | 7 +- tc-ssn/README.md | 16 ++- tc-ssn/anet_toolkit/.gitignore | 0 .../anet_toolkit/Evaluation/eval_detection.py | 0 tc-ssn/anet_toolkit/Evaluation/utils.py | 0 tc-ssn/combined_eval_detection_results.py | 42 +++--- tc-ssn/combined_refine.py | 16 ++- .../coin_small_tag_train_proposal_list.txt | 0 .../data/coin_small_tag_val_proposal_list.txt | 0 tc-ssn/data/dataset_cfg.yaml | 0 tc-ssn/data/reference_models.yaml | 0 tc-ssn/data_processing.py | 39 +++--- tc-ssn/eval_detection_results.py | 38 ++---- tc-ssn/evaluate.py | 121 ++---------------- tc-ssn/fusion_eval_detection_results.py | 2 - ...n_pkl_generation_eval_detection_results.py | 38 ++---- tc-ssn/gen_matrix.py | 16 ++- tc-ssn/ops/__init__.py | 0 tc-ssn/ops/__init__.pyc | Bin tc-ssn/ops/anet_db.py | 0 tc-ssn/ops/anet_db.pyc | Bin tc-ssn/ops/coinsmallnet_db.py | 0 tc-ssn/ops/detection_metrics.py | 0 tc-ssn/ops/detection_metrics.pyc | Bin tc-ssn/ops/io.py | 0 tc-ssn/ops/io.pyc | Bin tc-ssn/ops/metrics.py | 0 tc-ssn/ops/metrics.pyc | Bin tc-ssn/ops/sequence_funcs.py | 0 tc-ssn/ops/sequence_funcs.pyc | Bin tc-ssn/ops/ssn_ops.py | 0 tc-ssn/ops/thumos_db.py | 0 tc-ssn/ops/thumos_db.pyc | Bin tc-ssn/ops/utils.py | 0 tc-ssn/ops/utils.pyc | Bin tc-ssn/ops/video_funcs.py | 0 tc-ssn/ssn_dataset.py | 0 tc-ssn/transforms.py | 0 38 files changed, 110 insertions(+), 225 deletions(-) mode change 100755 => 100644 README.md mode change 100755 => 100644 tc-ssn/README.md mode change 100755 => 100644 tc-ssn/anet_toolkit/.gitignore mode change 100755 => 100644 tc-ssn/anet_toolkit/Evaluation/eval_detection.py mode change 100755 => 100644 tc-ssn/anet_toolkit/Evaluation/utils.py mode change 100755 => 100644 tc-ssn/data/coin_small_tag_train_proposal_list.txt mode change 100755 => 100644 tc-ssn/data/coin_small_tag_val_proposal_list.txt mode change 100755 => 100644 tc-ssn/data/dataset_cfg.yaml mode change 100755 => 100644 tc-ssn/data/reference_models.yaml mode change 100755 => 100644 tc-ssn/ops/__init__.py mode change 100755 => 100644 tc-ssn/ops/__init__.pyc mode change 100755 => 100644 tc-ssn/ops/anet_db.py mode change 100755 => 100644 tc-ssn/ops/anet_db.pyc mode change 100755 => 100644 tc-ssn/ops/coinsmallnet_db.py mode change 100755 => 100644 tc-ssn/ops/detection_metrics.py mode change 100755 => 100644 tc-ssn/ops/detection_metrics.pyc mode change 100755 => 100644 tc-ssn/ops/io.py mode change 100755 => 100644 tc-ssn/ops/io.pyc mode change 100755 => 100644 tc-ssn/ops/metrics.py mode change 100755 => 100644 tc-ssn/ops/metrics.pyc mode change 100755 => 100644 tc-ssn/ops/sequence_funcs.py mode change 100755 => 100644 tc-ssn/ops/sequence_funcs.pyc mode change 100755 => 100644 tc-ssn/ops/ssn_ops.py mode change 100755 => 100644 tc-ssn/ops/thumos_db.py mode change 100755 => 100644 tc-ssn/ops/thumos_db.pyc mode change 100755 => 100644 tc-ssn/ops/utils.py mode change 100755 => 100644 tc-ssn/ops/utils.pyc mode change 100755 => 100644 tc-ssn/ops/video_funcs.py mode change 100755 => 100644 tc-ssn/ssn_dataset.py mode change 100755 => 100644 tc-ssn/transforms.py diff --git a/README.md b/README.md old mode 100755 new mode 100644 index 6bf8c91..c360b20 --- a/README.md +++ b/README.md @@ -17,8 +17,13 @@ Note that, these methods use frame-wise fisher vector as video representation, w ### References [1] Y. Zhao, Y. Xiong, L. Wang, Z. Wu, X. Tang, and D. Lin. Temporal action detection with structured segment networks. In ICCV, pages 2933–2942, 2017. + [2] H. Xu, A. Das, and K. Saenko. R-C3D: region convolutional 3d network for temporal activity detection. In ICCV, pages 5794–5803, 2017. + [3] A. Richard, H. Kuehne, and J. Gall. Action sets: Weakly supervised action segmentation without ordering constraints. In CVPR, pages 5987–5996, 2018. + [4] A. Richard, H. Kuehne, A. Iqbal, and J. Gall. Neuralnetwork-viterbi: A framework for weakly supervised video learning. In CVPR, pages 7386–7395, 2018. + [5] L. Ding and C. Xu. Weakly-supervised action segmentation with iterative soft boundary assignment. In CVPR, pages 6508–6516, 2018. -[6] J. Donahue, L. A. Hendricks, M. Rohrbach, S. Venugopalan, S. Guadarrama, K. Saenko, and T. Darrell. Long-term recurrent convolutional networks for visual recognition and description. TPAMI, 39(4):677–691, 2017. \ No newline at end of file + +[6] J. Donahue, L. A. Hendricks, M. Rohrbach, S. Venugopalan, S. Guadarrama, K. Saenko, and T. Darrell. Long-term recurrent convolutional networks for visual recognition and description. TPAMI, 39(4):677–691, 2017. diff --git a/tc-ssn/README.md b/tc-ssn/README.md old mode 100755 new mode 100644 index d2293a2..3b1f136 --- a/tc-ssn/README.md +++ b/tc-ssn/README.md @@ -9,6 +9,20 @@ - terminaltables 3.1.0 - pandas 0.23.4 +### The structure of SSN score file + +The score file dumped by SSN is in format of `pkl`. It is serialised from a python `dict` in which the paths of video frames serve as keys and a 4-element tuple of numpy arrays serve as values. The meaning of four arrays is described as following: + +* The shape of the 1st array in the tuple is (N,2) where N denotes the proposal number. The elements in this array indicates the lower and higher bounds of the proposal ranges. +* The shape of the 2nd array in the tuple is (N,K+1) where K denotes the number of action classes. There are the actionness scores in this array. +* The shape of the 3rd array in the tuple is (N,K). There are the completeness scores presented by SSN in this array. +* The shape of the 4th array in the tuple is (N,K,2). There are the regression scores in this array. The regression score is given as a 2-element array \[`center_regression`, `duration_regression`\]. The regression operation could be formularised as: + +``` +regressed_center = range_renter+range_duration*center_regression +regressed_duration = range_duration*exp(duration_regression) +``` + ### Get combined score file The standalone score file of combined scores is required while refining the combined scores of RGB and Flow modality. The program derived from the original evaluation program is used to export the combined scores to a standalone `pkl` file. These programs are `fusion_pkl_generation_eval_detection_results.py` and `fusion_eval_detection_results.py`. Either the program exports the same `pkl` file. @@ -49,4 +63,4 @@ python3 combined_refine.py -c -i -o --externel_score -``` \ No newline at end of file +``` diff --git a/tc-ssn/anet_toolkit/.gitignore b/tc-ssn/anet_toolkit/.gitignore old mode 100755 new mode 100644 diff --git a/tc-ssn/anet_toolkit/Evaluation/eval_detection.py b/tc-ssn/anet_toolkit/Evaluation/eval_detection.py old mode 100755 new mode 100644 diff --git a/tc-ssn/anet_toolkit/Evaluation/utils.py b/tc-ssn/anet_toolkit/Evaluation/utils.py old mode 100755 new mode 100644 diff --git a/tc-ssn/combined_eval_detection_results.py b/tc-ssn/combined_eval_detection_results.py index 595a2ab..4c296ad 100755 --- a/tc-ssn/combined_eval_detection_results.py +++ b/tc-ssn/combined_eval_detection_results.py @@ -43,8 +43,6 @@ num_class = dataset_configs['num_class'] test_prop_file = 'data/{}_proposal_list.txt'.format(dataset_configs['test_list']) evaluate.number_label = num_class -# print('hhh') -# print(test_prop_file) nms_threshold = args.nms_threshold if args.nms_threshold else dataset_configs['evaluation']['nms_threshold'] top_k = args.top_k if args.top_k else dataset_configs['evaluation']['top_k'] @@ -125,7 +123,7 @@ def gen_detection_results(video_id, score_tp): # load combined scores from external numpys ex_vid = video_id.split("/")[-1] - ex_scores = np.load(os.path.join(args.externel_score,ex_vid + ".npy")) + ex_scores = np.load(os.path.join(args.externel_score,"proposal_" + ex_vid + ".npy")) combined_scores = ex_scores[:,:,4] keep_idx = np.argsort(combined_scores.ravel())[-top_k:] @@ -245,49 +243,39 @@ def callback(rst): ap_values[rst[0], rst[1]] = rst[2][0] ar_values[rst[0], rst[1]] = rst[2][1] -zdy_miou = np.zeros((num_class,)) +zdy_miou = np.zeros((num_class,)) # used to store the mIoU of each classes -pku_gt_by_class = [[] for i in range(num_class)] -pku_prediction_by_class = [[] for i in range(num_class)] -pku_gt = [] -pku_prediction = [] +gt_by_class = [[] for i in range(num_class)] +prediction_by_class = [[] for i in range(num_class)] +gt = [] +prediction = [] for cls in range(num_class): for zdy_record in gt_by_cls[cls].itertuples(): - pku_gt_by_class[cls].append([cls,zdy_record[2],zdy_record[3],1,zdy_record[1]]) - pku_gt += pku_gt_by_class[cls] + gt_by_class[cls].append([cls,zdy_record[2],zdy_record[3],1,zdy_record[1]]) + gt += gt_by_class[cls] for zdy_record in plain_detections[cls].itertuples(): - pku_prediction_by_class[cls].append([zdy_record[2],zdy_record[3],zdy_record[4],zdy_record[5],zdy_record[1]]) - pku_prediction += pku_prediction_by_class[cls] + prediction_by_class[cls].append([zdy_record[2],zdy_record[3],zdy_record[4],zdy_record[5],zdy_record[1]]) + prediction += prediction_by_class[cls] if cls!=0: - zdy_miou[cls] = evaluate.miou(pku_prediction_by_class[cls],pku_gt_by_class[cls]) + zdy_miou[cls] = evaluate.miou(prediction_by_class[cls],gt_by_class[cls]) miou = zdy_miou[1:].mean() -print(str(len(pku_gt))) -print(str(len(pku_prediction))) +print(str(len(gt))) +print(str(len(prediction))) f1_values = np.zeros((len(iou_range),)) pool = Pool(args.ap_workers) jobs = [] for iou_idx, min_overlap in enumerate(iou_range): -#for iou_idx, min_overlap in enumerate([0.6]): for cls in range(num_class): - #for cls in [304]: - #jobs.append(pool.apply_async(eval_ap, args=([min_overlap], iou_idx, cls, gt_by_cls[cls], plain_detections[cls],),callback=callback)) - jobs.append(pool.apply_async(eval_ap, args=([min_overlap], iou_idx, cls, pku_gt_by_class[cls], pku_prediction_by_class[cls],),callback=callback)) - f1 = evaluate.f1(pku_prediction,min_overlap,pku_gt) + jobs.append(pool.apply_async(eval_ap, args=([min_overlap], iou_idx, cls, gt_by_class[cls], prediction_by_class[cls],),callback=callback)) + f1 = evaluate.f1(prediction,min_overlap,gt) f1_values[iou_idx] = f1 pool.close() pool.join() print("Evaluation done.\n\n") -"""for zdy_i,zdy_iou in enumerate(iou_range): - with open("accuracy_per_cls/cls_pku{:f}.txt".format(zdy_iou),"w") as zdy_f: - for zdy_cls in range(num_class): - zdy_f.write("{:d}\t{:.04f}\n".format(zdy_cls,ap_values[zdy_cls][zdy_i]))""" - -#map_iou = ap_values[1:,:].mean(axis=0) -#mar = ar_values[1:,:].mean(axis=0) map_iou = ap_values.mean(axis=0) mar = ar_values.mean(axis=0) display_title = "Detection Performance on {}".format(args.dataset) diff --git a/tc-ssn/combined_refine.py b/tc-ssn/combined_refine.py index 82b0f3b..0c4a708 100755 --- a/tc-ssn/combined_refine.py +++ b/tc-ssn/combined_refine.py @@ -1,6 +1,11 @@ #!/usr/bin/python3 -#import json +""" +Refine the scores combined from actionness and completeness scores outputed by SSN. + +Last revision: Danyang Zhang @THU_IVG @Mar 6th, 2019 CST +""" + import numpy as np import os import os.path @@ -14,7 +19,7 @@ parser.add_argument("--target","-o",action="store",type=str,default="test_gt_score_combined_refined_fusion") args = parser.parse_args() -constraints = np.load(args.constraints) +constraints = np.load(args.constraints) # constraints matrix target_class_count,action_class_count = constraints.shape numpy_dir = args.src_score @@ -32,17 +37,18 @@ vid = np_file[np_file.find("_")+1:np_file.rfind(".")] premat = np.load(os.path.join(numpy_dir,np_file)) combined = premat[:,:,4] - #print(str(combined.shape)) video_combined = np.sum(combined,axis=0) target_class_combined = np.zeros((target_class_count,)) for target_cls in range(target_class_count): for act_cls in range(action_class_count): if constraints[target_cls][act_cls]==1: - target_class_combined[target_cls] = video_combined[act_cls] - probable_target_class = np.argmax(target_class_combined) + target_class_combined[target_cls] += video_combined[act_cls] + # aggregate the scores of the action classes under the identical task/target class + probable_target_class = np.argmax(target_class_combined) # infer the probable task class mask = np.full(combined.shape,math.exp(-2)) mask[:,0] = 1 mask[:,np.where(constraints[probable_target_class])[0]] = 1 combined *= mask + # refine the combined scores premat[:,:,4] = combined np.save(os.path.join(target_dir,np_file),premat) diff --git a/tc-ssn/data/coin_small_tag_train_proposal_list.txt b/tc-ssn/data/coin_small_tag_train_proposal_list.txt old mode 100755 new mode 100644 diff --git a/tc-ssn/data/coin_small_tag_val_proposal_list.txt b/tc-ssn/data/coin_small_tag_val_proposal_list.txt old mode 100755 new mode 100644 diff --git a/tc-ssn/data/dataset_cfg.yaml b/tc-ssn/data/dataset_cfg.yaml old mode 100755 new mode 100644 diff --git a/tc-ssn/data/reference_models.yaml b/tc-ssn/data/reference_models.yaml old mode 100755 new mode 100644 diff --git a/tc-ssn/data_processing.py b/tc-ssn/data_processing.py index bb52790..57ee9f5 100755 --- a/tc-ssn/data_processing.py +++ b/tc-ssn/data_processing.py @@ -1,5 +1,11 @@ #!/usr/bin/python3 +""" +Transfer the pkl scores to npy. + +Last revision: Danyang Zhang @THU_IVG @Mar 6th, 2019 CST +""" + import numpy as np import json import os @@ -23,7 +29,6 @@ for v in scores: vid = v.split("/")[-1] - #video_duration = annotations[vid]["end"]-annotations[vid]["start"] video_duration = annotations[vid]["duration"] proposals = scores[v][0] @@ -31,41 +36,29 @@ completeness = scores[v][2] regression = scores[v][3] - score_max = np.max(actionness[:,1:],axis=-1) - exp_score = np.exp(actionness[:,1:]-score_max[...,None]) + score_max = np.max(actionness,axis=-1) + exp_score = np.exp(actionness-score_max[...,None]) exp_com = np.exp(completeness) - combined_scores = (exp_score/np.sum(exp_score,axis=-1)[...,None])*exp_com + combined_scores = (exp_score/np.sum(exp_score,axis=-1)[...,None])[:,1:]*exp_com + # combined scores are calculated as softmax(actionness)*exp(completeness) according to the code offered by SSN proposal_count = len(proposals) class_count = completeness.shape[1] proposal_npy = np.zeros((proposal_count,class_count,7)) + # the columns in proposal_npy: + # start of the proposal range, end of the proposal range, exp(actionness), exp(completeness), combined score, actionness, completeness + for i in range(proposal_count): start = proposals[i][0]*video_duration end = proposals[i][1]*video_duration for c in range(class_count): - center_proportion = (proposals[i][0]+proposals[i][1])/2. - duration_proportion = proposals[i][1]-proposals[i][0] - center_proportion += regression[i][c][0]*duration_proportion - duration_proportion *= math.exp(regression[i][c][1]) - start_proportion = center_proportion-duration_proportion/2. - end_proportion = center_proportion+duration_proportion/2. - start_proportion = max(start_proportion,0.) - start_proportion = min(start_proportion,1.) - end_proportion = max(end_proportion,0.) - end_proportion = min(end_proportion,1.) - #pre_cls["regressed_interval"] = (start_proportion*video_duration,end_proportion*video_duration) - - proposal_npy[i][c][0] = start_proportion*video_duration - proposal_npy[i][c][1] = end_proportion*video_duration - proposal_npy[i][c][2] = exp_score[i][c] + proposal_npy[i][c][0] = proposals[i][0] + proposal_npy[i][c][1] = proposals[i][1] + proposal_npy[i][c][2] = exp_score[i][c+1] proposal_npy[i][c][3] = exp_com[i][c] proposal_npy[i][c][4] = combined_scores[i][c] proposal_npy[i][c][5] = actionness[i][c+1] proposal_npy[i][c][6] = completeness[i][c] - npy_name = os.path.join(output_prefix,"proposal_" + vid) np.save(npy_name,proposal_npy) - np.save(npy_name + "_groundtruth",groundtruth_npy) - #prediction_dict[vid]["prediction_numpy"] = npy_name + ".npy" - #prediction_dict[vid]["groundtruth_numpy"] = npy_name + "_groundtruth" + ".npy" diff --git a/tc-ssn/eval_detection_results.py b/tc-ssn/eval_detection_results.py index a9ec9e9..ad2b0e2 100755 --- a/tc-ssn/eval_detection_results.py +++ b/tc-ssn/eval_detection_results.py @@ -40,8 +40,6 @@ num_class = dataset_configs['num_class'] test_prop_file = 'data/{}_proposal_list.txt'.format(dataset_configs['test_list']) evaluate.number_label = num_class -# print('hhh') -# print(test_prop_file) nms_threshold = args.nms_threshold if args.nms_threshold else dataset_configs['evaluation']['nms_threshold'] top_k = args.top_k if args.top_k else dataset_configs['evaluation']['top_k'] @@ -239,47 +237,37 @@ def callback(rst): zdy_miou = np.zeros((num_class,)) -pku_gt_by_class = [[] for i in range(num_class)] -pku_prediction_by_class = [[] for i in range(num_class)] -pku_gt = [] -pku_prediction = [] +gt_by_class = [[] for i in range(num_class)] +prediction_by_class = [[] for i in range(num_class)] +gt = [] +prediction = [] for cls in range(num_class): for zdy_record in gt_by_cls[cls].itertuples(): - pku_gt_by_class[cls].append([cls,zdy_record[2],zdy_record[3],1,zdy_record[1]]) - pku_gt += pku_gt_by_class[cls] + gt_by_class[cls].append([cls,zdy_record[2],zdy_record[3],1,zdy_record[1]]) + gt += gt_by_class[cls] for zdy_record in plain_detections[cls].itertuples(): - pku_prediction_by_class[cls].append([zdy_record[2],zdy_record[3],zdy_record[4],zdy_record[5],zdy_record[1]]) - pku_prediction += pku_prediction_by_class[cls] + prediction_by_class[cls].append([zdy_record[2],zdy_record[3],zdy_record[4],zdy_record[5],zdy_record[1]]) + prediction += prediction_by_class[cls] if cls!=0: - zdy_miou[cls] = evaluate.miou(pku_prediction_by_class[cls],pku_gt_by_class[cls]) + zdy_miou[cls] = evaluate.miou(prediction_by_class[cls],gt_by_class[cls]) miou = zdy_miou[1:].mean() -print(str(len(pku_gt))) -print(str(len(pku_prediction))) +print(str(len(gt))) +print(str(len(prediction))) f1_values = np.zeros((len(iou_range),)) pool = Pool(args.ap_workers) jobs = [] for iou_idx, min_overlap in enumerate(iou_range): -#for iou_idx, min_overlap in enumerate([0.6]): for cls in range(num_class): - #for cls in [304]: - #jobs.append(pool.apply_async(eval_ap, args=([min_overlap], iou_idx, cls, gt_by_cls[cls], plain_detections[cls],),callback=callback)) - jobs.append(pool.apply_async(eval_ap, args=([min_overlap], iou_idx, cls, pku_gt_by_class[cls], pku_prediction_by_class[cls],),callback=callback)) - f1 = evaluate.f1(pku_prediction,min_overlap,pku_gt) + jobs.append(pool.apply_async(eval_ap, args=([min_overlap], iou_idx, cls, gt_by_class[cls], prediction_by_class[cls],),callback=callback)) + f1 = evaluate.f1(prediction,min_overlap,gt) f1_values[iou_idx] = f1 pool.close() pool.join() print("Evaluation done.\n\n") -"""for zdy_i,zdy_iou in enumerate(iou_range): - with open("accuracy_per_cls/cls_pku{:f}.txt".format(zdy_iou),"w") as zdy_f: - for zdy_cls in range(num_class): - zdy_f.write("{:d}\t{:.04f}\n".format(zdy_cls,ap_values[zdy_cls][zdy_i]))""" - -#map_iou = ap_values[1:,:].mean(axis=0) -#mar = ar_values[1:,:].mean(axis=0) map_iou = ap_values.mean(axis=0) mar = ar_values.mean(axis=0) display_title = "Detection Performance on {}".format(args.dataset) diff --git a/tc-ssn/evaluate.py b/tc-ssn/evaluate.py index 90c0789..00f957d 100755 --- a/tc-ssn/evaluate.py +++ b/tc-ssn/evaluate.py @@ -1,20 +1,12 @@ -# Copyright (c) 2017 Chunhui_Liu@STRUCT_ICST_PKU, All rights reserved. -# -# evaluation protocols used for PKU-MMD dataset -# http://www.icst.pku.edu.cn/struct/Projects/PKUMMD.html -# -# In proposal folder: -# each file contains results for one video. -# several lines in one file, -# each line contain: label, start_frame, end_frame, confidence +""" +Evaluation utilisation function model. Derived from the evaluation code from PKU-MMD (https://github.com/ECHO960/PKU-MMD). Several mistakes and imcompatible with python3 features are corrected. + +Last revision: Danyang Zhang @THU_IVG @Mar 6th, 2019 CST +""" import os import numpy as np -#import matplotlib.pyplot as plt -source_folder = '/home/lch/PKU-3D/result/detc/' -ground_folder = '/mnt/hdd/PKUMMD/test_label_0330/' -fig_folder = '/home/lch/PKU-3D/src/fig/' theta = 0.5 #overlap ratio number_label = 52 @@ -54,44 +46,11 @@ def match(lst, ratio, ground): for y in index_map[int(lst[x][0])]: if (overlap(lst[x], ground[y]) < ratio): continue if cos_map[x]!=-1 and overlap(lst[x], ground[y]) < overlap(lst[x], ground[cos_map[x]]): continue - #if count_map[y]>0: - #continue cos_map[x] = y if (cos_map[x] != -1): count_map[cos_map[x]] += 1 positive = sum([(x>0) for x in count_map]) return cos_map, count_map, positive -# plot_fig: plot precision-recall figure of given proposal -# @lst: list of proposals(label, start, end, confidence, video_name) -# @ratio: overlap ratio -# @ground: list of ground truth(label, start, end, confidence, video_name) -# @method: method name -"""def plot_fig(lst, ratio, ground, method): - lst.sort(key = lambda x:x[3]) # sorted by confidence - cos_map, count_map, positive = match(lst, ratio, ground) - number_proposal = len(lst) - number_ground = len(ground) - old_precision, old_recall = calc_pr(positive, number_proposal, number_ground) - - recalls = [old_recall] - precisions = [old_precision] - for x in xrange(len(lst)): - number_proposal -= 1; - if (cos_map[x] == -1): continue - count_map[cos_map[x]] -= 1; - if (count_map[cos_map[x]] == 0) : positive -= 1; - - precision, recall = calc_pr(positive, number_proposal, number_ground) - if precision>old_precision: - old_precision = precision - recalls.append(recall) - precisions.append(old_precision) - old_recall = recall - fig = plt.figure() - plt.axis([0,1,0,1]) - plt.plot(recalls,precisions,'r') - plt.savefig('%s%s.png'%(fig_folder,method))""" - # f1-score: # @lst: list of proposals(label, start, end, confidence, video_name) # @ratio: overlap ratio @@ -136,14 +95,15 @@ def ap(lst, ratio, ground): if precision>old_precision: old_precision = precision old_recall = recall - #print(score) return score,total_recall def miou(lst,ground): + """ + calculate mIoU through all the predictions + """ cos_map,count_map,positive = match(lst,0,ground) miou = 0 count = len(lst) - #print("{:d}: {:d}".format(count,len(ground))) real_count = 0 for x in range(count): if cos_map[x]!=-1: @@ -152,10 +112,12 @@ def miou(lst,ground): return miou/float(real_count) if real_count!=0 else 0. def miou_per_v(lst,ground): + """ + calculate mIoU through all the predictions in one video first, then average the obtained mIoUs through single video. + """ cos_map,count_map,positive = match(lst,0,ground) count = len(lst) v_miou = {} - #print("{:d}: {:d}".format(count,len(ground))) for x in range(count): if cos_map[x]!=-1: v_id = lst[x][4] @@ -169,64 +131,3 @@ def miou_per_v(lst,ground): miou += v_miou[v][0]/float(v_miou[v][1]) miou /= len(v_miou) return miou - -# process: calculate scores for each method -"""def process(method): - folderpath = source_folder+method+'/' - - v_props = [] # proposal list separated by video - v_grounds = [] # ground-truth list separated by video - - #========== find all proposals separated by video======== - for video in os.listdir(folderpath): - prop = open(folderpath+video,'r').readlines() - prop = [prop[x].replace(",", " ") for x in xrange(len(prop))] - prop = [[float(y) for y in prop[x].split()] for x in xrange(len(prop))] - ground = open(ground_folder+video,'r').readlines() - ground = [ground[x].replace(",", " ") for x in xrange(len(ground))] - ground = [[float(y) for y in ground[x].split()] for x in xrange(len(ground))] - #append video name - for x in prop: x.append(video) - for x in ground: x.append(video) - v_props.append(prop) - v_grounds.append(ground) - - #========== find all proposals separated by action categories======== - # proposal list separated by class - a_props = [[] for x in xrange(number_label)] - # ground-truth list separated by class - a_grounds = [[] for x in xrange(number_label)] - - for x in xrange(len(v_props)): - for y in xrange(len(v_props[x])): - a_props[int(v_props[x][y][0])].append(v_props[x][y]) - - for x in xrange(len(v_grounds)): - for y in xrange(len(v_grounds[x])): - a_grounds[int(v_grounds[x][y][0])].append(v_grounds[x][y]) - - #========== find all proposals======== - all_props = sum(a_props,[]) - all_grounds = sum(a_grounds, []) - - #========== calculate protocols======== - print "================================================" - print "evaluation for method: %s"%method - print "---- for theta = %lf"%theta - print "-------- F1 = ", f1(all_props, theta,all_grounds) - print "-------- AP = ", ap(all_props, theta,all_grounds) - print "-------- mAP_action = ", sum([ap(a_props[x+1], theta, a_grounds[x+1]) \ - for x in xrange(number_label-1)])/(number_label-1) - print "-------- mAP_video = ", sum([ap(v_props[x], theta, v_grounds[x]) \ - for x in xrange(len(v_props))])/len(v_props) - print "-------- 2DAP = ", sum([ap(all_props, (ratio+1)*0.05, all_grounds) \ - for ratio in xrange(20)])/20 - - plot_fig(all_props, theta, all_grounds, method) - print "===============================================" """ - -if __name__ == '__main__': - methods = os.listdir(source_folder) - methods.sort() - for method in methods: - process(method) diff --git a/tc-ssn/fusion_eval_detection_results.py b/tc-ssn/fusion_eval_detection_results.py index 4b34160..feace6d 100755 --- a/tc-ssn/fusion_eval_detection_results.py +++ b/tc-ssn/fusion_eval_detection_results.py @@ -38,8 +38,6 @@ dataset_configs = get_configs(args.dataset) num_class = dataset_configs['num_class'] test_prop_file = 'data/{}_proposal_list.txt'.format(dataset_configs['test_list']) -# print('hhh') -# print(test_prop_file) nms_threshold = args.nms_threshold if args.nms_threshold else dataset_configs['evaluation']['nms_threshold'] top_k = args.top_k if args.top_k else dataset_configs['evaluation']['top_k'] diff --git a/tc-ssn/fusion_pkl_generation_eval_detection_results.py b/tc-ssn/fusion_pkl_generation_eval_detection_results.py index 46a4a8c..5483385 100755 --- a/tc-ssn/fusion_pkl_generation_eval_detection_results.py +++ b/tc-ssn/fusion_pkl_generation_eval_detection_results.py @@ -41,8 +41,6 @@ num_class = dataset_configs['num_class'] test_prop_file = 'data/{}_proposal_list.txt'.format(dataset_configs['test_list']) evaluate.number_label = num_class -# print('hhh') -# print(test_prop_file) nms_threshold = args.nms_threshold if args.nms_threshold else dataset_configs['evaluation']['nms_threshold'] top_k = args.top_k if args.top_k else dataset_configs['evaluation']['top_k'] @@ -241,47 +239,37 @@ def callback(rst): zdy_miou = np.zeros((num_class,)) -pku_gt_by_class = [[] for i in range(num_class)] -pku_prediction_by_class = [[] for i in range(num_class)] -pku_gt = [] -pku_prediction = [] +gt_by_class = [[] for i in range(num_class)] +prediction_by_class = [[] for i in range(num_class)] +gt = [] +prediction = [] for cls in range(num_class): for zdy_record in gt_by_cls[cls].itertuples(): - pku_gt_by_class[cls].append([cls,zdy_record[2],zdy_record[3],1,zdy_record[1]]) - pku_gt += pku_gt_by_class[cls] + gt_by_class[cls].append([cls,zdy_record[2],zdy_record[3],1,zdy_record[1]]) + gt += gt_by_class[cls] for zdy_record in plain_detections[cls].itertuples(): - pku_prediction_by_class[cls].append([zdy_record[2],zdy_record[3],zdy_record[4],zdy_record[5],zdy_record[1]]) - pku_prediction += pku_prediction_by_class[cls] + prediction_by_class[cls].append([zdy_record[2],zdy_record[3],zdy_record[4],zdy_record[5],zdy_record[1]]) + prediction += prediction_by_class[cls] if cls!=0: - zdy_miou[cls] = evaluate.miou(pku_prediction_by_class[cls],pku_gt_by_class[cls]) + zdy_miou[cls] = evaluate.miou(prediction_by_class[cls],gt_by_class[cls]) miou = zdy_miou[1:].mean() -print(str(len(pku_gt))) -print(str(len(pku_prediction))) +print(str(len(gt))) +print(str(len(prediction))) f1_values = np.zeros((len(iou_range),)) pool = Pool(args.ap_workers) jobs = [] for iou_idx, min_overlap in enumerate(iou_range): -#for iou_idx, min_overlap in enumerate([0.6]): for cls in range(num_class): - #for cls in [304]: - #jobs.append(pool.apply_async(eval_ap, args=([min_overlap], iou_idx, cls, gt_by_cls[cls], plain_detections[cls],),callback=callback)) - jobs.append(pool.apply_async(eval_ap, args=([min_overlap], iou_idx, cls, pku_gt_by_class[cls], pku_prediction_by_class[cls],),callback=callback)) - f1 = evaluate.f1(pku_prediction,min_overlap,pku_gt) + jobs.append(pool.apply_async(eval_ap, args=([min_overlap], iou_idx, cls, gt_by_class[cls], prediction_by_class[cls],),callback=callback)) + f1 = evaluate.f1(prediction,min_overlap,gt) f1_values[iou_idx] = f1 pool.close() pool.join() print("Evaluation done.\n\n") -"""for zdy_i,zdy_iou in enumerate(iou_range): - with open("accuracy_per_cls/cls_pku{:f}.txt".format(zdy_iou),"w") as zdy_f: - for zdy_cls in range(num_class): - zdy_f.write("{:d}\t{:.04f}\n".format(zdy_cls,ap_values[zdy_cls][zdy_i]))""" - -#map_iou = ap_values[1:,:].mean(axis=0) -#mar = ar_values[1:,:].mean(axis=0) map_iou = ap_values.mean(axis=0) mar = ar_values.mean(axis=0) display_title = "Detection Performance on {}".format(args.dataset) diff --git a/tc-ssn/gen_matrix.py b/tc-ssn/gen_matrix.py index d10e5bc..10bb016 100755 --- a/tc-ssn/gen_matrix.py +++ b/tc-ssn/gen_matrix.py @@ -1,5 +1,11 @@ #!/usr/bin/python3 +""" +Generate the constraints matrix of the label lexicon. + +Last revision: Danyang Zhang @THU_IVG @Mar 6th, 2019 CST +""" + import numpy as np import json import sys @@ -10,15 +16,13 @@ with open(json_file) as f: database = json.load(f)["database"] -label_set = list(sorted(set(database[v]["class"] for v in database))) -action_set = set() +label_set = list(sorted(set(database[v]["class"] for v in database))) # the set of the task labels +action_set = set() # the set of the action labels for v in database: action_set |= set(int(an["id"]) for an in database[v]["annotation"]) action_set = list(sorted(action_set)) -label_count = len(label_set) -#action_count = len(action_set) -action_count = action_set[-1] -#min_action_id = action_set[0] +label_count = len(label_set) # the number of the task labels +action_count = action_set[-1] # the number of the action labels matrix = np.zeros((label_count,action_count)) for v in database: diff --git a/tc-ssn/ops/__init__.py b/tc-ssn/ops/__init__.py old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/__init__.pyc b/tc-ssn/ops/__init__.pyc old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/anet_db.py b/tc-ssn/ops/anet_db.py old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/anet_db.pyc b/tc-ssn/ops/anet_db.pyc old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/coinsmallnet_db.py b/tc-ssn/ops/coinsmallnet_db.py old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/detection_metrics.py b/tc-ssn/ops/detection_metrics.py old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/detection_metrics.pyc b/tc-ssn/ops/detection_metrics.pyc old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/io.py b/tc-ssn/ops/io.py old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/io.pyc b/tc-ssn/ops/io.pyc old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/metrics.py b/tc-ssn/ops/metrics.py old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/metrics.pyc b/tc-ssn/ops/metrics.pyc old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/sequence_funcs.py b/tc-ssn/ops/sequence_funcs.py old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/sequence_funcs.pyc b/tc-ssn/ops/sequence_funcs.pyc old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/ssn_ops.py b/tc-ssn/ops/ssn_ops.py old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/thumos_db.py b/tc-ssn/ops/thumos_db.py old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/thumos_db.pyc b/tc-ssn/ops/thumos_db.pyc old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/utils.py b/tc-ssn/ops/utils.py old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/utils.pyc b/tc-ssn/ops/utils.pyc old mode 100755 new mode 100644 diff --git a/tc-ssn/ops/video_funcs.py b/tc-ssn/ops/video_funcs.py old mode 100755 new mode 100644 diff --git a/tc-ssn/ssn_dataset.py b/tc-ssn/ssn_dataset.py old mode 100755 new mode 100644 diff --git a/tc-ssn/transforms.py b/tc-ssn/transforms.py old mode 100755 new mode 100644