diff --git a/.gitignore b/.gitignore index d98c30b..7067ed8 100644 --- a/.gitignore +++ b/.gitignore @@ -166,4 +166,5 @@ data/ content-based-*.ipynb cbir_dev.ipynb *.ipynb -*.png \ No newline at end of file +*.png +*.out \ No newline at end of file diff --git a/cbir/metrics.py b/cbir/metrics.py index 6bfc1ef..5298f13 100644 --- a/cbir/metrics.py +++ b/cbir/metrics.py @@ -1,9 +1,14 @@ def average_precision(predictions, ground_truths, k): top_k_predictions = predictions[:k] - relevant_items = sum(1 for pred in top_k_predictions if pred in ground_truths) + hit_count = 1 + relevant_items = [] + for i, pred in enumerate(top_k_predictions): + if pred in ground_truths: + relevant_items.append(hit_count/(i+1)) + hit_count += 1 - ap = relevant_items / len(top_k_predictions) + ap = sum(relevant_items) / hit_count return ap @@ -12,7 +17,8 @@ def hit_rate(predictions, ground_truths, k): relevant_items = sum(1 for pred in top_k_predictions if pred in ground_truths) - hit = 1 if relevant_items >= 1 else 0 + # hit = 1 if relevant_items >= 1 else 0 + hit = relevant_items / len(top_k_predictions) return hit diff --git a/ensemble_all.py b/ensemble_all.py index b0a774f..1b7ce5c 100644 --- a/ensemble_all.py +++ b/ensemble_all.py @@ -61,7 +61,7 @@ columns=[ "k", "distance2score", - "weight(sift vs color)", + "weight(sift vs color vs resnet)", "map@1", "map@5", "map@10", @@ -82,10 +82,10 @@ siftbow = SIFTBOWExtractor(mode="tfidf") sift_array_store = NPArrayStore(retrieve=KNNRetrieval(metric="manhattan")) -rgb_histogram = RGBHistogram(n_bin=4, h_type="region") +rgb_histogram = RGBHistogram(n_bin=4, h_type="region", n_slice=5) color_array_store = NPArrayStore(retrieve=KNNRetrieval(metric="cosine")) -resnet = ResNetExtractor(model="resnet152", device="cuda") +resnet = ResNetExtractor(model="resnet101", device="cuda") resnet_array_store = NPArrayStore(retrieve=KNNRetrieval(metric="cosine")) # Fitting siftbow with train data print("Fitting BOW for n_clusters kmeans: ", 96) @@ -178,6 +178,7 @@ for i in r: predicted.append(i.index) class_preds = np.take(dataset.targets, predicted, axis=0) + predicted = np.array(predicted).tolist() ap1.append(average_precision(class_preds.tolist(), [g.tolist()], 1)) hit1.append(hit_rate(class_preds.tolist(), [g.tolist()], 1)) recall1.append(recall(predicted, np.where(np.isin(np.array(dataset.targets), [g.tolist()]))[0], 1)) diff --git a/ensemble_sift_color.py b/ensemble_sift_color.py index fce5b9d..4076e6b 100644 --- a/ensemble_sift_color.py +++ b/ensemble_sift_color.py @@ -82,7 +82,7 @@ siftbow = SIFTBOWExtractor(mode="tfidf") sift_array_store = NPArrayStore(retrieve=KNNRetrieval(metric="manhattan")) -rgb_histogram = RGBHistogram(n_bin=4, h_type="region") +rgb_histogram = RGBHistogram(n_bin=4, h_type="region", n_slice=5) color_array_store = NPArrayStore(retrieve=KNNRetrieval(metric="cosine")) # Fitting siftbow with train data diff --git a/eval.out b/eval.out deleted file mode 100644 index 40d7fe9..0000000 --- a/eval.out +++ /dev/null @@ -1,330 +0,0 @@ -Evaluate for bins: 2 h_type: region with knn metric: euclidean -map@1: 0.331058 map@5: 0.267986 map@10: 0.239795 hit_rate@1: 0.331058 hit_rate@5: 0.483276 hit_rate@10: 0.550853 recall@1: 0.003012 recall@5: 0.010598 recall@10: 0.017388 recall@100: 0.078739 recall@1000: 0.310192 avg_indexing_time: 0.000992 avg_retrieval_time: 0.000971 -Evaluate for bins: 2 h_type: region with knn metric: cosine -map@1: 0.324232 map@5: 0.26744 map@10: 0.239454 hit_rate@1: 0.324232 hit_rate@5: 0.475768 hit_rate@10: 0.554949 recall@1: 0.00294 recall@5: 0.01054 recall@10: 0.017698 recall@100: 0.080455 recall@1000: 0.312565 avg_indexing_time: 0.000998 avg_retrieval_time: 0.001458 -Evaluate for bins: 2 h_type: global with knn metric: euclidean -map@1: 0.141297 map@5: 0.106758 map@10: 0.099454 hit_rate@1: 0.141297 hit_rate@5: 0.288737 hit_rate@10: 0.382935 recall@1: 0.001118 recall@5: 0.003906 recall@10: 0.007064 recall@100: 0.044466 recall@1000: 0.252985 avg_indexing_time: 0.000994 avg_retrieval_time: 0.000713 -Evaluate for bins: 2 h_type: global with knn metric: cosine -map@1: 0.138567 map@5: 0.107167 map@10: 0.098157 hit_rate@1: 0.138567 hit_rate@5: 0.296246 hit_rate@10: 0.386348 recall@1: 0.00113 recall@5: 0.003966 recall@10: 0.007086 recall@100: 0.044571 recall@1000: 0.254884 avg_indexing_time: 0.000988 avg_retrieval_time: 0.001246 -Evaluate for bins: 4 h_type: region with knn metric: euclidean -map@1: 0.369966 map@5: 0.292014 map@10: 0.262184 hit_rate@1: 0.369966 hit_rate@5: 0.498294 hit_rate@10: 0.584983 recall@1: 0.003627 recall@5: 0.011819 recall@10: 0.019838 recall@100: 0.083359 recall@1000: 0.318225 avg_indexing_time: 0.001009 avg_retrieval_time: 0.001027 -Evaluate for bins: 4 h_type: region with knn metric: cosine -map@1: 0.373379 map@5: 0.298976 map@10: 0.263481 hit_rate@1: 0.373379 hit_rate@5: 0.509215 hit_rate@10: 0.578157 recall@1: 0.003735 recall@5: 0.012632 recall@10: 0.020159 recall@100: 0.088442 recall@1000: 0.322851 avg_indexing_time: 0.000988 avg_retrieval_time: 0.003767 -Evaluate for bins: 4 h_type: global with knn metric: euclidean -map@1: 0.243686 map@5: 0.194266 map@10: 0.175973 hit_rate@1: 0.243686 hit_rate@5: 0.411604 hit_rate@10: 0.488055 recall@1: 0.001998 recall@5: 0.007104 recall@10: 0.01214 recall@100: 0.06312 recall@1000: 0.282745 avg_indexing_time: 0.000986 avg_retrieval_time: 0.000991 -Evaluate for bins: 4 h_type: global with knn metric: cosine -map@1: 0.238908 map@5: 0.193993 map@10: 0.175563 hit_rate@1: 0.238908 hit_rate@5: 0.417747 hit_rate@10: 0.486007 recall@1: 0.002068 recall@5: 0.007408 recall@10: 0.012499 recall@100: 0.063244 recall@1000: 0.288249 avg_indexing_time: 0.000996 avg_retrieval_time: 0.001339 -Evaluate for bins: 8 h_type: region with knn metric: euclidean -map@1: 0.352901 map@5: 0.273311 map@10: 0.23884 hit_rate@1: 0.352901 hit_rate@5: 0.483276 hit_rate@10: 0.548123 recall@1: 0.003375 recall@5: 0.011001 recall@10: 0.017678 recall@100: 0.076686 recall@1000: 0.308355 avg_indexing_time: 0.001082 avg_retrieval_time: 0.002736 -Evaluate for bins: 8 h_type: region with knn metric: cosine -map@1: 0.356997 map@5: 0.279317 map@10: 0.248601 hit_rate@1: 0.356997 hit_rate@5: 0.486007 hit_rate@10: 0.546758 recall@1: 0.003496 recall@5: 0.011554 recall@10: 0.018921 recall@100: 0.084385 recall@1000: 0.325361 avg_indexing_time: 0.001062 avg_retrieval_time: 0.025872 -Evaluate for bins: 8 h_type: global with knn metric: euclidean -map@1: 0.275085 map@5: 0.216246 map@10: 0.189283 hit_rate@1: 0.275085 hit_rate@5: 0.427304 hit_rate@10: 0.48942 recall@1: 0.002263 recall@5: 0.007664 recall@10: 0.012529 recall@100: 0.061587 recall@1000: 0.292441 avg_indexing_time: 0.001007 avg_retrieval_time: 0.002483 -Evaluate for bins: 8 h_type: global with knn metric: cosine -map@1: 0.269625 map@5: 0.207918 map@10: 0.182935 hit_rate@1: 0.269625 hit_rate@5: 0.417065 hit_rate@10: 0.496246 recall@1: 0.002455 recall@5: 0.00777 recall@10: 0.012896 recall@100: 0.063367 recall@1000: 0.297811 avg_indexing_time: 0.001001 avg_retrieval_time: 0.002484 -Evaluate for bins: 12 h_type: region with knn metric: euclidean -map@1: 0.327645 map@5: 0.254334 map@10: 0.221365 hit_rate@1: 0.327645 hit_rate@5: 0.458703 hit_rate@10: 0.529693 recall@1: 0.00304 recall@5: 0.0106 recall@10: 0.017018 recall@100: 0.073402 recall@1000: 0.298012 avg_indexing_time: 0.001484 avg_retrieval_time: 0.007101 -Evaluate for bins: 12 h_type: region with knn metric: cosine -map@1: 0.338567 map@5: 0.272901 map@10: 0.239181 hit_rate@1: 0.338567 hit_rate@5: 0.479181 hit_rate@10: 0.543345 recall@1: 0.003345 recall@5: 0.011587 recall@10: 0.018738 recall@100: 0.083657 recall@1000: 0.322693 avg_indexing_time: 0.001499 avg_retrieval_time: 0.067846 -Evaluate for bins: 12 h_type: global with knn metric: euclidean -map@1: 0.285324 map@5: 0.215017 map@10: 0.190512 hit_rate@1: 0.285324 hit_rate@5: 0.410239 hit_rate@10: 0.487372 recall@1: 0.002313 recall@5: 0.007455 recall@10: 0.012528 recall@100: 0.061049 recall@1000: 0.296934 avg_indexing_time: 0.001011 avg_retrieval_time: 0.005967 -Evaluate for bins: 12 h_type: global with knn metric: cosine -map@1: 0.278498 map@5: 0.208464 map@10: 0.181229 hit_rate@1: 0.278498 hit_rate@5: 0.410239 hit_rate@10: 0.481911 recall@1: 0.002527 recall@5: 0.007385 recall@10: 0.01234 recall@100: 0.062641 recall@1000: 0.300355 avg_indexing_time: 0.000999 avg_retrieval_time: 0.007284 -Fitting BOW for n_clusters kmeans: 32 -Fit Kmeans clustering to create BOW -Fit IDF for TF-IDF Transformation -Complete Fitting SIFT BOW Extractor -Evaluate for kmeans cluster: 32 vector type: tfidf with knn metric: cosine -Evaluate for n_clusters kmeans: 32 -map@1: 0.205461 map@5: 0.164232 map@10: 0.149488 hit_rate@1: 0.205461 hit_rate@5: 0.389761 hit_rate@10: 0.479181 recall@1: 0.001975 recall@5: 0.007142 recall@10: 0.012058 recall@100: 0.067544 recall@1000: 0.323383 avg_indexing_time: 0.008164 avg_retrieval_time: 0.005317 fitting_time: 20.962966 -Evaluate for kmeans cluster: 32 vector type: tfidf with knn metric: euclidean -Evaluate for n_clusters kmeans: 32 -map@1: 0.205461 map@5: 0.164232 map@10: 0.149488 hit_rate@1: 0.205461 hit_rate@5: 0.389761 hit_rate@10: 0.479181 recall@1: 0.001975 recall@5: 0.007142 recall@10: 0.012058 recall@100: 0.067544 recall@1000: 0.323383 avg_indexing_time: 0.008251 avg_retrieval_time: 0.002861 fitting_time: 20.962966 -Evaluate for kmeans cluster: 32 vector type: tfidf with knn metric: manhattan -Evaluate for n_clusters kmeans: 32 -map@1: 0.211604 map@5: 0.165597 map@10: 0.148259 hit_rate@1: 0.211604 hit_rate@5: 0.393174 hit_rate@10: 0.490102 recall@1: 0.002018 recall@5: 0.007452 recall@10: 0.012401 recall@100: 0.070791 recall@1000: 0.337613 avg_indexing_time: 0.008192 avg_retrieval_time: 0.002848 fitting_time: 20.962966 -Evaluate for kmeans cluster: 32 vector type: bow with knn metric: cosine -Evaluate for n_clusters kmeans: 32 -map@1: 0.205461 map@5: 0.164232 map@10: 0.149488 hit_rate@1: 0.205461 hit_rate@5: 0.389761 hit_rate@10: 0.479181 recall@1: 0.001975 recall@5: 0.007142 recall@10: 0.012058 recall@100: 0.067544 recall@1000: 0.323383 avg_indexing_time: 0.008211 avg_retrieval_time: 0.00538 fitting_time: 20.962966 -Evaluate for kmeans cluster: 32 vector type: bow with knn metric: euclidean -Evaluate for n_clusters kmeans: 32 -map@1: 0.205461 map@5: 0.164232 map@10: 0.149488 hit_rate@1: 0.205461 hit_rate@5: 0.389761 hit_rate@10: 0.479181 recall@1: 0.001975 recall@5: 0.007142 recall@10: 0.012058 recall@100: 0.067544 recall@1000: 0.323383 avg_indexing_time: 0.008154 avg_retrieval_time: 0.002898 fitting_time: 20.962966 -Evaluate for kmeans cluster: 32 vector type: bow with knn metric: manhattan -Evaluate for n_clusters kmeans: 32 -map@1: 0.211604 map@5: 0.165597 map@10: 0.148259 hit_rate@1: 0.211604 hit_rate@5: 0.393174 hit_rate@10: 0.490102 recall@1: 0.002018 recall@5: 0.007452 recall@10: 0.012401 recall@100: 0.070791 recall@1000: 0.337613 avg_indexing_time: 0.00824 avg_retrieval_time: 0.002835 fitting_time: 20.962966 -Fitting BOW for n_clusters kmeans: 64 -Fit Kmeans clustering to create BOW -Fit IDF for TF-IDF Transformation -Complete Fitting SIFT BOW Extractor -Evaluate for kmeans cluster: 64 vector type: tfidf with knn metric: cosine -Evaluate for n_clusters kmeans: 64 -map@1: 0.204096 map@5: 0.166962 map@10: 0.15058 hit_rate@1: 0.204096 hit_rate@5: 0.39727 hit_rate@10: 0.495563 recall@1: 0.002139 recall@5: 0.007708 recall@10: 0.013283 recall@100: 0.070053 recall@1000: 0.320327 avg_indexing_time: 0.008273 avg_retrieval_time: 0.005517 fitting_time: 30.953784 -Evaluate for kmeans cluster: 64 vector type: tfidf with knn metric: euclidean -Evaluate for n_clusters kmeans: 64 -map@1: 0.204096 map@5: 0.166962 map@10: 0.15058 hit_rate@1: 0.204096 hit_rate@5: 0.39727 hit_rate@10: 0.495563 recall@1: 0.002139 recall@5: 0.007708 recall@10: 0.013283 recall@100: 0.070053 recall@1000: 0.320327 avg_indexing_time: 0.008147 avg_retrieval_time: 0.002911 fitting_time: 30.953784 -Evaluate for kmeans cluster: 64 vector type: tfidf with knn metric: manhattan -Evaluate for n_clusters kmeans: 64 -map@1: 0.217065 map@5: 0.172014 map@10: 0.155836 hit_rate@1: 0.217065 hit_rate@5: 0.401365 hit_rate@10: 0.514676 recall@1: 0.00227 recall@5: 0.008095 recall@10: 0.01434 recall@100: 0.077102 recall@1000: 0.348248 avg_indexing_time: 0.008133 avg_retrieval_time: 0.002857 fitting_time: 30.953784 -Evaluate for kmeans cluster: 64 vector type: bow with knn metric: cosine -Evaluate for n_clusters kmeans: 64 -map@1: 0.204096 map@5: 0.166962 map@10: 0.15058 hit_rate@1: 0.204096 hit_rate@5: 0.39727 hit_rate@10: 0.495563 recall@1: 0.002139 recall@5: 0.007708 recall@10: 0.013283 recall@100: 0.070053 recall@1000: 0.320327 avg_indexing_time: 0.008321 avg_retrieval_time: 0.00551 fitting_time: 30.953784 -Evaluate for kmeans cluster: 64 vector type: bow with knn metric: euclidean -Evaluate for n_clusters kmeans: 64 -map@1: 0.204096 map@5: 0.166962 map@10: 0.15058 hit_rate@1: 0.204096 hit_rate@5: 0.39727 hit_rate@10: 0.495563 recall@1: 0.002139 recall@5: 0.007708 recall@10: 0.013283 recall@100: 0.070053 recall@1000: 0.320327 avg_indexing_time: 0.008275 avg_retrieval_time: 0.00292 fitting_time: 30.953784 -Evaluate for kmeans cluster: 64 vector type: bow with knn metric: manhattan -Evaluate for n_clusters kmeans: 64 -map@1: 0.217065 map@5: 0.172014 map@10: 0.155836 hit_rate@1: 0.217065 hit_rate@5: 0.401365 hit_rate@10: 0.514676 recall@1: 0.00227 recall@5: 0.008095 recall@10: 0.01434 recall@100: 0.077102 recall@1000: 0.348248 avg_indexing_time: 0.008282 avg_retrieval_time: 0.002846 fitting_time: 30.953784 -Fitting BOW for n_clusters kmeans: 96 -Fit Kmeans clustering to create BOW -Fit IDF for TF-IDF Transformation -Complete Fitting SIFT BOW Extractor -Evaluate for kmeans cluster: 96 vector type: tfidf with knn metric: cosine -Evaluate for n_clusters kmeans: 96 -map@1: 0.205461 map@5: 0.165734 map@10: 0.143959 hit_rate@1: 0.205461 hit_rate@5: 0.384983 hit_rate@10: 0.492833 recall@1: 0.002265 recall@5: 0.008179 recall@10: 0.013076 recall@100: 0.06592 recall@1000: 0.296878 avg_indexing_time: 0.008306 avg_retrieval_time: 0.005895 fitting_time: 40.023622 -Evaluate for kmeans cluster: 96 vector type: tfidf with knn metric: euclidean -Evaluate for n_clusters kmeans: 96 -map@1: 0.205461 map@5: 0.165734 map@10: 0.143959 hit_rate@1: 0.205461 hit_rate@5: 0.384983 hit_rate@10: 0.492833 recall@1: 0.002265 recall@5: 0.008179 recall@10: 0.013076 recall@100: 0.06592 recall@1000: 0.296878 avg_indexing_time: 0.008298 avg_retrieval_time: 0.002899 fitting_time: 40.023622 -Evaluate for kmeans cluster: 96 vector type: tfidf with knn metric: manhattan -Evaluate for n_clusters kmeans: 96 -map@1: 0.221843 map@5: 0.178703 map@10: 0.154812 hit_rate@1: 0.221843 hit_rate@5: 0.414334 hit_rate@10: 0.517406 recall@1: 0.002413 recall@5: 0.008944 recall@10: 0.01491 recall@100: 0.074755 recall@1000: 0.333368 avg_indexing_time: 0.008094 avg_retrieval_time: 0.002874 fitting_time: 40.023622 -Evaluate for kmeans cluster: 96 vector type: bow with knn metric: cosine -Evaluate for n_clusters kmeans: 96 -map@1: 0.205461 map@5: 0.165734 map@10: 0.143959 hit_rate@1: 0.205461 hit_rate@5: 0.384983 hit_rate@10: 0.492833 recall@1: 0.002265 recall@5: 0.008179 recall@10: 0.013076 recall@100: 0.06592 recall@1000: 0.296878 avg_indexing_time: 0.008244 avg_retrieval_time: 0.005679 fitting_time: 40.023622 -Evaluate for kmeans cluster: 96 vector type: bow with knn metric: euclidean -Evaluate for n_clusters kmeans: 96 -map@1: 0.205461 map@5: 0.165734 map@10: 0.143959 hit_rate@1: 0.205461 hit_rate@5: 0.384983 hit_rate@10: 0.492833 recall@1: 0.002265 recall@5: 0.008179 recall@10: 0.013076 recall@100: 0.06592 recall@1000: 0.296878 avg_indexing_time: 0.008176 avg_retrieval_time: 0.002938 fitting_time: 40.023622 -Evaluate for kmeans cluster: 96 vector type: bow with knn metric: manhattan -Evaluate for n_clusters kmeans: 96 -map@1: 0.221843 map@5: 0.178703 map@10: 0.154812 hit_rate@1: 0.221843 hit_rate@5: 0.414334 hit_rate@10: 0.517406 recall@1: 0.002413 recall@5: 0.008944 recall@10: 0.01491 recall@100: 0.074755 recall@1000: 0.333368 avg_indexing_time: 0.008263 avg_retrieval_time: 0.002847 fitting_time: 40.023622 -Fitting BOW for n_clusters kmeans: 128 -Fit Kmeans clustering to create BOW -Fit IDF for TF-IDF Transformation -Complete Fitting SIFT BOW Extractor -Evaluate for kmeans cluster: 128 vector type: tfidf with knn metric: cosine -Evaluate for n_clusters kmeans: 128 -map@1: 0.208191 map@5: 0.166962 map@10: 0.1443 hit_rate@1: 0.208191 hit_rate@5: 0.391809 hit_rate@10: 0.484642 recall@1: 0.002254 recall@5: 0.007971 recall@10: 0.012924 recall@100: 0.065877 recall@1000: 0.298475 avg_indexing_time: 0.00824 avg_retrieval_time: 0.005746 fitting_time: 49.517996 -Evaluate for kmeans cluster: 128 vector type: tfidf with knn metric: euclidean -Evaluate for n_clusters kmeans: 128 -map@1: 0.208191 map@5: 0.166962 map@10: 0.1443 hit_rate@1: 0.208191 hit_rate@5: 0.391809 hit_rate@10: 0.484642 recall@1: 0.002254 recall@5: 0.007971 recall@10: 0.012924 recall@100: 0.065877 recall@1000: 0.298475 avg_indexing_time: 0.008304 avg_retrieval_time: 0.002976 fitting_time: 49.517996 -Evaluate for kmeans cluster: 128 vector type: tfidf with knn metric: manhattan -Evaluate for n_clusters kmeans: 128 -map@1: 0.215017 map@5: 0.175563 map@10: 0.155154 hit_rate@1: 0.215017 hit_rate@5: 0.404096 hit_rate@10: 0.512628 recall@1: 0.002474 recall@5: 0.008643 recall@10: 0.0147 recall@100: 0.076088 recall@1000: 0.343889 avg_indexing_time: 0.008166 avg_retrieval_time: 0.002879 fitting_time: 49.517996 -Evaluate for kmeans cluster: 128 vector type: bow with knn metric: cosine -Evaluate for n_clusters kmeans: 128 -map@1: 0.208191 map@5: 0.166962 map@10: 0.1443 hit_rate@1: 0.208191 hit_rate@5: 0.391809 hit_rate@10: 0.484642 recall@1: 0.002254 recall@5: 0.007971 recall@10: 0.012924 recall@100: 0.065877 recall@1000: 0.298475 avg_indexing_time: 0.008287 avg_retrieval_time: 0.005987 fitting_time: 49.517996 -Evaluate for kmeans cluster: 128 vector type: bow with knn metric: euclidean -Evaluate for n_clusters kmeans: 128 -map@1: 0.208191 map@5: 0.166962 map@10: 0.1443 hit_rate@1: 0.208191 hit_rate@5: 0.391809 hit_rate@10: 0.484642 recall@1: 0.002254 recall@5: 0.007971 recall@10: 0.012924 recall@100: 0.065877 recall@1000: 0.298475 avg_indexing_time: 0.008283 avg_retrieval_time: 0.003006 fitting_time: 49.517996 -Evaluate for kmeans cluster: 128 vector type: bow with knn metric: manhattan -Evaluate for n_clusters kmeans: 128 -map@1: 0.215017 map@5: 0.175563 map@10: 0.155154 hit_rate@1: 0.215017 hit_rate@5: 0.404096 hit_rate@10: 0.512628 recall@1: 0.002474 recall@5: 0.008643 recall@10: 0.0147 recall@100: 0.076088 recall@1000: 0.343889 avg_indexing_time: 0.008294 avg_retrieval_time: 0.002885 fitting_time: 49.517996 -Fitting BOW for n_clusters kmeans: 256 -Fit Kmeans clustering to create BOW -Fit IDF for TF-IDF Transformation -Complete Fitting SIFT BOW Extractor -Evaluate for kmeans cluster: 256 vector type: tfidf with knn metric: cosine -Evaluate for n_clusters kmeans: 256 -map@1: 0.185666 map@5: 0.151536 map@10: 0.134471 hit_rate@1: 0.185666 hit_rate@5: 0.353584 hit_rate@10: 0.461433 recall@1: 0.001867 recall@5: 0.007289 recall@10: 0.012354 recall@100: 0.058084 recall@1000: 0.272928 avg_indexing_time: 0.008459 avg_retrieval_time: 0.006563 fitting_time: 86.50311 -Evaluate for kmeans cluster: 256 vector type: tfidf with knn metric: euclidean -Evaluate for n_clusters kmeans: 256 -map@1: 0.185666 map@5: 0.151536 map@10: 0.134471 hit_rate@1: 0.185666 hit_rate@5: 0.353584 hit_rate@10: 0.461433 recall@1: 0.001867 recall@5: 0.007289 recall@10: 0.012354 recall@100: 0.058084 recall@1000: 0.272928 avg_indexing_time: 0.008313 avg_retrieval_time: 0.00315 fitting_time: 86.50311 -Evaluate for kmeans cluster: 256 vector type: tfidf with knn metric: manhattan -Evaluate for n_clusters kmeans: 256 -map@1: 0.188396 map@5: 0.154676 map@10: 0.139659 hit_rate@1: 0.188396 hit_rate@5: 0.359044 hit_rate@10: 0.453242 recall@1: 0.001966 recall@5: 0.007702 recall@10: 0.013112 recall@100: 0.063983 recall@1000: 0.314161 avg_indexing_time: 0.008386 avg_retrieval_time: 0.002934 fitting_time: 86.50311 -Evaluate for kmeans cluster: 256 vector type: bow with knn metric: cosine -Evaluate for n_clusters kmeans: 256 -map@1: 0.185666 map@5: 0.151536 map@10: 0.134471 hit_rate@1: 0.185666 hit_rate@5: 0.353584 hit_rate@10: 0.461433 recall@1: 0.001867 recall@5: 0.007289 recall@10: 0.012354 recall@100: 0.058084 recall@1000: 0.272928 avg_indexing_time: 0.008225 avg_retrieval_time: 0.006406 fitting_time: 86.50311 -Evaluate for kmeans cluster: 256 vector type: bow with knn metric: euclidean -Evaluate for n_clusters kmeans: 256 -map@1: 0.185666 map@5: 0.151536 map@10: 0.134471 hit_rate@1: 0.185666 hit_rate@5: 0.353584 hit_rate@10: 0.461433 recall@1: 0.001867 recall@5: 0.007289 recall@10: 0.012354 recall@100: 0.058084 recall@1000: 0.272928 avg_indexing_time: 0.008303 avg_retrieval_time: 0.003149 fitting_time: 86.50311 -Evaluate for kmeans cluster: 256 vector type: bow with knn metric: manhattan -Evaluate for n_clusters kmeans: 256 -map@1: 0.188396 map@5: 0.154676 map@10: 0.139659 hit_rate@1: 0.188396 hit_rate@5: 0.359044 hit_rate@10: 0.453242 recall@1: 0.001966 recall@5: 0.007702 recall@10: 0.013112 recall@100: 0.063983 recall@1000: 0.314161 avg_indexing_time: 0.008362 avg_retrieval_time: 0.002945 fitting_time: 86.50311 -Evaluate for model: resnet18 with knn metric: euclidean -map@1: 0.798635 map@5: 0.745256 map@10: 0.704573 hit_rate@1: 0.798635 hit_rate@5: 0.912628 hit_rate@10: 0.939249 recall@1: 0.012276 recall@5: 0.055231 recall@10: 0.101105 recall@100: 0.424683 recall@1000: 0.819254 avg_indexing_time: 0.001124 avg_retrieval_time: 0.001946 -Evaluate for model: resnet18 with knn metric: cosine -map@1: 0.830034 map@5: 0.773788 map@10: 0.73802 hit_rate@1: 0.830034 hit_rate@5: 0.931741 hit_rate@10: 0.949488 recall@1: 0.012926 recall@5: 0.058723 recall@10: 0.109164 recall@100: 0.473179 recall@1000: 0.87688 avg_indexing_time: 0.001075 avg_retrieval_time: 0.003492 -Evaluate for model: resnet34 with knn metric: euclidean -map@1: 0.838908 map@5: 0.787577 map@10: 0.750444 hit_rate@1: 0.838908 hit_rate@5: 0.931058 hit_rate@10: 0.957679 recall@1: 0.01328 recall@5: 0.060532 recall@10: 0.113124 recall@100: 0.485548 recall@1000: 0.867162 avg_indexing_time: 0.001206 avg_retrieval_time: 0.002053 -Evaluate for model: resnet34 with knn metric: cosine -map@1: 0.853925 map@5: 0.813515 map@10: 0.777884 hit_rate@1: 0.853925 hit_rate@5: 0.948123 hit_rate@10: 0.967235 recall@1: 0.013635 recall@5: 0.063591 recall@10: 0.119369 recall@100: 0.532325 recall@1000: 0.916735 avg_indexing_time: 0.001199 avg_retrieval_time: 0.003521 -Evaluate for model: resnet50 with knn metric: euclidean -map@1: 0.845051 map@5: 0.786212 map@10: 0.75058 hit_rate@1: 0.845051 hit_rate@5: 0.933106 hit_rate@10: 0.957679 recall@1: 0.013334 recall@5: 0.059963 recall@10: 0.112308 recall@100: 0.484118 recall@1000: 0.871346 avg_indexing_time: 0.002949 avg_retrieval_time: 0.004597 -Evaluate for model: resnet50 with knn metric: cosine -map@1: 0.864164 map@5: 0.807918 map@10: 0.775085 hit_rate@1: 0.864164 hit_rate@5: 0.942662 hit_rate@10: 0.963823 recall@1: 0.013846 recall@5: 0.06263 recall@10: 0.117603 recall@100: 0.525629 recall@1000: 0.910529 avg_indexing_time: 0.002932 avg_retrieval_time: 0.014151 -Evaluate for model: resnet101 with knn metric: euclidean -map@1: 0.867577 map@5: 0.817201 map@10: 0.783959 hit_rate@1: 0.867577 hit_rate@5: 0.948805 hit_rate@10: 0.967918 recall@1: 0.013994 recall@5: 0.064536 recall@10: 0.122024 recall@100: 0.524354 recall@1000: 0.895706 avg_indexing_time: 0.003545 avg_retrieval_time: 0.004784 -Evaluate for model: resnet101 with knn metric: cosine -map@1: 0.875085 map@5: 0.838362 map@10: 0.803891 hit_rate@1: 0.875085 hit_rate@5: 0.956997 hit_rate@10: 0.970648 recall@1: 0.014133 recall@5: 0.066704 recall@10: 0.126178 recall@100: 0.560759 recall@1000: 0.931237 avg_indexing_time: 0.003557 avg_retrieval_time: 0.014368 -Evaluate for model: resnet152 with knn metric: euclidean -map@1: 0.865529 map@5: 0.80901 map@10: 0.773379 hit_rate@1: 0.865529 hit_rate@5: 0.959044 hit_rate@10: 0.972696 recall@1: 0.013978 recall@5: 0.063623 recall@10: 0.119524 recall@100: 0.523001 recall@1000: 0.899989 avg_indexing_time: 0.004256 avg_retrieval_time: 0.00493 -Evaluate for model: resnet152 with knn metric: cosine -map@1: 0.881911 map@5: 0.828259 map@10: 0.797884 hit_rate@1: 0.881911 hit_rate@5: 0.965188 hit_rate@10: 0.975427 recall@1: 0.014181 recall@5: 0.065997 recall@10: 0.124864 recall@100: 0.562479 recall@1000: 0.932087 avg_indexing_time: 0.00429 avg_retrieval_time: 0.014589 -Fitting BOW for n_clusters kmeans: 96 -Fit Kmeans clustering to create BOW -Fit IDF for TF-IDF Transformation -Complete Fitting SIFT BOW Extractor -Evaluate for init k: 100 with d2s: exp with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380205 map@5: 0.302116 map@10: 0.267304 hit_rate@1: 0.380205 hit_rate@5: 0.513993 hit_rate@10: 0.586348 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.019569 -Evaluate for init k: 100 with d2s: exp with weight: (0.4, 0.6) (sift vs color) -map@1: 0.382253 map@5: 0.306348 map@10: 0.269556 hit_rate@1: 0.382253 hit_rate@5: 0.520137 hit_rate@10: 0.589761 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001201 -Evaluate for init k: 100 with d2s: exp with weight: (0.5, 0.5) (sift vs color) -map@1: 0.387713 map@5: 0.310307 map@10: 0.272218 hit_rate@1: 0.387713 hit_rate@5: 0.524915 hit_rate@10: 0.596587 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001259 -Evaluate for init k: 100 with d2s: exp with weight: (0.6, 0.4) (sift vs color) -map@1: 0.395904 map@5: 0.312901 map@10: 0.275358 hit_rate@1: 0.395904 hit_rate@5: 0.527645 hit_rate@10: 0.602048 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001197 -Evaluate for init k: 100 with d2s: exp with weight: (0.8, 0.2) (sift vs color) -map@1: 0.392491 map@5: 0.322457 map@10: 0.283072 hit_rate@1: 0.392491 hit_rate@5: 0.536519 hit_rate@10: 0.611604 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001179 -Evaluate for init k: 100 with d2s: log with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380205 map@5: 0.302116 map@10: 0.267304 hit_rate@1: 0.380205 hit_rate@5: 0.513993 hit_rate@10: 0.586348 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.019583 -Evaluate for init k: 100 with d2s: log with weight: (0.4, 0.6) (sift vs color) -map@1: 0.382253 map@5: 0.306348 map@10: 0.269556 hit_rate@1: 0.382253 hit_rate@5: 0.520137 hit_rate@10: 0.589761 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001294 -Evaluate for init k: 100 with d2s: log with weight: (0.5, 0.5) (sift vs color) -map@1: 0.387713 map@5: 0.310307 map@10: 0.272218 hit_rate@1: 0.387713 hit_rate@5: 0.524915 hit_rate@10: 0.596587 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.00123 -Evaluate for init k: 100 with d2s: log with weight: (0.6, 0.4) (sift vs color) -map@1: 0.395904 map@5: 0.312901 map@10: 0.275358 hit_rate@1: 0.395904 hit_rate@5: 0.527645 hit_rate@10: 0.602048 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001325 -Evaluate for init k: 100 with d2s: log with weight: (0.8, 0.2) (sift vs color) -map@1: 0.392491 map@5: 0.322457 map@10: 0.283072 hit_rate@1: 0.392491 hit_rate@5: 0.536519 hit_rate@10: 0.611604 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001253 -Evaluate for init k: 100 with d2s: logistic with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380205 map@5: 0.302116 map@10: 0.267304 hit_rate@1: 0.380205 hit_rate@5: 0.513993 hit_rate@10: 0.586348 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.019711 -Evaluate for init k: 100 with d2s: logistic with weight: (0.4, 0.6) (sift vs color) -map@1: 0.382253 map@5: 0.306348 map@10: 0.269556 hit_rate@1: 0.382253 hit_rate@5: 0.520137 hit_rate@10: 0.589761 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001219 -Evaluate for init k: 100 with d2s: logistic with weight: (0.5, 0.5) (sift vs color) -map@1: 0.387713 map@5: 0.310307 map@10: 0.272218 hit_rate@1: 0.387713 hit_rate@5: 0.524915 hit_rate@10: 0.596587 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001312 -Evaluate for init k: 100 with d2s: logistic with weight: (0.6, 0.4) (sift vs color) -map@1: 0.395904 map@5: 0.312901 map@10: 0.275358 hit_rate@1: 0.395904 hit_rate@5: 0.527645 hit_rate@10: 0.602048 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001332 -Evaluate for init k: 100 with d2s: logistic with weight: (0.8, 0.2) (sift vs color) -map@1: 0.392491 map@5: 0.322457 map@10: 0.283072 hit_rate@1: 0.392491 hit_rate@5: 0.536519 hit_rate@10: 0.611604 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001213 -Evaluate for init k: 100 with d2s: gaussian with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380205 map@5: 0.302116 map@10: 0.267304 hit_rate@1: 0.380205 hit_rate@5: 0.513993 hit_rate@10: 0.586348 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.019745 -Evaluate for init k: 100 with d2s: gaussian with weight: (0.4, 0.6) (sift vs color) -map@1: 0.382253 map@5: 0.306348 map@10: 0.269556 hit_rate@1: 0.382253 hit_rate@5: 0.520137 hit_rate@10: 0.589761 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001193 -Evaluate for init k: 100 with d2s: gaussian with weight: (0.5, 0.5) (sift vs color) -map@1: 0.387713 map@5: 0.310307 map@10: 0.272218 hit_rate@1: 0.387713 hit_rate@5: 0.524915 hit_rate@10: 0.596587 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001307 -Evaluate for init k: 100 with d2s: gaussian with weight: (0.6, 0.4) (sift vs color) -map@1: 0.395904 map@5: 0.312901 map@10: 0.275358 hit_rate@1: 0.395904 hit_rate@5: 0.527645 hit_rate@10: 0.602048 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001326 -Evaluate for init k: 100 with d2s: gaussian with weight: (0.8, 0.2) (sift vs color) -map@1: 0.392491 map@5: 0.322457 map@10: 0.283072 hit_rate@1: 0.392491 hit_rate@5: 0.536519 hit_rate@10: 0.611604 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.00135 -Evaluate for init k: 100 with d2s: inverse with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380205 map@5: 0.302116 map@10: 0.267304 hit_rate@1: 0.380205 hit_rate@5: 0.513993 hit_rate@10: 0.586348 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.019797 -Evaluate for init k: 100 with d2s: inverse with weight: (0.4, 0.6) (sift vs color) -map@1: 0.382253 map@5: 0.306348 map@10: 0.269556 hit_rate@1: 0.382253 hit_rate@5: 0.520137 hit_rate@10: 0.589761 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001224 -Evaluate for init k: 100 with d2s: inverse with weight: (0.5, 0.5) (sift vs color) -map@1: 0.387713 map@5: 0.310307 map@10: 0.272218 hit_rate@1: 0.387713 hit_rate@5: 0.524915 hit_rate@10: 0.596587 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001354 -Evaluate for init k: 100 with d2s: inverse with weight: (0.6, 0.4) (sift vs color) -map@1: 0.395904 map@5: 0.312901 map@10: 0.275358 hit_rate@1: 0.395904 hit_rate@5: 0.527645 hit_rate@10: 0.602048 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001285 -Evaluate for init k: 100 with d2s: inverse with weight: (0.8, 0.2) (sift vs color) -map@1: 0.392491 map@5: 0.322457 map@10: 0.283072 hit_rate@1: 0.392491 hit_rate@5: 0.536519 hit_rate@10: 0.611604 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001363 -Evaluate for init k: 1000 with d2s: exp with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380887 map@5: 0.303345 map@10: 0.268259 hit_rate@1: 0.380887 hit_rate@5: 0.517406 hit_rate@10: 0.586348 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.02034 -Evaluate for init k: 1000 with d2s: exp with weight: (0.4, 0.6) (sift vs color) -map@1: 0.3843 map@5: 0.308805 map@10: 0.272287 hit_rate@1: 0.3843 hit_rate@5: 0.523549 hit_rate@10: 0.593857 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001552 -Evaluate for init k: 1000 with d2s: exp with weight: (0.5, 0.5) (sift vs color) -map@1: 0.389761 map@5: 0.312491 map@10: 0.276451 hit_rate@1: 0.389761 hit_rate@5: 0.524915 hit_rate@10: 0.598635 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001538 -Evaluate for init k: 1000 with d2s: exp with weight: (0.6, 0.4) (sift vs color) -map@1: 0.395222 map@5: 0.316724 map@10: 0.281365 hit_rate@1: 0.395222 hit_rate@5: 0.531741 hit_rate@10: 0.604096 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001529 -Evaluate for init k: 1000 with d2s: exp with weight: (0.8, 0.2) (sift vs color) -map@1: 0.402048 map@5: 0.328055 map@10: 0.296246 hit_rate@1: 0.402048 hit_rate@5: 0.539932 hit_rate@10: 0.61843 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001522 -Evaluate for init k: 1000 with d2s: log with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380887 map@5: 0.303345 map@10: 0.268259 hit_rate@1: 0.380887 hit_rate@5: 0.517406 hit_rate@10: 0.586348 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.020417 -Evaluate for init k: 1000 with d2s: log with weight: (0.4, 0.6) (sift vs color) -map@1: 0.3843 map@5: 0.308805 map@10: 0.272287 hit_rate@1: 0.3843 hit_rate@5: 0.523549 hit_rate@10: 0.593857 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001539 -Evaluate for init k: 1000 with d2s: log with weight: (0.5, 0.5) (sift vs color) -map@1: 0.389761 map@5: 0.312491 map@10: 0.276451 hit_rate@1: 0.389761 hit_rate@5: 0.524915 hit_rate@10: 0.598635 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001527 -Evaluate for init k: 1000 with d2s: log with weight: (0.6, 0.4) (sift vs color) -map@1: 0.395222 map@5: 0.316724 map@10: 0.281365 hit_rate@1: 0.395222 hit_rate@5: 0.531741 hit_rate@10: 0.604096 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001522 -Evaluate for init k: 1000 with d2s: log with weight: (0.8, 0.2) (sift vs color) -map@1: 0.402048 map@5: 0.328055 map@10: 0.296246 hit_rate@1: 0.402048 hit_rate@5: 0.539932 hit_rate@10: 0.61843 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001513 -Evaluate for init k: 1000 with d2s: logistic with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380887 map@5: 0.303345 map@10: 0.268259 hit_rate@1: 0.380887 hit_rate@5: 0.517406 hit_rate@10: 0.586348 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.020387 -Evaluate for init k: 1000 with d2s: logistic with weight: (0.4, 0.6) (sift vs color) -map@1: 0.3843 map@5: 0.308805 map@10: 0.272287 hit_rate@1: 0.3843 hit_rate@5: 0.523549 hit_rate@10: 0.593857 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001394 -Evaluate for init k: 1000 with d2s: logistic with weight: (0.5, 0.5) (sift vs color) -map@1: 0.389761 map@5: 0.312491 map@10: 0.276451 hit_rate@1: 0.389761 hit_rate@5: 0.524915 hit_rate@10: 0.598635 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.00157 -Evaluate for init k: 1000 with d2s: logistic with weight: (0.6, 0.4) (sift vs color) -map@1: 0.395222 map@5: 0.316724 map@10: 0.281365 hit_rate@1: 0.395222 hit_rate@5: 0.531741 hit_rate@10: 0.604096 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001587 -Evaluate for init k: 1000 with d2s: logistic with weight: (0.8, 0.2) (sift vs color) -map@1: 0.402048 map@5: 0.328055 map@10: 0.296246 hit_rate@1: 0.402048 hit_rate@5: 0.539932 hit_rate@10: 0.61843 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001605 -Evaluate for init k: 1000 with d2s: gaussian with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380887 map@5: 0.303345 map@10: 0.268259 hit_rate@1: 0.380887 hit_rate@5: 0.517406 hit_rate@10: 0.586348 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.020298 -Evaluate for init k: 1000 with d2s: gaussian with weight: (0.4, 0.6) (sift vs color) -map@1: 0.3843 map@5: 0.308805 map@10: 0.272287 hit_rate@1: 0.3843 hit_rate@5: 0.523549 hit_rate@10: 0.593857 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.00169 -Evaluate for init k: 1000 with d2s: gaussian with weight: (0.5, 0.5) (sift vs color) -map@1: 0.389761 map@5: 0.312491 map@10: 0.276451 hit_rate@1: 0.389761 hit_rate@5: 0.524915 hit_rate@10: 0.598635 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001463 -Evaluate for init k: 1000 with d2s: gaussian with weight: (0.6, 0.4) (sift vs color) -map@1: 0.395222 map@5: 0.316724 map@10: 0.281365 hit_rate@1: 0.395222 hit_rate@5: 0.531741 hit_rate@10: 0.604096 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001432 -Evaluate for init k: 1000 with d2s: gaussian with weight: (0.8, 0.2) (sift vs color) -map@1: 0.402048 map@5: 0.328055 map@10: 0.296246 hit_rate@1: 0.402048 hit_rate@5: 0.539932 hit_rate@10: 0.61843 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001657 -Evaluate for init k: 1000 with d2s: inverse with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380887 map@5: 0.303345 map@10: 0.268259 hit_rate@1: 0.380887 hit_rate@5: 0.517406 hit_rate@10: 0.586348 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.020334 -Evaluate for init k: 1000 with d2s: inverse with weight: (0.4, 0.6) (sift vs color) -map@1: 0.3843 map@5: 0.308805 map@10: 0.272287 hit_rate@1: 0.3843 hit_rate@5: 0.523549 hit_rate@10: 0.593857 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001483 -Evaluate for init k: 1000 with d2s: inverse with weight: (0.5, 0.5) (sift vs color) -map@1: 0.389761 map@5: 0.312491 map@10: 0.276451 hit_rate@1: 0.389761 hit_rate@5: 0.524915 hit_rate@10: 0.598635 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001779 -Evaluate for init k: 1000 with d2s: inverse with weight: (0.6, 0.4) (sift vs color) -map@1: 0.395222 map@5: 0.316724 map@10: 0.281365 hit_rate@1: 0.395222 hit_rate@5: 0.531741 hit_rate@10: 0.604096 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.001509 -Evaluate for init k: 1000 with d2s: inverse with weight: (0.8, 0.2) (sift vs color) -map@1: 0.402048 map@5: 0.328055 map@10: 0.296246 hit_rate@1: 0.402048 hit_rate@5: 0.539932 hit_rate@10: 0.61843 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.00148 -Evaluate for init k: 6353 with d2s: exp with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380887 map@5: 0.302662 map@10: 0.267986 hit_rate@1: 0.380887 hit_rate@5: 0.516724 hit_rate@10: 0.585666 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.023527 -Evaluate for init k: 6353 with d2s: exp with weight: (0.4, 0.6) (sift vs color) -map@1: 0.382253 map@5: 0.308396 map@10: 0.271536 hit_rate@1: 0.382253 hit_rate@5: 0.523549 hit_rate@10: 0.593174 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.003392 -Evaluate for init k: 6353 with d2s: exp with weight: (0.5, 0.5) (sift vs color) -map@1: 0.387713 map@5: 0.311945 map@10: 0.274812 hit_rate@1: 0.387713 hit_rate@5: 0.52628 hit_rate@10: 0.59727 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.002839 -Evaluate for init k: 6353 with d2s: exp with weight: (0.6, 0.4) (sift vs color) -map@1: 0.394539 map@5: 0.316177 map@10: 0.280341 hit_rate@1: 0.394539 hit_rate@5: 0.532423 hit_rate@10: 0.60273 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.003343 -Evaluate for init k: 6353 with d2s: exp with weight: (0.8, 0.2) (sift vs color) -map@1: 0.402048 map@5: 0.325461 map@10: 0.295017 hit_rate@1: 0.402048 hit_rate@5: 0.535836 hit_rate@10: 0.617747 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.003434 -Evaluate for init k: 6353 with d2s: log with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380887 map@5: 0.302662 map@10: 0.267986 hit_rate@1: 0.380887 hit_rate@5: 0.516724 hit_rate@10: 0.585666 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.023525 -Evaluate for init k: 6353 with d2s: log with weight: (0.4, 0.6) (sift vs color) -map@1: 0.382253 map@5: 0.308396 map@10: 0.271536 hit_rate@1: 0.382253 hit_rate@5: 0.523549 hit_rate@10: 0.593174 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.004414 -Evaluate for init k: 6353 with d2s: log with weight: (0.5, 0.5) (sift vs color) -map@1: 0.387713 map@5: 0.311945 map@10: 0.274812 hit_rate@1: 0.387713 hit_rate@5: 0.52628 hit_rate@10: 0.59727 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.003072 -Evaluate for init k: 6353 with d2s: log with weight: (0.6, 0.4) (sift vs color) -map@1: 0.394539 map@5: 0.316177 map@10: 0.280341 hit_rate@1: 0.394539 hit_rate@5: 0.532423 hit_rate@10: 0.60273 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.002821 -Evaluate for init k: 6353 with d2s: log with weight: (0.8, 0.2) (sift vs color) -map@1: 0.402048 map@5: 0.325461 map@10: 0.295017 hit_rate@1: 0.402048 hit_rate@5: 0.535836 hit_rate@10: 0.617747 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.003754 -Evaluate for init k: 6353 with d2s: logistic with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380887 map@5: 0.302662 map@10: 0.267986 hit_rate@1: 0.380887 hit_rate@5: 0.516724 hit_rate@10: 0.585666 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.023343 -Evaluate for init k: 6353 with d2s: logistic with weight: (0.4, 0.6) (sift vs color) -map@1: 0.382253 map@5: 0.308396 map@10: 0.271536 hit_rate@1: 0.382253 hit_rate@5: 0.523549 hit_rate@10: 0.593174 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.00468 -Evaluate for init k: 6353 with d2s: logistic with weight: (0.5, 0.5) (sift vs color) -map@1: 0.387713 map@5: 0.311945 map@10: 0.274812 hit_rate@1: 0.387713 hit_rate@5: 0.52628 hit_rate@10: 0.59727 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.002964 -Evaluate for init k: 6353 with d2s: logistic with weight: (0.6, 0.4) (sift vs color) -map@1: 0.394539 map@5: 0.316177 map@10: 0.280341 hit_rate@1: 0.394539 hit_rate@5: 0.532423 hit_rate@10: 0.60273 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.002862 -Evaluate for init k: 6353 with d2s: logistic with weight: (0.8, 0.2) (sift vs color) -map@1: 0.402048 map@5: 0.325461 map@10: 0.295017 hit_rate@1: 0.402048 hit_rate@5: 0.535836 hit_rate@10: 0.617747 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.004115 -Evaluate for init k: 6353 with d2s: gaussian with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380887 map@5: 0.302662 map@10: 0.267986 hit_rate@1: 0.380887 hit_rate@5: 0.516724 hit_rate@10: 0.585666 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.023535 -Evaluate for init k: 6353 with d2s: gaussian with weight: (0.4, 0.6) (sift vs color) -map@1: 0.382253 map@5: 0.308396 map@10: 0.271536 hit_rate@1: 0.382253 hit_rate@5: 0.523549 hit_rate@10: 0.593174 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.002947 -Evaluate for init k: 6353 with d2s: gaussian with weight: (0.5, 0.5) (sift vs color) -map@1: 0.387713 map@5: 0.311945 map@10: 0.274812 hit_rate@1: 0.387713 hit_rate@5: 0.52628 hit_rate@10: 0.59727 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.002871 -Evaluate for init k: 6353 with d2s: gaussian with weight: (0.6, 0.4) (sift vs color) -map@1: 0.394539 map@5: 0.316177 map@10: 0.280341 hit_rate@1: 0.394539 hit_rate@5: 0.532423 hit_rate@10: 0.60273 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.004603 -Evaluate for init k: 6353 with d2s: gaussian with weight: (0.8, 0.2) (sift vs color) -map@1: 0.402048 map@5: 0.325461 map@10: 0.295017 hit_rate@1: 0.402048 hit_rate@5: 0.535836 hit_rate@10: 0.617747 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.002983 -Evaluate for init k: 6353 with d2s: inverse with weight: (0.2, 0.8) (sift vs color) -map@1: 0.380887 map@5: 0.302662 map@10: 0.267986 hit_rate@1: 0.380887 hit_rate@5: 0.516724 hit_rate@10: 0.585666 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.023903 -Evaluate for init k: 6353 with d2s: inverse with weight: (0.4, 0.6) (sift vs color) -map@1: 0.382253 map@5: 0.308396 map@10: 0.271536 hit_rate@1: 0.382253 hit_rate@5: 0.523549 hit_rate@10: 0.593174 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.003559 -Evaluate for init k: 6353 with d2s: inverse with weight: (0.5, 0.5) (sift vs color) -map@1: 0.387713 map@5: 0.311945 map@10: 0.274812 hit_rate@1: 0.387713 hit_rate@5: 0.52628 hit_rate@10: 0.59727 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.002912 -Evaluate for init k: 6353 with d2s: inverse with weight: (0.6, 0.4) (sift vs color) -map@1: 0.394539 map@5: 0.316177 map@10: 0.280341 hit_rate@1: 0.394539 hit_rate@5: 0.532423 hit_rate@10: 0.60273 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.004916 -Evaluate for init k: 6353 with d2s: inverse with weight: (0.8, 0.2) (sift vs color) -map@1: 0.402048 map@5: 0.325461 map@10: 0.295017 hit_rate@1: 0.402048 hit_rate@5: 0.535836 hit_rate@10: 0.617747 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.008771 avg_retrieval_time: 0.003062 -Fitting BOW for n_clusters kmeans: 96 -Fit Kmeans clustering to create BOW -Fit IDF for TF-IDF Transformation -Complete Fitting SIFT BOW Extractor -Evaluate for init k: 6353 with d2s: exp with weight: (0.16, 0.04, 0.8) (sift vs color vs resnet) -map@1: 0.879863 map@5: 0.827304 map@10: 0.795017 hit_rate@1: 0.879863 hit_rate@5: 0.965188 hit_rate@10: 0.974061 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.012997 avg_retrieval_time: 0.043442 -Evaluate for init k: 6353 with d2s: exp with weight: (0.32, 0.08, 0.6) (sift vs color vs resnet) -map@1: 0.869625 map@5: 0.814334 map@10: 0.780819 hit_rate@1: 0.869625 hit_rate@5: 0.956997 hit_rate@10: 0.973379 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.012997 avg_retrieval_time: 0.003675 -Evaluate for init k: 6353 with d2s: exp with weight: (0.4, 0.1, 0.5) (sift vs color vs resnet) -map@1: 0.85802 map@5: 0.803413 map@10: 0.764642 hit_rate@1: 0.85802 hit_rate@5: 0.952901 hit_rate@10: 0.969283 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.012997 avg_retrieval_time: 0.004133 -Evaluate for init k: 6353 with d2s: exp with weight: (0.48, 0.12, 0.4) (sift vs color vs resnet) -map@1: 0.838225 map@5: 0.778703 map@10: 0.736997 hit_rate@1: 0.838225 hit_rate@5: 0.944027 hit_rate@10: 0.965188 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.012997 avg_retrieval_time: 0.004157 -Evaluate for init k: 6353 with d2s: exp with weight: (0.64, 0.16, 0.2) (sift vs color vs resnet) -map@1: 0.745392 map@5: 0.672765 map@10: 0.62157 hit_rate@1: 0.745392 hit_rate@5: 0.884642 hit_rate@10: 0.924232 recall@1: 0.0 recall@5: 0.0 recall@10: 0.0 recall@100: 0.0 recall@1000: 0.0 avg_indexing_time: 0.012997 avg_retrieval_time: 0.003683 diff --git a/evaluation.sh b/evaluation.sh index ff32a6d..b234304 100644 --- a/evaluation.sh +++ b/evaluation.sh @@ -1,5 +1,5 @@ # TQDM_DISABLE=1 python histogram_knn_eval.py # TQDM_DISABLE=1 python sift_knn_eval.py # TQDM_DISABLE=1 python resnet_knn_gpu_eval.py -TQDM_DISABLE=1 python ensemble_sift_color.py +# TQDM_DISABLE=1 python ensemble_sift_color.py TQDM_DISABLE=1 python ensemble_all.py \ No newline at end of file diff --git a/histogram_knn_eval.py b/histogram_knn_eval.py index 677b624..e9c7c5e 100644 --- a/histogram_knn_eval.py +++ b/histogram_knn_eval.py @@ -61,6 +61,7 @@ columns=[ "bins", "htype", + "slice", "metric", "map@1", "map@5", @@ -80,12 +81,13 @@ n_bins = [2, 4, 8, 12] h_types = ["region", "global"] +n_slices = [2, 3, 4, 5] knn_metrics = ["euclidean", "cosine"] -for bins, h_type, metric in grid(n_bins, h_types, knn_metrics): - print("Evaluate for bins: ", bins, " h_type: ", h_type, "", " with knn metric: ", metric) +for bins, h_type, n_slice, metric in grid(n_bins, h_types, n_slices, knn_metrics): + print("Evaluate for bins: ", bins, " h_type: ", h_type, " n_slice: ", n_slice, " with knn metric: ", metric) # Initialization - rgb_histogram = RGBHistogram(n_bin=bins, h_type=h_type) + rgb_histogram = RGBHistogram(n_bin=bins, n_slice=n_slice, h_type=h_type) array_store = NPArrayStore(retrieve=KNNRetrieval(metric=metric)) cbir = CBIR(rgb_histogram, array_store) @@ -152,6 +154,7 @@ { "bins": [bins], "htype": [h_type], + "slice": [n_slice], "metric": [metric], "map@1": [map1], "map@5": [map5], diff --git a/out/ensemble_sift_color_knn_eval.csv b/out/ensemble_sift_color_knn_eval.csv index 71538d9..d01e967 100644 --- a/out/ensemble_sift_color_knn_eval.csv +++ b/out/ensemble_sift_color_knn_eval.csv @@ -1,76 +1,76 @@ k,distance2score,weight(sift vs color),map@1,map@5,map@10,hit_rate@1,hit_rate@5,hit_rate@10,recall@1,recall@5,recall@10,recall@100,recall@1000,avg_indexing_time,avg_retrieval_time -100,exp,"(0.2, 0.8)",0.380205,0.302116,0.267304,0.380205,0.513993,0.586348,0.0,0.0,0.0,0.0,0.0,0.008771,0.019569 -100,exp,"(0.4, 0.6)",0.382253,0.306348,0.269556,0.382253,0.520137,0.589761,0.0,0.0,0.0,0.0,0.0,0.008771,0.001201 -100,exp,"(0.5, 0.5)",0.387713,0.310307,0.272218,0.387713,0.524915,0.596587,0.0,0.0,0.0,0.0,0.0,0.008771,0.001259 -100,exp,"(0.6, 0.4)",0.395904,0.312901,0.275358,0.395904,0.527645,0.602048,0.0,0.0,0.0,0.0,0.0,0.008771,0.001197 -100,exp,"(0.8, 0.2)",0.392491,0.322457,0.283072,0.392491,0.536519,0.611604,0.0,0.0,0.0,0.0,0.0,0.008771,0.001179 -100,log,"(0.2, 0.8)",0.380205,0.302116,0.267304,0.380205,0.513993,0.586348,0.0,0.0,0.0,0.0,0.0,0.008771,0.019583 -100,log,"(0.4, 0.6)",0.382253,0.306348,0.269556,0.382253,0.520137,0.589761,0.0,0.0,0.0,0.0,0.0,0.008771,0.001294 -100,log,"(0.5, 0.5)",0.387713,0.310307,0.272218,0.387713,0.524915,0.596587,0.0,0.0,0.0,0.0,0.0,0.008771,0.00123 -100,log,"(0.6, 0.4)",0.395904,0.312901,0.275358,0.395904,0.527645,0.602048,0.0,0.0,0.0,0.0,0.0,0.008771,0.001325 -100,log,"(0.8, 0.2)",0.392491,0.322457,0.283072,0.392491,0.536519,0.611604,0.0,0.0,0.0,0.0,0.0,0.008771,0.001253 -100,logistic,"(0.2, 0.8)",0.380205,0.302116,0.267304,0.380205,0.513993,0.586348,0.0,0.0,0.0,0.0,0.0,0.008771,0.019711 -100,logistic,"(0.4, 0.6)",0.382253,0.306348,0.269556,0.382253,0.520137,0.589761,0.0,0.0,0.0,0.0,0.0,0.008771,0.001219 -100,logistic,"(0.5, 0.5)",0.387713,0.310307,0.272218,0.387713,0.524915,0.596587,0.0,0.0,0.0,0.0,0.0,0.008771,0.001312 -100,logistic,"(0.6, 0.4)",0.395904,0.312901,0.275358,0.395904,0.527645,0.602048,0.0,0.0,0.0,0.0,0.0,0.008771,0.001332 -100,logistic,"(0.8, 0.2)",0.392491,0.322457,0.283072,0.392491,0.536519,0.611604,0.0,0.0,0.0,0.0,0.0,0.008771,0.001213 -100,gaussian,"(0.2, 0.8)",0.380205,0.302116,0.267304,0.380205,0.513993,0.586348,0.0,0.0,0.0,0.0,0.0,0.008771,0.019745 -100,gaussian,"(0.4, 0.6)",0.382253,0.306348,0.269556,0.382253,0.520137,0.589761,0.0,0.0,0.0,0.0,0.0,0.008771,0.001193 -100,gaussian,"(0.5, 0.5)",0.387713,0.310307,0.272218,0.387713,0.524915,0.596587,0.0,0.0,0.0,0.0,0.0,0.008771,0.001307 -100,gaussian,"(0.6, 0.4)",0.395904,0.312901,0.275358,0.395904,0.527645,0.602048,0.0,0.0,0.0,0.0,0.0,0.008771,0.001326 -100,gaussian,"(0.8, 0.2)",0.392491,0.322457,0.283072,0.392491,0.536519,0.611604,0.0,0.0,0.0,0.0,0.0,0.008771,0.00135 -100,inverse,"(0.2, 0.8)",0.380205,0.302116,0.267304,0.380205,0.513993,0.586348,0.0,0.0,0.0,0.0,0.0,0.008771,0.019797 -100,inverse,"(0.4, 0.6)",0.382253,0.306348,0.269556,0.382253,0.520137,0.589761,0.0,0.0,0.0,0.0,0.0,0.008771,0.001224 -100,inverse,"(0.5, 0.5)",0.387713,0.310307,0.272218,0.387713,0.524915,0.596587,0.0,0.0,0.0,0.0,0.0,0.008771,0.001354 -100,inverse,"(0.6, 0.4)",0.395904,0.312901,0.275358,0.395904,0.527645,0.602048,0.0,0.0,0.0,0.0,0.0,0.008771,0.001285 -100,inverse,"(0.8, 0.2)",0.392491,0.322457,0.283072,0.392491,0.536519,0.611604,0.0,0.0,0.0,0.0,0.0,0.008771,0.001363 -1000,exp,"(0.2, 0.8)",0.380887,0.303345,0.268259,0.380887,0.517406,0.586348,0.0,0.0,0.0,0.0,0.0,0.008771,0.02034 -1000,exp,"(0.4, 0.6)",0.3843,0.308805,0.272287,0.3843,0.523549,0.593857,0.0,0.0,0.0,0.0,0.0,0.008771,0.001552 -1000,exp,"(0.5, 0.5)",0.389761,0.312491,0.276451,0.389761,0.524915,0.598635,0.0,0.0,0.0,0.0,0.0,0.008771,0.001538 -1000,exp,"(0.6, 0.4)",0.395222,0.316724,0.281365,0.395222,0.531741,0.604096,0.0,0.0,0.0,0.0,0.0,0.008771,0.001529 -1000,exp,"(0.8, 0.2)",0.402048,0.328055,0.296246,0.402048,0.539932,0.61843,0.0,0.0,0.0,0.0,0.0,0.008771,0.001522 -1000,log,"(0.2, 0.8)",0.380887,0.303345,0.268259,0.380887,0.517406,0.586348,0.0,0.0,0.0,0.0,0.0,0.008771,0.020417 -1000,log,"(0.4, 0.6)",0.3843,0.308805,0.272287,0.3843,0.523549,0.593857,0.0,0.0,0.0,0.0,0.0,0.008771,0.001539 -1000,log,"(0.5, 0.5)",0.389761,0.312491,0.276451,0.389761,0.524915,0.598635,0.0,0.0,0.0,0.0,0.0,0.008771,0.001527 -1000,log,"(0.6, 0.4)",0.395222,0.316724,0.281365,0.395222,0.531741,0.604096,0.0,0.0,0.0,0.0,0.0,0.008771,0.001522 -1000,log,"(0.8, 0.2)",0.402048,0.328055,0.296246,0.402048,0.539932,0.61843,0.0,0.0,0.0,0.0,0.0,0.008771,0.001513 -1000,logistic,"(0.2, 0.8)",0.380887,0.303345,0.268259,0.380887,0.517406,0.586348,0.0,0.0,0.0,0.0,0.0,0.008771,0.020387 -1000,logistic,"(0.4, 0.6)",0.3843,0.308805,0.272287,0.3843,0.523549,0.593857,0.0,0.0,0.0,0.0,0.0,0.008771,0.001394 -1000,logistic,"(0.5, 0.5)",0.389761,0.312491,0.276451,0.389761,0.524915,0.598635,0.0,0.0,0.0,0.0,0.0,0.008771,0.00157 -1000,logistic,"(0.6, 0.4)",0.395222,0.316724,0.281365,0.395222,0.531741,0.604096,0.0,0.0,0.0,0.0,0.0,0.008771,0.001587 -1000,logistic,"(0.8, 0.2)",0.402048,0.328055,0.296246,0.402048,0.539932,0.61843,0.0,0.0,0.0,0.0,0.0,0.008771,0.001605 -1000,gaussian,"(0.2, 0.8)",0.380887,0.303345,0.268259,0.380887,0.517406,0.586348,0.0,0.0,0.0,0.0,0.0,0.008771,0.020298 -1000,gaussian,"(0.4, 0.6)",0.3843,0.308805,0.272287,0.3843,0.523549,0.593857,0.0,0.0,0.0,0.0,0.0,0.008771,0.00169 -1000,gaussian,"(0.5, 0.5)",0.389761,0.312491,0.276451,0.389761,0.524915,0.598635,0.0,0.0,0.0,0.0,0.0,0.008771,0.001463 -1000,gaussian,"(0.6, 0.4)",0.395222,0.316724,0.281365,0.395222,0.531741,0.604096,0.0,0.0,0.0,0.0,0.0,0.008771,0.001432 -1000,gaussian,"(0.8, 0.2)",0.402048,0.328055,0.296246,0.402048,0.539932,0.61843,0.0,0.0,0.0,0.0,0.0,0.008771,0.001657 -1000,inverse,"(0.2, 0.8)",0.380887,0.303345,0.268259,0.380887,0.517406,0.586348,0.0,0.0,0.0,0.0,0.0,0.008771,0.020334 -1000,inverse,"(0.4, 0.6)",0.3843,0.308805,0.272287,0.3843,0.523549,0.593857,0.0,0.0,0.0,0.0,0.0,0.008771,0.001483 -1000,inverse,"(0.5, 0.5)",0.389761,0.312491,0.276451,0.389761,0.524915,0.598635,0.0,0.0,0.0,0.0,0.0,0.008771,0.001779 -1000,inverse,"(0.6, 0.4)",0.395222,0.316724,0.281365,0.395222,0.531741,0.604096,0.0,0.0,0.0,0.0,0.0,0.008771,0.001509 -1000,inverse,"(0.8, 0.2)",0.402048,0.328055,0.296246,0.402048,0.539932,0.61843,0.0,0.0,0.0,0.0,0.0,0.008771,0.00148 -6353,exp,"(0.2, 0.8)",0.380887,0.302662,0.267986,0.380887,0.516724,0.585666,0.0,0.0,0.0,0.0,0.0,0.008771,0.023527 -6353,exp,"(0.4, 0.6)",0.382253,0.308396,0.271536,0.382253,0.523549,0.593174,0.0,0.0,0.0,0.0,0.0,0.008771,0.003392 -6353,exp,"(0.5, 0.5)",0.387713,0.311945,0.274812,0.387713,0.52628,0.59727,0.0,0.0,0.0,0.0,0.0,0.008771,0.002839 -6353,exp,"(0.6, 0.4)",0.394539,0.316177,0.280341,0.394539,0.532423,0.60273,0.0,0.0,0.0,0.0,0.0,0.008771,0.003343 -6353,exp,"(0.8, 0.2)",0.402048,0.325461,0.295017,0.402048,0.535836,0.617747,0.0,0.0,0.0,0.0,0.0,0.008771,0.003434 -6353,log,"(0.2, 0.8)",0.380887,0.302662,0.267986,0.380887,0.516724,0.585666,0.0,0.0,0.0,0.0,0.0,0.008771,0.023525 -6353,log,"(0.4, 0.6)",0.382253,0.308396,0.271536,0.382253,0.523549,0.593174,0.0,0.0,0.0,0.0,0.0,0.008771,0.004414 -6353,log,"(0.5, 0.5)",0.387713,0.311945,0.274812,0.387713,0.52628,0.59727,0.0,0.0,0.0,0.0,0.0,0.008771,0.003072 -6353,log,"(0.6, 0.4)",0.394539,0.316177,0.280341,0.394539,0.532423,0.60273,0.0,0.0,0.0,0.0,0.0,0.008771,0.002821 -6353,log,"(0.8, 0.2)",0.402048,0.325461,0.295017,0.402048,0.535836,0.617747,0.0,0.0,0.0,0.0,0.0,0.008771,0.003754 -6353,logistic,"(0.2, 0.8)",0.380887,0.302662,0.267986,0.380887,0.516724,0.585666,0.0,0.0,0.0,0.0,0.0,0.008771,0.023343 -6353,logistic,"(0.4, 0.6)",0.382253,0.308396,0.271536,0.382253,0.523549,0.593174,0.0,0.0,0.0,0.0,0.0,0.008771,0.00468 -6353,logistic,"(0.5, 0.5)",0.387713,0.311945,0.274812,0.387713,0.52628,0.59727,0.0,0.0,0.0,0.0,0.0,0.008771,0.002964 -6353,logistic,"(0.6, 0.4)",0.394539,0.316177,0.280341,0.394539,0.532423,0.60273,0.0,0.0,0.0,0.0,0.0,0.008771,0.002862 -6353,logistic,"(0.8, 0.2)",0.402048,0.325461,0.295017,0.402048,0.535836,0.617747,0.0,0.0,0.0,0.0,0.0,0.008771,0.004115 -6353,gaussian,"(0.2, 0.8)",0.380887,0.302662,0.267986,0.380887,0.516724,0.585666,0.0,0.0,0.0,0.0,0.0,0.008771,0.023535 -6353,gaussian,"(0.4, 0.6)",0.382253,0.308396,0.271536,0.382253,0.523549,0.593174,0.0,0.0,0.0,0.0,0.0,0.008771,0.002947 -6353,gaussian,"(0.5, 0.5)",0.387713,0.311945,0.274812,0.387713,0.52628,0.59727,0.0,0.0,0.0,0.0,0.0,0.008771,0.002871 -6353,gaussian,"(0.6, 0.4)",0.394539,0.316177,0.280341,0.394539,0.532423,0.60273,0.0,0.0,0.0,0.0,0.0,0.008771,0.004603 -6353,gaussian,"(0.8, 0.2)",0.402048,0.325461,0.295017,0.402048,0.535836,0.617747,0.0,0.0,0.0,0.0,0.0,0.008771,0.002983 -6353,inverse,"(0.2, 0.8)",0.380887,0.302662,0.267986,0.380887,0.516724,0.585666,0.0,0.0,0.0,0.0,0.0,0.008771,0.023903 -6353,inverse,"(0.4, 0.6)",0.382253,0.308396,0.271536,0.382253,0.523549,0.593174,0.0,0.0,0.0,0.0,0.0,0.008771,0.003559 -6353,inverse,"(0.5, 0.5)",0.387713,0.311945,0.274812,0.387713,0.52628,0.59727,0.0,0.0,0.0,0.0,0.0,0.008771,0.002912 -6353,inverse,"(0.6, 0.4)",0.394539,0.316177,0.280341,0.394539,0.532423,0.60273,0.0,0.0,0.0,0.0,0.0,0.008771,0.004916 -6353,inverse,"(0.8, 0.2)",0.402048,0.325461,0.295017,0.402048,0.535836,0.617747,0.0,0.0,0.0,0.0,0.0,0.008771,0.003062 +100,exp,"(0.2, 0.8)",0.198635,0.31428,0.331616,0.39727,0.322048,0.285597,0.003983,0.013779,0.022572,0.09315,0.266744,0.008954,0.0255 +100,exp,"(0.4, 0.6)",0.203072,0.318663,0.33624,0.406143,0.324778,0.289078,0.004175,0.013922,0.022992,0.09315,0.266744,0.008954,0.001288 +100,exp,"(0.5, 0.5)",0.206826,0.321197,0.338386,0.413652,0.327099,0.290853,0.004342,0.014118,0.023173,0.09315,0.266744,0.008954,0.001253 +100,exp,"(0.6, 0.4)",0.206485,0.323003,0.339997,0.412969,0.331195,0.294539,0.004288,0.014505,0.023846,0.09315,0.266744,0.008954,0.001244 +100,exp,"(0.8, 0.2)",0.206143,0.325448,0.343636,0.412287,0.336928,0.300137,0.004252,0.014975,0.024643,0.093171,0.266744,0.008954,0.001252 +100,log,"(0.2, 0.8)",0.198635,0.31428,0.331616,0.39727,0.322048,0.285597,0.003983,0.013779,0.022572,0.09315,0.266744,0.008954,0.025786 +100,log,"(0.4, 0.6)",0.203072,0.318663,0.33624,0.406143,0.324778,0.289078,0.004175,0.013922,0.022992,0.09315,0.266744,0.008954,0.001284 +100,log,"(0.5, 0.5)",0.206826,0.321197,0.338386,0.413652,0.327099,0.290853,0.004342,0.014118,0.023173,0.09315,0.266744,0.008954,0.0012 +100,log,"(0.6, 0.4)",0.206485,0.323003,0.339997,0.412969,0.331195,0.294539,0.004288,0.014505,0.023846,0.09315,0.266744,0.008954,0.001213 +100,log,"(0.8, 0.2)",0.206143,0.325448,0.343636,0.412287,0.336928,0.300137,0.004252,0.014975,0.024643,0.093171,0.266744,0.008954,0.00133 +100,logistic,"(0.2, 0.8)",0.198635,0.31428,0.331616,0.39727,0.322048,0.285597,0.003983,0.013779,0.022572,0.09315,0.266744,0.008954,0.025714 +100,logistic,"(0.4, 0.6)",0.203072,0.318663,0.33624,0.406143,0.324778,0.289078,0.004175,0.013922,0.022992,0.09315,0.266744,0.008954,0.001213 +100,logistic,"(0.5, 0.5)",0.206826,0.321197,0.338386,0.413652,0.327099,0.290853,0.004342,0.014118,0.023173,0.09315,0.266744,0.008954,0.001169 +100,logistic,"(0.6, 0.4)",0.206485,0.323003,0.339997,0.412969,0.331195,0.294539,0.004288,0.014505,0.023846,0.09315,0.266744,0.008954,0.001278 +100,logistic,"(0.8, 0.2)",0.206143,0.325448,0.343636,0.412287,0.336928,0.300137,0.004252,0.014975,0.024643,0.093171,0.266744,0.008954,0.001246 +100,gaussian,"(0.2, 0.8)",0.198635,0.31428,0.331616,0.39727,0.322048,0.285597,0.003983,0.013779,0.022572,0.09315,0.266744,0.008954,0.025725 +100,gaussian,"(0.4, 0.6)",0.203072,0.318663,0.33624,0.406143,0.324778,0.289078,0.004175,0.013922,0.022992,0.09315,0.266744,0.008954,0.001146 +100,gaussian,"(0.5, 0.5)",0.206826,0.321197,0.338386,0.413652,0.327099,0.290853,0.004342,0.014118,0.023173,0.09315,0.266744,0.008954,0.001217 +100,gaussian,"(0.6, 0.4)",0.206485,0.323003,0.339997,0.412969,0.331195,0.294539,0.004288,0.014505,0.023846,0.09315,0.266744,0.008954,0.001169 +100,gaussian,"(0.8, 0.2)",0.206143,0.325448,0.343636,0.412287,0.336928,0.300137,0.004252,0.014975,0.024643,0.093171,0.266744,0.008954,0.001266 +100,inverse,"(0.2, 0.8)",0.198635,0.31428,0.331616,0.39727,0.322048,0.285597,0.003983,0.013779,0.022572,0.09315,0.266744,0.008954,0.025712 +100,inverse,"(0.4, 0.6)",0.203072,0.318663,0.33624,0.406143,0.324778,0.289078,0.004175,0.013922,0.022992,0.09315,0.266744,0.008954,0.001294 +100,inverse,"(0.5, 0.5)",0.206826,0.321197,0.338386,0.413652,0.327099,0.290853,0.004342,0.014118,0.023173,0.09315,0.266744,0.008954,0.001159 +100,inverse,"(0.6, 0.4)",0.206485,0.323003,0.339997,0.412969,0.331195,0.294539,0.004288,0.014505,0.023846,0.09315,0.266744,0.008954,0.001121 +100,inverse,"(0.8, 0.2)",0.206143,0.325448,0.343636,0.412287,0.336928,0.300137,0.004252,0.014975,0.024643,0.093171,0.266744,0.008954,0.001288 +1000,exp,"(0.2, 0.8)",0.198976,0.314601,0.331823,0.397952,0.322594,0.285939,0.004007,0.013844,0.02259,0.094267,0.337464,0.008954,0.026104 +1000,exp,"(0.4, 0.6)",0.203072,0.319717,0.336625,0.406143,0.326826,0.289966,0.004123,0.014177,0.023206,0.095813,0.337464,0.008954,0.00143 +1000,exp,"(0.5, 0.5)",0.206143,0.323687,0.340101,0.412287,0.331058,0.292218,0.00421,0.014469,0.023491,0.096925,0.337464,0.008954,0.001455 +1000,exp,"(0.6, 0.4)",0.208532,0.326688,0.343234,0.417065,0.334881,0.29727,0.004294,0.014855,0.024363,0.09863,0.337464,0.008954,0.001487 +1000,exp,"(0.8, 0.2)",0.207167,0.331217,0.349034,0.414334,0.345392,0.309829,0.004335,0.015661,0.026229,0.105402,0.337492,0.008954,0.001379 +1000,log,"(0.2, 0.8)",0.198976,0.314601,0.331823,0.397952,0.322594,0.285939,0.004007,0.013844,0.02259,0.094267,0.337464,0.008954,0.026199 +1000,log,"(0.4, 0.6)",0.203072,0.319717,0.336625,0.406143,0.326826,0.289966,0.004123,0.014177,0.023206,0.095813,0.337464,0.008954,0.001465 +1000,log,"(0.5, 0.5)",0.206143,0.323687,0.340101,0.412287,0.331058,0.292218,0.00421,0.014469,0.023491,0.096925,0.337464,0.008954,0.0013 +1000,log,"(0.6, 0.4)",0.208532,0.326688,0.343234,0.417065,0.334881,0.29727,0.004294,0.014855,0.024363,0.09863,0.337464,0.008954,0.001397 +1000,log,"(0.8, 0.2)",0.207167,0.331217,0.349034,0.414334,0.345392,0.309829,0.004335,0.015661,0.026229,0.105402,0.337492,0.008954,0.001396 +1000,logistic,"(0.2, 0.8)",0.198976,0.314601,0.331823,0.397952,0.322594,0.285939,0.004007,0.013844,0.02259,0.094267,0.337464,0.008954,0.026129 +1000,logistic,"(0.4, 0.6)",0.203072,0.319717,0.336625,0.406143,0.326826,0.289966,0.004123,0.014177,0.023206,0.095813,0.337464,0.008954,0.001503 +1000,logistic,"(0.5, 0.5)",0.206143,0.323687,0.340101,0.412287,0.331058,0.292218,0.00421,0.014469,0.023491,0.096925,0.337464,0.008954,0.001331 +1000,logistic,"(0.6, 0.4)",0.208532,0.326688,0.343234,0.417065,0.334881,0.29727,0.004294,0.014855,0.024363,0.09863,0.337464,0.008954,0.001312 +1000,logistic,"(0.8, 0.2)",0.207167,0.331217,0.349034,0.414334,0.345392,0.309829,0.004335,0.015661,0.026229,0.105402,0.337492,0.008954,0.00147 +1000,gaussian,"(0.2, 0.8)",0.198976,0.314601,0.331823,0.397952,0.322594,0.285939,0.004007,0.013844,0.02259,0.094267,0.337464,0.008954,0.026107 +1000,gaussian,"(0.4, 0.6)",0.203072,0.319717,0.336625,0.406143,0.326826,0.289966,0.004123,0.014177,0.023206,0.095813,0.337464,0.008954,0.001356 +1000,gaussian,"(0.5, 0.5)",0.206143,0.323687,0.340101,0.412287,0.331058,0.292218,0.00421,0.014469,0.023491,0.096925,0.337464,0.008954,0.001561 +1000,gaussian,"(0.6, 0.4)",0.208532,0.326688,0.343234,0.417065,0.334881,0.29727,0.004294,0.014855,0.024363,0.09863,0.337464,0.008954,0.001393 +1000,gaussian,"(0.8, 0.2)",0.207167,0.331217,0.349034,0.414334,0.345392,0.309829,0.004335,0.015661,0.026229,0.105402,0.337492,0.008954,0.001388 +1000,inverse,"(0.2, 0.8)",0.198976,0.314601,0.331823,0.397952,0.322594,0.285939,0.004007,0.013844,0.02259,0.094267,0.337464,0.008954,0.026237 +1000,inverse,"(0.4, 0.6)",0.203072,0.319717,0.336625,0.406143,0.326826,0.289966,0.004123,0.014177,0.023206,0.095813,0.337464,0.008954,0.00145 +1000,inverse,"(0.5, 0.5)",0.206143,0.323687,0.340101,0.412287,0.331058,0.292218,0.00421,0.014469,0.023491,0.096925,0.337464,0.008954,0.001159 +1000,inverse,"(0.6, 0.4)",0.208532,0.326688,0.343234,0.417065,0.334881,0.29727,0.004294,0.014855,0.024363,0.09863,0.337464,0.008954,0.00116 +1000,inverse,"(0.8, 0.2)",0.207167,0.331217,0.349034,0.414334,0.345392,0.309829,0.004335,0.015661,0.026229,0.105402,0.337492,0.008954,0.001385 +6353,exp,"(0.2, 0.8)",0.199317,0.314411,0.331725,0.398635,0.321775,0.286212,0.004018,0.013769,0.022648,0.094229,0.339025,0.008954,0.028546 +6353,exp,"(0.4, 0.6)",0.202389,0.319082,0.336141,0.404778,0.32587,0.289898,0.004144,0.014095,0.023139,0.095605,0.341442,0.008954,0.003125 +6353,exp,"(0.5, 0.5)",0.206143,0.3228,0.339587,0.412287,0.329283,0.291672,0.004247,0.014387,0.023458,0.096719,0.343167,0.008954,0.00273 +6353,exp,"(0.6, 0.4)",0.210239,0.326368,0.343102,0.420478,0.332833,0.295904,0.004357,0.014678,0.024162,0.098462,0.345969,0.008954,0.003122 +6353,exp,"(0.8, 0.2)",0.206143,0.331647,0.349323,0.412287,0.345256,0.308055,0.004279,0.01564,0.02606,0.10541,0.356509,0.008954,0.002721 +6353,log,"(0.2, 0.8)",0.199317,0.314411,0.331725,0.398635,0.321775,0.286212,0.004018,0.013769,0.022648,0.094229,0.339025,0.008954,0.028992 +6353,log,"(0.4, 0.6)",0.202389,0.319082,0.336141,0.404778,0.32587,0.289898,0.004144,0.014095,0.023139,0.095605,0.341442,0.008954,0.002743 +6353,log,"(0.5, 0.5)",0.206143,0.3228,0.339587,0.412287,0.329283,0.291672,0.004247,0.014387,0.023458,0.096719,0.343167,0.008954,0.002762 +6353,log,"(0.6, 0.4)",0.210239,0.326368,0.343102,0.420478,0.332833,0.295904,0.004357,0.014678,0.024162,0.098462,0.345969,0.008954,0.003423 +6353,log,"(0.8, 0.2)",0.206143,0.331647,0.349323,0.412287,0.345256,0.308055,0.004279,0.01564,0.02606,0.10541,0.356509,0.008954,0.002762 +6353,logistic,"(0.2, 0.8)",0.199317,0.314411,0.331725,0.398635,0.321775,0.286212,0.004018,0.013769,0.022648,0.094229,0.339025,0.008954,0.028217 +6353,logistic,"(0.4, 0.6)",0.202389,0.319082,0.336141,0.404778,0.32587,0.289898,0.004144,0.014095,0.023139,0.095605,0.341442,0.008954,0.003558 +6353,logistic,"(0.5, 0.5)",0.206143,0.3228,0.339587,0.412287,0.329283,0.291672,0.004247,0.014387,0.023458,0.096719,0.343167,0.008954,0.002773 +6353,logistic,"(0.6, 0.4)",0.210239,0.326368,0.343102,0.420478,0.332833,0.295904,0.004357,0.014678,0.024162,0.098462,0.345969,0.008954,0.002789 +6353,logistic,"(0.8, 0.2)",0.206143,0.331647,0.349323,0.412287,0.345256,0.308055,0.004279,0.01564,0.02606,0.10541,0.356509,0.008954,0.002784 +6353,gaussian,"(0.2, 0.8)",0.199317,0.314411,0.331725,0.398635,0.321775,0.286212,0.004018,0.013769,0.022648,0.094229,0.339025,0.008954,0.028535 +6353,gaussian,"(0.4, 0.6)",0.202389,0.319082,0.336141,0.404778,0.32587,0.289898,0.004144,0.014095,0.023139,0.095605,0.341442,0.008954,0.002811 +6353,gaussian,"(0.5, 0.5)",0.206143,0.3228,0.339587,0.412287,0.329283,0.291672,0.004247,0.014387,0.023458,0.096719,0.343167,0.008954,0.002808 +6353,gaussian,"(0.6, 0.4)",0.210239,0.326368,0.343102,0.420478,0.332833,0.295904,0.004357,0.014678,0.024162,0.098462,0.345969,0.008954,0.00281 +6353,gaussian,"(0.8, 0.2)",0.206143,0.331647,0.349323,0.412287,0.345256,0.308055,0.004279,0.01564,0.02606,0.10541,0.356509,0.008954,0.002816 +6353,inverse,"(0.2, 0.8)",0.199317,0.314411,0.331725,0.398635,0.321775,0.286212,0.004018,0.013769,0.022648,0.094229,0.339025,0.008954,0.028661 +6353,inverse,"(0.4, 0.6)",0.202389,0.319082,0.336141,0.404778,0.32587,0.289898,0.004144,0.014095,0.023139,0.095605,0.341442,0.008954,0.002819 +6353,inverse,"(0.5, 0.5)",0.206143,0.3228,0.339587,0.412287,0.329283,0.291672,0.004247,0.014387,0.023458,0.096719,0.343167,0.008954,0.002841 +6353,inverse,"(0.6, 0.4)",0.210239,0.326368,0.343102,0.420478,0.332833,0.295904,0.004357,0.014678,0.024162,0.098462,0.345969,0.008954,0.002836 +6353,inverse,"(0.8, 0.2)",0.206143,0.331647,0.349323,0.412287,0.345256,0.308055,0.004279,0.01564,0.02606,0.10541,0.356509,0.008954,0.002848 diff --git a/out/ensemble_sift_color_resnet_knn_eval.csv b/out/ensemble_sift_color_resnet_knn_eval.csv index 47a041e..99d9530 100644 --- a/out/ensemble_sift_color_resnet_knn_eval.csv +++ b/out/ensemble_sift_color_resnet_knn_eval.csv @@ -1,6 +1,6 @@ -k,distance2score,weight(sift vs color),map@1,map@5,map@10,hit_rate@1,hit_rate@5,hit_rate@10,recall@1,recall@5,recall@10,recall@100,recall@1000,avg_indexing_time,avg_retrieval_time,weight(sift vs color vs resnet) -6353,exp,,0.879863,0.827304,0.795017,0.879863,0.965188,0.974061,0.0,0.0,0.0,0.0,0.0,0.012997,0.043442,"(0.16, 0.04, 0.8)" -6353,exp,,0.869625,0.814334,0.780819,0.869625,0.956997,0.973379,0.0,0.0,0.0,0.0,0.0,0.012997,0.003675,"(0.32, 0.08, 0.6)" -6353,exp,,0.85802,0.803413,0.764642,0.85802,0.952901,0.969283,0.0,0.0,0.0,0.0,0.0,0.012997,0.004133,"(0.4, 0.1, 0.5)" -6353,exp,,0.838225,0.778703,0.736997,0.838225,0.944027,0.965188,0.0,0.0,0.0,0.0,0.0,0.012997,0.004157,"(0.48, 0.12, 0.4)" -6353,exp,,0.745392,0.672765,0.62157,0.745392,0.884642,0.924232,0.0,0.0,0.0,0.0,0.0,0.012997,0.003683,"(0.64, 0.16, 0.2)" +k,distance2score,weight(sift vs color vs resnet),map@1,map@5,map@10,hit_rate@1,hit_rate@5,hit_rate@10,recall@1,recall@5,recall@10,recall@100,recall@1000,avg_indexing_time,avg_retrieval_time +6353,exp,"(0.16, 0.04, 0.8)",0.436519,0.722009,0.774087,0.873038,0.838771,0.801843,0.014099,0.066712,0.125761,0.557948,0.929494,0.012309,0.048446 +6353,exp,"(0.32, 0.08, 0.6)",0.433788,0.716522,0.766847,0.867577,0.826485,0.790034,0.013908,0.065267,0.122725,0.54261,0.921546,0.012309,0.003712 +6353,exp,"(0.4, 0.1, 0.5)",0.430375,0.707799,0.756372,0.860751,0.813925,0.77802,0.013712,0.063743,0.119679,0.523309,0.910957,0.012309,0.00409 +6353,exp,"(0.48, 0.12, 0.4)",0.422867,0.693414,0.739999,0.845734,0.790853,0.7557,0.013352,0.061367,0.114479,0.488053,0.889254,0.012309,0.00414 +6353,exp,"(0.64, 0.16, 0.2)",0.379181,0.619229,0.65654,0.758362,0.689283,0.636792,0.011448,0.049881,0.087719,0.34638,0.756345,0.012309,0.004134 diff --git a/out/histogram_knn_eval.csv b/out/histogram_knn_eval.csv index f64d10e..4b59713 100644 --- a/out/histogram_knn_eval.csv +++ b/out/histogram_knn_eval.csv @@ -1,17 +1,65 @@ -bins,htype,metric,map@1,map@5,map@10,hit_rate@1,hit_rate@5,hit_rate@10,recall@1,recall@5,recall@10,recall@100,recall@1000,avg_indexing_time,avg_retrieval_time -2,region,euclidean,0.331058,0.267986,0.239795,0.331058,0.483276,0.550853,0.003012,0.010598,0.017388,0.078739,0.310192,0.000992,0.000971 -2,region,cosine,0.324232,0.26744,0.239454,0.324232,0.475768,0.554949,0.00294,0.01054,0.017698,0.080455,0.312565,0.000998,0.001458 -2,global,euclidean,0.141297,0.106758,0.099454,0.141297,0.288737,0.382935,0.001118,0.003906,0.007064,0.044466,0.252985,0.000994,0.000713 -2,global,cosine,0.138567,0.107167,0.098157,0.138567,0.296246,0.386348,0.00113,0.003966,0.007086,0.044571,0.254884,0.000988,0.001246 -4,region,euclidean,0.369966,0.292014,0.262184,0.369966,0.498294,0.584983,0.003627,0.011819,0.019838,0.083359,0.318225,0.001009,0.001027 -4,region,cosine,0.373379,0.298976,0.263481,0.373379,0.509215,0.578157,0.003735,0.012632,0.020159,0.088442,0.322851,0.000988,0.003767 -4,global,euclidean,0.243686,0.194266,0.175973,0.243686,0.411604,0.488055,0.001998,0.007104,0.01214,0.06312,0.282745,0.000986,0.000991 -4,global,cosine,0.238908,0.193993,0.175563,0.238908,0.417747,0.486007,0.002068,0.007408,0.012499,0.063244,0.288249,0.000996,0.001339 -8,region,euclidean,0.352901,0.273311,0.23884,0.352901,0.483276,0.548123,0.003375,0.011001,0.017678,0.076686,0.308355,0.001082,0.002736 -8,region,cosine,0.356997,0.279317,0.248601,0.356997,0.486007,0.546758,0.003496,0.011554,0.018921,0.084385,0.325361,0.001062,0.025872 -8,global,euclidean,0.275085,0.216246,0.189283,0.275085,0.427304,0.48942,0.002263,0.007664,0.012529,0.061587,0.292441,0.001007,0.002483 -8,global,cosine,0.269625,0.207918,0.182935,0.269625,0.417065,0.496246,0.002455,0.00777,0.012896,0.063367,0.297811,0.001001,0.002484 -12,region,euclidean,0.327645,0.254334,0.221365,0.327645,0.458703,0.529693,0.00304,0.0106,0.017018,0.073402,0.298012,0.001484,0.007101 -12,region,cosine,0.338567,0.272901,0.239181,0.338567,0.479181,0.543345,0.003345,0.011587,0.018738,0.083657,0.322693,0.001499,0.067846 -12,global,euclidean,0.285324,0.215017,0.190512,0.285324,0.410239,0.487372,0.002313,0.007455,0.012528,0.061049,0.296934,0.001011,0.005967 -12,global,cosine,0.278498,0.208464,0.181229,0.278498,0.410239,0.481911,0.002527,0.007385,0.01234,0.062641,0.300355,0.000999,0.007284 +bins,htype,n_slice,metric,map@1,map@5,map@10,hit_rate@1,hit_rate@5,hit_rate@10,recall@1,recall@5,recall@10,recall@100,recall@1000,avg_indexing_time,avg_retrieval_time +2,region,2,euclidean,0.106826,0.173152,0.18433,0.213652,0.165597,0.149488,0.001938,0.006562,0.010754,0.05622,0.26924,0.000879,0.000923 +2,region,2,cosine,0.102048,0.170466,0.18241,0.204096,0.164369,0.149147,0.001868,0.006417,0.011055,0.056983,0.272556,0.000866,0.000817 +2,region,3,euclidean,0.16587,0.264355,0.278764,0.331741,0.26744,0.238976,0.003029,0.010494,0.017425,0.078481,0.310125,0.001258,0.004554 +2,region,3,cosine,0.16314,0.261268,0.276275,0.32628,0.266075,0.238498,0.002953,0.010534,0.017511,0.08071,0.31222,0.00089,0.000947 +2,region,4,euclidean,0.173379,0.28108,0.296014,0.346758,0.288737,0.257474,0.003234,0.011733,0.019019,0.083794,0.319931,0.00093,0.003416 +2,region,4,cosine,0.172014,0.281812,0.298373,0.344027,0.291877,0.260137,0.003285,0.012027,0.019646,0.084507,0.320707,0.001403,0.005181 +2,region,5,euclidean,0.181911,0.292352,0.308463,0.363823,0.30116,0.266894,0.003433,0.012313,0.019938,0.086442,0.325511,0.001429,0.00333 +2,region,5,cosine,0.186348,0.295673,0.311676,0.372696,0.302935,0.272355,0.00364,0.012746,0.021034,0.087252,0.323152,0.001085,0.00535 +2,global,2,euclidean,0.071672,0.115802,0.125626,0.143345,0.106758,0.100205,0.001211,0.003891,0.00713,0.044607,0.253462,0.001433,0.000895 +2,global,2,cosine,0.066894,0.114906,0.124947,0.133788,0.108669,0.098635,0.001114,0.004044,0.007102,0.044757,0.255004,0.000957,0.000796 +2,global,3,euclidean,0.071672,0.115802,0.125626,0.143345,0.106758,0.100205,0.001211,0.003891,0.00713,0.044607,0.253462,0.000952,0.000726 +2,global,3,cosine,0.066894,0.114906,0.124947,0.133788,0.108669,0.098635,0.001114,0.004044,0.007102,0.044757,0.255004,0.000949,0.00079 +2,global,4,euclidean,0.071672,0.115802,0.125626,0.143345,0.106758,0.100205,0.001211,0.003891,0.00713,0.044607,0.253462,0.001182,0.000876 +2,global,4,cosine,0.066894,0.114906,0.124947,0.133788,0.108669,0.098635,0.001114,0.004044,0.007102,0.044757,0.255004,0.001169,0.000936 +2,global,5,euclidean,0.071672,0.115802,0.125626,0.143345,0.106758,0.100205,0.001211,0.003891,0.00713,0.044607,0.253462,0.001175,0.000925 +2,global,5,cosine,0.066894,0.114906,0.124947,0.133788,0.108669,0.098635,0.001114,0.004044,0.007102,0.044757,0.255004,0.001155,0.000952 +4,region,2,euclidean,0.156314,0.241024,0.254166,0.312628,0.238635,0.213584,0.002866,0.009144,0.015148,0.067827,0.29085,0.001166,0.00521 +4,region,2,cosine,0.151195,0.238561,0.251629,0.302389,0.236041,0.20901,0.002689,0.009304,0.015255,0.070237,0.29478,0.000878,0.003411 +4,region,3,euclidean,0.183276,0.288435,0.303298,0.366553,0.294334,0.263549,0.003521,0.011923,0.019939,0.083408,0.317844,0.000948,0.001089 +4,region,3,cosine,0.186348,0.29344,0.307859,0.372696,0.300341,0.264778,0.003696,0.012621,0.02028,0.088461,0.323241,0.00094,0.005043 +4,region,4,euclidean,0.187713,0.294155,0.310326,0.375427,0.300478,0.269147,0.003728,0.012061,0.020195,0.085404,0.326883,0.001,0.001219 +4,region,4,cosine,0.191126,0.303041,0.318549,0.382253,0.311126,0.277884,0.003869,0.012984,0.0215,0.091159,0.333826,0.001348,0.011747 +4,region,5,euclidean,0.190785,0.299006,0.314577,0.38157,0.304437,0.274061,0.003686,0.012554,0.020947,0.087145,0.329653,0.001466,0.008605 +4,region,5,cosine,0.198294,0.311802,0.328849,0.396587,0.319044,0.283481,0.004002,0.013556,0.022234,0.09315,0.337464,0.001409,0.012869 +4,global,2,euclidean,0.121843,0.199067,0.210732,0.243686,0.195495,0.175973,0.002061,0.007142,0.012122,0.06316,0.28265,0.00089,0.001038 +4,global,2,cosine,0.120137,0.198503,0.211282,0.240273,0.193993,0.175563,0.002134,0.007359,0.012559,0.063193,0.288359,0.000884,0.000842 +4,global,3,euclidean,0.121843,0.199067,0.210732,0.243686,0.195495,0.175973,0.002061,0.007142,0.012122,0.06316,0.28265,0.000884,0.003273 +4,global,3,cosine,0.120137,0.198503,0.211282,0.240273,0.193993,0.175563,0.002134,0.007359,0.012559,0.063193,0.288359,0.001179,0.001081 +4,global,4,euclidean,0.121843,0.199067,0.210732,0.243686,0.195495,0.175973,0.002061,0.007142,0.012122,0.06316,0.28265,0.001179,0.006888 +4,global,4,cosine,0.120137,0.198503,0.211282,0.240273,0.193993,0.175563,0.002134,0.007359,0.012559,0.063193,0.288359,0.001135,0.001096 +4,global,5,euclidean,0.121843,0.199067,0.210732,0.243686,0.195495,0.175973,0.002061,0.007142,0.012122,0.06316,0.28265,0.00097,0.001047 +4,global,5,cosine,0.120137,0.198503,0.211282,0.240273,0.193993,0.175563,0.002134,0.007359,0.012559,0.063193,0.288359,0.000893,0.00083 +8,region,2,euclidean,0.15802,0.242164,0.252948,0.316041,0.233038,0.206348,0.002934,0.008869,0.014289,0.064734,0.292216,0.000961,0.001677 +8,region,2,cosine,0.153925,0.234716,0.245759,0.30785,0.226348,0.197747,0.002963,0.009102,0.014268,0.06829,0.30101,0.000962,0.016444 +8,region,3,euclidean,0.175085,0.273295,0.284824,0.350171,0.274266,0.23884,0.003292,0.011137,0.017778,0.077265,0.30839,0.001452,0.010769 +8,region,3,cosine,0.180887,0.278588,0.291229,0.361775,0.280683,0.248191,0.003525,0.011633,0.018993,0.084323,0.325748,0.001424,0.028089 +8,region,4,euclidean,0.172696,0.267865,0.281322,0.345392,0.266485,0.237543,0.003317,0.010827,0.017912,0.076892,0.310431,0.001609,0.015663 +8,region,4,cosine,0.184642,0.286573,0.301817,0.369283,0.288874,0.259727,0.003563,0.011691,0.019297,0.087151,0.332759,0.001515,0.043942 +8,region,5,euclidean,0.174744,0.2688,0.281835,0.349488,0.268942,0.234744,0.003373,0.011235,0.018235,0.076498,0.310709,0.001614,0.006288 +8,region,5,cosine,0.191126,0.296932,0.31225,0.382253,0.301297,0.268942,0.003784,0.012575,0.020292,0.089297,0.336926,0.001542,0.062881 +8,global,2,euclidean,0.138908,0.219745,0.230661,0.277816,0.216928,0.190922,0.002308,0.00765,0.012624,0.061646,0.292162,0.001217,0.010844 +8,global,2,cosine,0.135836,0.214062,0.226496,0.271672,0.207509,0.18273,0.002467,0.007679,0.012749,0.063429,0.298087,0.000887,0.003786 +8,global,3,euclidean,0.138908,0.219745,0.230661,0.277816,0.216928,0.190922,0.002308,0.00765,0.012624,0.061646,0.292162,0.000895,0.002508 +8,global,3,cosine,0.135836,0.214062,0.226496,0.271672,0.207509,0.18273,0.002467,0.007679,0.012749,0.063429,0.298087,0.000878,0.00378 +8,global,4,euclidean,0.138908,0.219745,0.230661,0.277816,0.216928,0.190922,0.002308,0.00765,0.012624,0.061646,0.292162,0.000882,0.002841 +8,global,4,cosine,0.135836,0.214062,0.226496,0.271672,0.207509,0.18273,0.002467,0.007679,0.012749,0.063429,0.298087,0.001156,0.005243 +8,global,5,euclidean,0.138908,0.219745,0.230661,0.277816,0.216928,0.190922,0.002308,0.00765,0.012624,0.061646,0.292162,0.001074,0.012414 +8,global,5,cosine,0.135836,0.214062,0.226496,0.271672,0.207509,0.18273,0.002467,0.007679,0.012749,0.063429,0.298087,0.001154,0.004012 +12,region,2,euclidean,0.151195,0.232836,0.243736,0.302389,0.226894,0.1957,0.002794,0.008685,0.013843,0.063153,0.290062,0.001148,0.003813 +12,region,2,cosine,0.152218,0.233297,0.243683,0.304437,0.225666,0.194198,0.002989,0.009063,0.014293,0.067491,0.302006,0.001152,0.040478 +12,region,3,euclidean,0.162457,0.257606,0.268892,0.324915,0.256382,0.221092,0.003018,0.010735,0.017057,0.073701,0.297732,0.001567,0.00738 +12,region,3,cosine,0.17099,0.270285,0.283994,0.34198,0.273311,0.240887,0.00341,0.011592,0.018967,0.083543,0.32313,0.001556,0.07358 +12,region,4,euclidean,0.161433,0.249721,0.260406,0.322867,0.245051,0.210307,0.003072,0.010501,0.016416,0.072419,0.29958,0.002318,0.017848 +12,region,4,cosine,0.175427,0.28082,0.295944,0.350853,0.287372,0.25372,0.003373,0.011955,0.019279,0.087612,0.329533,0.002034,0.124746 +12,region,5,euclidean,0.157679,0.243503,0.254687,0.315358,0.236997,0.204164,0.003107,0.0105,0.016837,0.071375,0.298628,0.004012,0.026965 +12,region,5,cosine,0.181911,0.290048,0.305795,0.363823,0.298294,0.262526,0.003565,0.012416,0.020245,0.089304,0.334704,0.007279,0.191561 +12,global,2,euclidean,0.143686,0.220971,0.231971,0.287372,0.214198,0.189078,0.002316,0.007353,0.012357,0.060828,0.296786,0.000905,0.006021 +12,global,2,cosine,0.139932,0.217328,0.227818,0.279863,0.210648,0.182867,0.002527,0.007455,0.012323,0.062611,0.300949,0.000887,0.008011 +12,global,3,euclidean,0.143686,0.220971,0.231971,0.287372,0.214198,0.189078,0.002316,0.007353,0.012357,0.060828,0.296786,0.000904,0.005986 +12,global,3,cosine,0.139932,0.217328,0.227818,0.279863,0.210648,0.182867,0.002527,0.007455,0.012323,0.062611,0.300949,0.000902,0.008028 +12,global,4,euclidean,0.143686,0.220971,0.231971,0.287372,0.214198,0.189078,0.002316,0.007353,0.012357,0.060828,0.296786,0.000911,0.005983 +12,global,4,cosine,0.139932,0.217328,0.227818,0.279863,0.210648,0.182867,0.002527,0.007455,0.012323,0.062611,0.300949,0.000906,0.008035 +12,global,5,euclidean,0.143686,0.220971,0.231971,0.287372,0.214198,0.189078,0.002316,0.007353,0.012357,0.060828,0.296786,0.000897,0.006013 +12,global,5,cosine,0.139932,0.217328,0.227818,0.279863,0.210648,0.182867,0.002527,0.007455,0.012323,0.062611,0.300949,0.000904,0.008067 diff --git a/out/resnet_knn_cpu_eval.csv b/out/resnet_knn_cpu_eval.csv new file mode 100644 index 0000000..7c30600 --- /dev/null +++ b/out/resnet_knn_cpu_eval.csv @@ -0,0 +1,11 @@ +model,metric,map@1,map@5,map@10,hit_rate@1,hit_rate@5,hit_rate@10,recall@1,recall@5,recall@10,recall@100,recall@1000,avg_indexing_time,avg_retrieval_time +resnet18,euclidean,0.399659,0.655243,0.699318,0.799317,0.745256,0.704573,0.012291,0.055233,0.101115,0.424731,0.819271,0.007408,0.003459 +resnet18,cosine,0.415017,0.678607,0.724738,0.830034,0.773925,0.73802,0.012926,0.058735,0.109174,0.473221,0.876809,0.007412,0.005232 +resnet34,euclidean,0.419454,0.689021,0.736365,0.838908,0.787713,0.750307,0.013275,0.060561,0.113126,0.485451,0.867174,0.011084,0.004421 +resnet34,cosine,0.426962,0.704586,0.754328,0.853925,0.813379,0.777884,0.013635,0.063591,0.119375,0.532325,0.9167,0.01154,0.005935 +resnet50,euclidean,0.422184,0.688405,0.73499,0.844369,0.786075,0.750785,0.013318,0.059945,0.112337,0.484026,0.87145,0.026898,0.009914 +resnet50,cosine,0.432082,0.70548,0.75444,0.864164,0.808055,0.774812,0.013846,0.062628,0.117531,0.525654,0.910517,0.027144,0.019858 +resnet101,euclidean,0.433788,0.709703,0.759503,0.867577,0.817201,0.784027,0.013994,0.064548,0.122043,0.524385,0.895737,0.038445,0.012667 +resnet101,cosine,0.437543,0.72387,0.775381,0.875085,0.838362,0.803891,0.014133,0.066707,0.126186,0.560837,0.931262,0.038086,0.022556 +resnet152,euclidean,0.432765,0.706929,0.754837,0.865529,0.80901,0.773311,0.013978,0.063604,0.119526,0.522956,0.899991,0.054021,0.016194 +resnet152,cosine,0.441297,0.721474,0.772377,0.882594,0.828123,0.797406,0.014217,0.065943,0.124771,0.562465,0.932063,0.053849,0.025914 diff --git a/out/resnet_knn_gpu_eval.csv b/out/resnet_knn_gpu_eval.csv index edb4175..230e005 100644 --- a/out/resnet_knn_gpu_eval.csv +++ b/out/resnet_knn_gpu_eval.csv @@ -1,11 +1,11 @@ model,metric,map@1,map@5,map@10,hit_rate@1,hit_rate@5,hit_rate@10,recall@1,recall@5,recall@10,recall@100,recall@1000,avg_indexing_time,avg_retrieval_time -resnet18,euclidean,0.798635,0.745256,0.704573,0.798635,0.912628,0.939249,0.012276,0.055231,0.101105,0.424683,0.819254,0.001124,0.001946 -resnet18,cosine,0.830034,0.773788,0.73802,0.830034,0.931741,0.949488,0.012926,0.058723,0.109164,0.473179,0.87688,0.001075,0.003492 -resnet34,euclidean,0.838908,0.787577,0.750444,0.838908,0.931058,0.957679,0.01328,0.060532,0.113124,0.485548,0.867162,0.001206,0.002053 -resnet34,cosine,0.853925,0.813515,0.777884,0.853925,0.948123,0.967235,0.013635,0.063591,0.119369,0.532325,0.916735,0.001199,0.003521 -resnet50,euclidean,0.845051,0.786212,0.75058,0.845051,0.933106,0.957679,0.013334,0.059963,0.112308,0.484118,0.871346,0.002949,0.004597 -resnet50,cosine,0.864164,0.807918,0.775085,0.864164,0.942662,0.963823,0.013846,0.06263,0.117603,0.525629,0.910529,0.002932,0.014151 -resnet101,euclidean,0.867577,0.817201,0.783959,0.867577,0.948805,0.967918,0.013994,0.064536,0.122024,0.524354,0.895706,0.003545,0.004784 -resnet101,cosine,0.875085,0.838362,0.803891,0.875085,0.956997,0.970648,0.014133,0.066704,0.126178,0.560759,0.931237,0.003557,0.014368 -resnet152,euclidean,0.865529,0.80901,0.773379,0.865529,0.959044,0.972696,0.013978,0.063623,0.119524,0.523001,0.899989,0.004256,0.00493 -resnet152,cosine,0.881911,0.828259,0.797884,0.881911,0.965188,0.975427,0.014181,0.065997,0.124864,0.562479,0.932087,0.00429,0.014589 +resnet18,euclidean,0.399317,0.655163,0.699291,0.798635,0.745256,0.704573,0.012276,0.055231,0.101105,0.424683,0.819254,0.001132,0.002053 +resnet18,cosine,0.415017,0.678539,0.724733,0.830034,0.773788,0.73802,0.012926,0.058723,0.109164,0.473179,0.87688,0.001092,0.003454 +resnet34,euclidean,0.419454,0.688988,0.736372,0.838908,0.787577,0.750444,0.01328,0.060532,0.113124,0.485548,0.867162,0.001222,0.002073 +resnet34,cosine,0.426962,0.704501,0.754285,0.853925,0.813515,0.777884,0.013635,0.063591,0.119369,0.532325,0.916735,0.001231,0.003624 +resnet50,euclidean,0.422526,0.688562,0.734887,0.845051,0.786212,0.75058,0.013334,0.059963,0.112308,0.484118,0.871346,0.002925,0.004699 +resnet50,cosine,0.432082,0.705672,0.754654,0.864164,0.807918,0.775085,0.013846,0.06263,0.117603,0.525629,0.910529,0.002917,0.014149 +resnet101,euclidean,0.433788,0.709931,0.75956,0.867577,0.817201,0.783959,0.013994,0.064536,0.122024,0.524354,0.895706,0.003523,0.004955 +resnet101,cosine,0.437543,0.723758,0.775291,0.875085,0.838362,0.803891,0.014133,0.066704,0.126178,0.560759,0.931237,0.003549,0.010012 +resnet152,euclidean,0.432765,0.707015,0.754875,0.865529,0.80901,0.773379,0.013978,0.063623,0.119524,0.523001,0.899989,0.004293,0.006163 +resnet152,cosine,0.440956,0.721478,0.772529,0.881911,0.828259,0.797884,0.014181,0.065997,0.124864,0.562479,0.932087,0.004304,0.016572 diff --git a/out/sift_knn_eval.csv b/out/sift_knn_eval.csv index 531a051..c871b87 100644 --- a/out/sift_knn_eval.csv +++ b/out/sift_knn_eval.csv @@ -1,31 +1,31 @@ k,type,metric,map@1,map@5,map@10,hit_rate@1,hit_rate@5,hit_rate@10,recall@1,recall@5,recall@10,recall@100,recall@1000,avg_indexing_time,avg_retrieval_time,fitting_time -32,tfidf,cosine,0.205461,0.164232,0.149488,0.205461,0.389761,0.479181,0.001975,0.007142,0.012058,0.067544,0.323383,0.008164,0.005317,20.962966 -32,tfidf,euclidean,0.205461,0.164232,0.149488,0.205461,0.389761,0.479181,0.001975,0.007142,0.012058,0.067544,0.323383,0.008251,0.002861,20.962966 -32,tfidf,manhattan,0.211604,0.165597,0.148259,0.211604,0.393174,0.490102,0.002018,0.007452,0.012401,0.070791,0.337613,0.008192,0.002848,20.962966 -32,bow,cosine,0.205461,0.164232,0.149488,0.205461,0.389761,0.479181,0.001975,0.007142,0.012058,0.067544,0.323383,0.008211,0.00538,20.962966 -32,bow,euclidean,0.205461,0.164232,0.149488,0.205461,0.389761,0.479181,0.001975,0.007142,0.012058,0.067544,0.323383,0.008154,0.002898,20.962966 -32,bow,manhattan,0.211604,0.165597,0.148259,0.211604,0.393174,0.490102,0.002018,0.007452,0.012401,0.070791,0.337613,0.00824,0.002835,20.962966 -64,tfidf,cosine,0.204096,0.166962,0.15058,0.204096,0.39727,0.495563,0.002139,0.007708,0.013283,0.070053,0.320327,0.008273,0.005517,30.953784 -64,tfidf,euclidean,0.204096,0.166962,0.15058,0.204096,0.39727,0.495563,0.002139,0.007708,0.013283,0.070053,0.320327,0.008147,0.002911,30.953784 -64,tfidf,manhattan,0.217065,0.172014,0.155836,0.217065,0.401365,0.514676,0.00227,0.008095,0.01434,0.077102,0.348248,0.008133,0.002857,30.953784 -64,bow,cosine,0.204096,0.166962,0.15058,0.204096,0.39727,0.495563,0.002139,0.007708,0.013283,0.070053,0.320327,0.008321,0.00551,30.953784 -64,bow,euclidean,0.204096,0.166962,0.15058,0.204096,0.39727,0.495563,0.002139,0.007708,0.013283,0.070053,0.320327,0.008275,0.00292,30.953784 -64,bow,manhattan,0.217065,0.172014,0.155836,0.217065,0.401365,0.514676,0.00227,0.008095,0.01434,0.077102,0.348248,0.008282,0.002846,30.953784 -96,tfidf,cosine,0.205461,0.165734,0.143959,0.205461,0.384983,0.492833,0.002265,0.008179,0.013076,0.06592,0.296878,0.008306,0.005895,40.023622 -96,tfidf,euclidean,0.205461,0.165734,0.143959,0.205461,0.384983,0.492833,0.002265,0.008179,0.013076,0.06592,0.296878,0.008298,0.002899,40.023622 -96,tfidf,manhattan,0.221843,0.178703,0.154812,0.221843,0.414334,0.517406,0.002413,0.008944,0.01491,0.074755,0.333368,0.008094,0.002874,40.023622 -96,bow,cosine,0.205461,0.165734,0.143959,0.205461,0.384983,0.492833,0.002265,0.008179,0.013076,0.06592,0.296878,0.008244,0.005679,40.023622 -96,bow,euclidean,0.205461,0.165734,0.143959,0.205461,0.384983,0.492833,0.002265,0.008179,0.013076,0.06592,0.296878,0.008176,0.002938,40.023622 -96,bow,manhattan,0.221843,0.178703,0.154812,0.221843,0.414334,0.517406,0.002413,0.008944,0.01491,0.074755,0.333368,0.008263,0.002847,40.023622 -128,tfidf,cosine,0.208191,0.166962,0.1443,0.208191,0.391809,0.484642,0.002254,0.007971,0.012924,0.065877,0.298475,0.00824,0.005746,49.517996 -128,tfidf,euclidean,0.208191,0.166962,0.1443,0.208191,0.391809,0.484642,0.002254,0.007971,0.012924,0.065877,0.298475,0.008304,0.002976,49.517996 -128,tfidf,manhattan,0.215017,0.175563,0.155154,0.215017,0.404096,0.512628,0.002474,0.008643,0.0147,0.076088,0.343889,0.008166,0.002879,49.517996 -128,bow,cosine,0.208191,0.166962,0.1443,0.208191,0.391809,0.484642,0.002254,0.007971,0.012924,0.065877,0.298475,0.008287,0.005987,49.517996 -128,bow,euclidean,0.208191,0.166962,0.1443,0.208191,0.391809,0.484642,0.002254,0.007971,0.012924,0.065877,0.298475,0.008283,0.003006,49.517996 -128,bow,manhattan,0.215017,0.175563,0.155154,0.215017,0.404096,0.512628,0.002474,0.008643,0.0147,0.076088,0.343889,0.008294,0.002885,49.517996 -256,tfidf,cosine,0.185666,0.151536,0.134471,0.185666,0.353584,0.461433,0.001867,0.007289,0.012354,0.058084,0.272928,0.008459,0.006563,86.50311 -256,tfidf,euclidean,0.185666,0.151536,0.134471,0.185666,0.353584,0.461433,0.001867,0.007289,0.012354,0.058084,0.272928,0.008313,0.00315,86.50311 -256,tfidf,manhattan,0.188396,0.154676,0.139659,0.188396,0.359044,0.453242,0.001966,0.007702,0.013112,0.063983,0.314161,0.008386,0.002934,86.50311 -256,bow,cosine,0.185666,0.151536,0.134471,0.185666,0.353584,0.461433,0.001867,0.007289,0.012354,0.058084,0.272928,0.008225,0.006406,86.50311 -256,bow,euclidean,0.185666,0.151536,0.134471,0.185666,0.353584,0.461433,0.001867,0.007289,0.012354,0.058084,0.272928,0.008303,0.003149,86.50311 -256,bow,manhattan,0.188396,0.154676,0.139659,0.188396,0.359044,0.453242,0.001966,0.007702,0.013112,0.063983,0.314161,0.008362,0.002945,86.50311 +32,tfidf,cosine,0.10273,0.169297,0.180089,0.205461,0.163276,0.148805,0.00189,0.006965,0.011888,0.067505,0.323261,0.008232,0.00659,21.656763 +32,tfidf,euclidean,0.10273,0.169297,0.180089,0.205461,0.163276,0.148805,0.00189,0.006965,0.011888,0.067505,0.323261,0.011935,0.010341,21.656763 +32,tfidf,manhattan,0.100683,0.169626,0.18202,0.201365,0.166416,0.151195,0.001951,0.007423,0.012696,0.070767,0.337468,0.008295,0.002903,21.656763 +32,bow,cosine,0.10273,0.169297,0.180089,0.205461,0.163276,0.148805,0.00189,0.006965,0.011888,0.067505,0.323261,0.008233,0.005741,21.656763 +32,bow,euclidean,0.10273,0.169297,0.180089,0.205461,0.163276,0.148805,0.00189,0.006965,0.011888,0.067505,0.323261,0.011614,0.008929,21.656763 +32,bow,manhattan,0.100683,0.169626,0.18202,0.201365,0.166416,0.151195,0.001951,0.007423,0.012696,0.070767,0.337468,0.009232,0.002919,21.656763 +64,tfidf,cosine,0.104096,0.174637,0.186713,0.208191,0.169556,0.15256,0.002174,0.007639,0.013329,0.069772,0.320509,0.008424,0.005952,31.627918 +64,tfidf,euclidean,0.104096,0.174637,0.186713,0.208191,0.169556,0.15256,0.002174,0.007639,0.013329,0.069772,0.320509,0.012393,0.010553,31.627918 +64,tfidf,manhattan,0.107509,0.178845,0.192158,0.215017,0.173925,0.156928,0.002218,0.008196,0.014401,0.077029,0.348793,0.008497,0.002971,31.627918 +64,bow,cosine,0.104096,0.174637,0.186713,0.208191,0.169556,0.15256,0.002174,0.007639,0.013329,0.069772,0.320509,0.008539,0.005804,31.627918 +64,bow,euclidean,0.104096,0.174637,0.186713,0.208191,0.169556,0.15256,0.002174,0.007639,0.013329,0.069772,0.320509,0.008436,0.002972,31.627918 +64,bow,manhattan,0.107509,0.178845,0.192158,0.215017,0.173925,0.156928,0.002218,0.008196,0.014401,0.077029,0.348793,0.008398,0.00289,31.627918 +96,tfidf,cosine,0.104096,0.17209,0.182948,0.208191,0.165734,0.144096,0.00226,0.008142,0.013041,0.06597,0.29659,0.008393,0.005767,40.591512 +96,tfidf,euclidean,0.104096,0.17209,0.182948,0.208191,0.165734,0.144096,0.00226,0.008142,0.013041,0.06597,0.29659,0.00842,0.002927,40.591512 +96,tfidf,manhattan,0.109556,0.183257,0.194501,0.219113,0.178567,0.155836,0.002355,0.008961,0.015032,0.074333,0.333904,0.008205,0.002885,40.591512 +96,bow,cosine,0.104096,0.17209,0.182948,0.208191,0.165734,0.144096,0.00226,0.008142,0.013041,0.06597,0.29659,0.008321,0.005724,40.591512 +96,bow,euclidean,0.104096,0.17209,0.182948,0.208191,0.165734,0.144096,0.00226,0.008142,0.013041,0.06597,0.29659,0.00822,0.002938,40.591512 +96,bow,manhattan,0.109556,0.183257,0.194501,0.219113,0.178567,0.155836,0.002355,0.008961,0.015032,0.074333,0.333904,0.008179,0.002967,40.591512 +128,tfidf,cosine,0.101365,0.169825,0.180633,0.20273,0.164778,0.144915,0.00215,0.007843,0.012925,0.065729,0.298031,0.008121,0.005813,53.837433 +128,tfidf,euclidean,0.101365,0.169825,0.180633,0.20273,0.164778,0.144915,0.00215,0.007843,0.012925,0.065729,0.298031,0.008233,0.003011,53.837433 +128,tfidf,manhattan,0.107509,0.17803,0.190932,0.215017,0.173515,0.155836,0.002454,0.008646,0.014476,0.075875,0.343874,0.008241,0.002946,53.837433 +128,bow,cosine,0.101365,0.169825,0.180633,0.20273,0.164778,0.144915,0.00215,0.007843,0.012925,0.065729,0.298031,0.00838,0.005956,53.837433 +128,bow,euclidean,0.101365,0.169825,0.180633,0.20273,0.164778,0.144915,0.00215,0.007843,0.012925,0.065729,0.298031,0.008259,0.003041,53.837433 +128,bow,manhattan,0.107509,0.17803,0.190932,0.215017,0.173515,0.155836,0.002454,0.008646,0.014476,0.075875,0.343874,0.008303,0.002881,53.837433 +256,tfidf,cosine,0.092833,0.157529,0.167848,0.185666,0.15372,0.133174,0.001937,0.007386,0.012404,0.057853,0.273233,0.008392,0.007015,87.10075 +256,tfidf,euclidean,0.092833,0.157529,0.167848,0.185666,0.15372,0.133174,0.001937,0.007386,0.012404,0.057853,0.273233,0.008387,0.003144,87.10075 +256,tfidf,manhattan,0.094198,0.161344,0.171961,0.188396,0.159181,0.139863,0.002035,0.007839,0.013104,0.063376,0.31267,0.00851,0.002965,87.10075 +256,bow,cosine,0.092833,0.157529,0.167848,0.185666,0.15372,0.133174,0.001937,0.007386,0.012404,0.057853,0.273233,0.00845,0.006758,87.10075 +256,bow,euclidean,0.092833,0.157529,0.167848,0.185666,0.15372,0.133174,0.001937,0.007386,0.012404,0.057853,0.273233,0.00843,0.003153,87.10075 +256,bow,manhattan,0.094198,0.161344,0.171961,0.188396,0.159181,0.139863,0.002035,0.007839,0.013104,0.063376,0.31267,0.008601,0.002972,87.10075 diff --git a/resnet_knn_cpu_eval.py b/resnet_knn_cpu_eval.py new file mode 100644 index 0000000..eb8cf91 --- /dev/null +++ b/resnet_knn_cpu_eval.py @@ -0,0 +1,189 @@ +import gc +import warnings +from time import time + +import cv2 +import matplotlib.pyplot as plt +import pandas as pd +import torch +import torchvision +import torchvision.transforms as transforms +from PIL import Image + +from cbir import * +from cbir.pipeline import * +from cbir.utils.grid import grid + +# Ignore all warnings +warnings.filterwarnings("ignore") + +# Load data +mean = [0.485, 0.456, 0.406] +std = [0.229, 0.224, 0.225] +transform = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Resize((224, 224)), + # transforms.Normalize(mean, std), + # lambda x: torch.flip(x, [1]), + # transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + ] +) +dataset = torchvision.datasets.ImageFolder( + root="./data/caltech101/train", + transform=transform, +) + +valset = torchvision.datasets.ImageFolder( + root="./data/caltech101/val", + transform=transform, +) + +testset = torchvision.datasets.ImageFolder( + root="./data/caltech101/test", + transform=transform, +) + +dataloader = torch.utils.data.DataLoader( + dataset, batch_size=128, shuffle=False, num_workers=2 +) + +valloader = torch.utils.data.DataLoader( + valset, batch_size=128, shuffle=False, num_workers=2 +) + +testloader = torch.utils.data.DataLoader( + testset, batch_size=128, shuffle=False, num_workers=2 +) + +# BEGIN EVALUATION +eval = pd.DataFrame( + columns=[ + "model", + "metric", + "map@1", + "map@5", + "map@10", + "hit_rate@1", + "hit_rate@5", + "hit_rate@10", + "recall@1", + "recall@5", + "recall@10", + "recall@100", + "recall@1000", + "avg_indexing_time", + "avg_retrieval_time", + ] +) + +models = ["resnet18", "resnet34", "resnet50", "resnet101", "resnet152"] +knn_metrics = ["euclidean", "cosine"] +for model, metric in grid(models, knn_metrics): + print("Evaluate for model: ", model, " with knn metric: ", metric) + + # Initialization + resnet = ResNetExtractor(model=model, device="cpu") + array_store = NPArrayStore(retrieve=KNNRetrieval(metric=metric)) + cbir = CBIR(resnet, array_store) + + # Indexing + start = time() + for images, labels in tqdm(dataloader, desc="Indexing"): + images = images.numpy() + cbir.indexing(images) + avg_indexing_time = round((time() - start) / len(dataset), 6) + + # Retrieval + start = time() + rs = [] + ground_truth = [] + for images, labels in tqdm(testloader, desc="Retrieval"): + images = images.numpy() + rs.extend(cbir.retrieve(images, k=1000)) + ground_truth.extend(labels) + avg_retrieval_time = round((time() - start) / len(dataset), 6) + + # Evaluation + ap1 = [] + hit1 = [] + recall1 = [] + ap5 = [] + hit5 = [] + recall5 = [] + ap10 = [] + hit10 = [] + recall10 = [] + recall100 = [] + recall1000 = [] + for r, g in zip(rs, ground_truth): + predicted = [] + for i in r: + predicted.append(i.index) + class_preds = np.take(dataset.targets, predicted, axis=0) + ap1.append(average_precision(class_preds.tolist(), [g.tolist()], 1)) + hit1.append(hit_rate(class_preds.tolist(), [g.tolist()], 1)) + recall1.append(recall(predicted, np.where(np.isin(np.array(dataset.targets), [g.tolist()]))[0], 1)) + ap5.append(average_precision(class_preds.tolist(), [g.tolist()], 5)) + hit5.append(hit_rate(class_preds.tolist(), [g.tolist()], 5)) + recall5.append(recall(predicted, np.where(np.isin(np.array(dataset.targets), [g.tolist()]))[0], 5)) + ap10.append(average_precision(class_preds.tolist(), [g.tolist()], 10)) + hit10.append(hit_rate(class_preds.tolist(), [g.tolist()], 10)) + recall10.append(recall(predicted, np.where(np.isin(np.array(dataset.targets), [g.tolist()]))[0], 10)) + recall100.append(recall(predicted, np.where(np.isin(np.array(dataset.targets), [g.tolist()]))[0], 100)) + recall1000.append(recall(predicted, np.where(np.isin(np.array(dataset.targets), [g.tolist()]))[0], 1000)) + + + map1 = round(np.mean(ap1), 6) + avg_hit1 = round(np.mean(hit1), 6) + avg_recall1 = round(np.mean(recall1), 6) + map5 = round(np.mean(ap5), 6) + avg_hit5 = round(np.mean(hit5), 6) + avg_recall5 = round(np.mean(recall5), 6) + map10 = round(np.mean(ap10), 6) + avg_hit10 = round(np.mean(hit10), 6) + avg_recall10 = round(np.mean(recall10), 6) + avg_recall100 = round(np.mean(recall100), 6) + avg_recall1000 = round(np.mean(recall1000), 6) + + new_row = pd.DataFrame( + { + "model": [model], + "metric": [metric], + "map@1": [map1], + "map@5": [map5], + "map@10": [map10], + "hit_rate@1": [avg_hit1], + "hit_rate@5": [avg_hit5], + "hit_rate@10": [avg_hit10], + "recall@1": [avg_recall1], + "recall@5": [avg_recall5], + "recall@10": [avg_recall10], + "recall@100": [avg_recall100], + "recall@1000": [avg_recall1000], + "avg_indexing_time": [avg_indexing_time], + "avg_retrieval_time": [avg_retrieval_time], + } + ) + eval = pd.concat([eval, new_row], ignore_index=True) + print( + "map@1: ", map1, + "map@5: ", map5, + "map@10: ", map10, + "hit_rate@1: ", avg_hit1, + "hit_rate@5: ", avg_hit5, + "hit_rate@10: ", avg_hit10, + "recall@1: ", avg_recall1, + "recall@5: ", avg_recall5, + "recall@10: ", avg_recall10, + "recall@100: ", avg_recall100, + "recall@1000: ", avg_recall1000, + "avg_indexing_time: ", avg_indexing_time, + "avg_retrieval_time: ", avg_retrieval_time, + ) + + # Cleanup + del cbir + del array_store + gc.collect() +eval.to_csv("out/resnet_knn_cpu_eval.csv", index=False)