add the evaluation codes

README.md

@@ -1,5 +1,5 @@
 # Evaluation Toolbox for Salient Object Detection 
-Under construction ......
+To use this toolbox, read [README.md](./tools/README.md) in folder 'tools'.
 
 ## Datasets [[citations]](#benchmarks)
 - ECSSD  [`link`](http://www.cse.cuhk.edu.hk/leojia/projects/hsaliency/dataset.html)
@@ -667,7 +667,7 @@ All saliency maps are provided by the authors or calculated using their released
 - [x] add scores in [Table](#table)
 - [x] add pre-computed saliency maps
 - [x] add pre-computed `.mat` files
-- [ ] add evaluation codes
+- [x] add evaluation codes
 
 ## Cite This Repo
 **If you find this code useful in your research, please consider citing:**

tools/Curve_BenchCode/CalMAE.m

@@ -0,0 +1,21 @@
+function mae = CalMAE(smap, gtImg)
+% Code Author: Wangjiang Zhu
+% Email: [email protected]
+% Date: 3/24/2014
+[m,n,~] = size(gtImg);
+smap = im2double(smap(:,:,1));
+smap = imresize(smap,[m,n]);
+smap = (smap-min(smap(:)))/(max(smap(:))-min(smap(:)));
+
+if size(smap, 1) ~= size(gtImg, 1) || size(smap, 2) ~= size(gtImg, 2)
+    error('Saliency map and gt Image have different sizes!\n');
+end
+
+if ~islogical(gtImg)
+    gtImg = gtImg(:,:,1) > 128;
+end
+
+fgPixels = smap(gtImg);
+fgErrSum = length(fgPixels) - sum(fgPixels);
+bgErrSum = sum(smap(~gtImg));
+mae = (fgErrSum + bgErrSum) / numel(gtImg);

tools/Curve_BenchCode/Fmeasure_calu.m

@@ -0,0 +1,27 @@
+%%
+function Fmeasure = Fmeasure_calu(sMap,gtMap,gtsize)
+sumLabel =  2* mean(sMap(:)) ;
+if ( sumLabel > 1 )
+    sumLabel = 1;
+end
+
+Label3 = zeros( gtsize );
+Label3( sMap>=sumLabel ) = 1;
+
+NumRec = length( find( Label3==1 ) );
+LabelAnd = Label3 & gtMap;
+NumAnd = length( find ( LabelAnd==1 ) );
+num_obj = sum(sum(gtMap));
+
+if NumAnd == 0
+    PreFtem = 0;
+    RecallFtem = 0;
+    FmeasureF = 0;
+else
+    PreFtem = NumAnd/NumRec;
+    RecallFtem = NumAnd/num_obj;
+    FmeasureF = ( ( 1.3* PreFtem * RecallFtem ) / ( .3 * PreFtem + RecallFtem ) );
+end
+
+Fmeasure = [PreFtem, RecallFtem, FmeasureF];
+

tools/Curve_BenchCode/Fmeasure_calu_Rect.m

@@ -0,0 +1,32 @@
+%%
+function Fmeasure = Fmeasure_calu_Rect(sMap,gtMap,gtsize)
+sumLabel =  2* mean(sMap(:)) ;
+if ( sumLabel > 1 )
+    sumLabel = 1;
+end
+STATS = regionprops(uint8(sMap>=sumLabel),'BoundingBox');
+Label3 = zeros( gtsize );
+bbox = round([STATS.BoundingBox]);
+if (numel(bbox)>0)
+    Label3(bbox(2):(bbox(2)+bbox(4)-1),bbox(1):(bbox(1)+bbox(3)-1))=1;
+end
+
+NumRec = length( find( Label3==1 ) );
+LabelAnd = Label3 & gtMap;
+NumAnd = length( find ( LabelAnd==1 ) );
+num_obj = sum(sum(gtMap));
+
+if NumAnd == 0
+    PreFtem = 0;
+    RecallFtem = 0;
+    FmeasureF = 0;
+else
+    PreFtem = NumAnd/NumRec;
+    RecallFtem = NumAnd/num_obj;
+    FmeasureF = ( ( 1.3* PreFtem * RecallFtem ) / ( .3 * PreFtem + RecallFtem ) );
+end
+
+Fmeasure = [PreFtem, RecallFtem, FmeasureF];
+
+
+

tools/Curve_BenchCode/Pre_Recall_hitRate.m

@@ -0,0 +1,30 @@
+function [TPR, FPR, Pre, Recall, hitRate , falseAlarm] = Pre_Recall_hitRate(testMap,gtMap)
+%testMap：输入图像与阈值比较后的逻辑矩阵
+%gtMap：真值逻辑矩阵
+%输出：
+neg_gtMap = ~gtMap; %取相反数
+neg_testMap = ~testMap;
+
+hitCount = sum(sum(testMap.*gtMap));%二值分割后的图像中占真值中的元素（1）的个数（交）
+trueAvoidCount = sum(sum(neg_testMap.*neg_gtMap));%既不属于真值也不属于二值后的图像的元素的个数（1）
+missCount = sum(sum(testMap.*neg_gtMap));%二值分割后图像中错分的1的个数
+falseAvoidCount = sum(sum(neg_testMap.*gtMap));%二值后，真值中没有没检测到的个数
+
+if hitCount==0
+    Pre = 0;
+    Recall = 0;
+else
+    Pre = hitCount/(hitCount + missCount );
+    Recall = hitCount/(hitCount + falseAvoidCount);
+end
+
+TPR = Recall;
+FPR = missCount/(trueAvoidCount + missCount);
+falseAlarm = 1 - trueAvoidCount / (eps+trueAvoidCount + missCount);
+hitRate = hitCount / (eps+ hitCount + falseAvoidCount);
+end
+
+
+
+
+

tools/Curve_BenchCode/S_object.m

@@ -0,0 +1,58 @@
+function Q = S_object(prediction,GT)
+% S_object Computes the object similarity between foreground maps and ground
+% truth(as proposed in "Structure-measure:A new way to evaluate foreground 
+% maps" [Deng-Ping Fan et. al - ICCV 2017])
+% Usage:
+%   Q = S_object(prediction,GT)
+% Input:
+%   prediction - Binary/Non binary foreground map with values in the range
+%                [0 1]. Type: double.
+%   GT - Binary ground truth. Type: logical.
+% Output:
+%   Q - The object similarity score
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+% compute the similarity of the foreground in the object level
+prediction_fg = prediction;
+prediction_fg(~GT)=0;
+O_FG = Object(prediction_fg,GT);
+
+% compute the similarity of the background
+prediction_bg = 1.0 - prediction;
+prediction_bg(GT) = 0;
+O_BG = Object(prediction_bg,~GT);
+
+% combine the foreground measure and background measure together
+u = mean2(GT);
+Q = u * O_FG + (1 - u) * O_BG;
+
+end
+
+function score = Object(prediction,GT)
+
+% check the input 
+if isempty(prediction)
+    score = 0;
+    return;
+end
+if isinteger(prediction)
+    prediction = double(prediction);
+end
+if (~isa( prediction, 'double' ))
+    error('prediction should be of type: double');
+end
+if ((max(prediction(:))>1) || min(prediction(:))<0)
+    error('prediction should be in the range of [0 1]');
+end
+if(~islogical(GT))
+    error('GT should be of type: logical');
+end
+
+% compute the mean of the foreground or background in prediction
+x = mean2(prediction(GT));
+
+% compute the standard deviations of the foreground or background in prediction
+sigma_x = std(prediction(GT));
+
+score = 2.0 * x./(x^2 + 1.0 + sigma_x + eps);
+end

tools/Curve_BenchCode/S_region.m

@@ -0,0 +1,146 @@
+function Q = S_region(prediction,GT)
+% S_region computes the region similarity between the foreground map and
+% ground truth(as proposed in "Structure-measure:A new way to evaluate
+% foreground maps" [Deng-Ping Fan et. al - ICCV 2017])
+% Usage:
+%   Q = S_region(prediction,GT)
+% Input:
+%   prediction - Binary/Non binary foreground map with values in the range
+%                [0 1]. Type: double.
+%   GT - Binary ground truth. Type: logical.
+% Output:
+%   Q - The region similarity score
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+% find the centroid of the GT
+[X,Y] = centroid(GT);
+
+% divide GT into 4 regions
+[GT_1,GT_2,GT_3,GT_4,w1,w2,w3,w4] = divideGT(GT,X,Y);
+
+%Divede prediction into 4 regions
+[prediction_1,prediction_2,prediction_3,prediction_4] = Divideprediction(prediction,X,Y);
+
+%Compute the ssim score for each regions
+Q1 = ssim(prediction_1,GT_1);
+Q2 = ssim(prediction_2,GT_2);
+Q3 = ssim(prediction_3,GT_3);
+Q4 = ssim(prediction_4,GT_4);
+
+%Sum the 4 scores
+Q = w1 * Q1 + w2 * Q2 + w3 * Q3 + w4 * Q4;
+
+end
+
+function [X,Y] = centroid(GT)
+% Centroid Compute the centroid of the GT
+% Usage:
+%   [X,Y] = Centroid(GT)
+% Input:
+%   GT - Binary ground truth. Type: logical.
+% Output:
+%   [X,Y] - The coordinates of centroid.
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+[rows,cols] = size(GT);
+
+if(sum(GT(:))==0)
+    X = round(cols/2);
+    Y = round(rows/2);
+else     
+    total=sum(GT(:));
+    i=1:cols;
+    j=(1:rows)';
+    X=round(sum(sum(GT,1).*i)/total);
+    Y=round(sum(sum(GT,2).*j)/total);
+
+    %dGT = double(GT); 
+    %x = ones(rows,1)*(1:cols);
+    %y = (1:rows)'*ones(1,cols);
+    %area = sum(dGT(:));
+    %X = round(sum(sum(dGT.*x))/area);
+    %Y = round(sum(sum(dGT.*y))/area);
+end
+
+end
+
+% divide the GT into 4 regions according to the centroid of the GT and return the weights
+function [LT,RT,LB,RB,w1,w2,w3,w4] = divideGT(GT,X,Y)
+% LT - left top;
+% RT - right top;
+% LB - left bottom;
+% RB - right bottom;
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+%width and height of the GT
+[hei,wid] = size(GT);
+area = wid * hei;
+
+%copy the 4 regions 
+LT = GT(1:Y,1:X);
+RT = GT(1:Y,X+1:wid);
+LB = GT(Y+1:hei,1:X);
+RB = GT(Y+1:hei,X+1:wid);
+
+%The different weight (each block proportional to the GT foreground region).
+w1 = (X*Y)./area;
+w2 = ((wid-X)*Y)./area;
+w3 = (X*(hei-Y))./area;
+w4 = 1.0 - w1 - w2 - w3;
+end
+
+%Divide the prediction into 4 regions according to the centroid of the GT 
+function [LT,RT,LB,RB] = Divideprediction(prediction,X,Y)
+
+%width and height of the prediction
+[hei,wid] = size(prediction);
+
+%copy the 4 regions 
+LT = prediction(1:Y,1:X);
+RT = prediction(1:Y,X+1:wid);
+LB = prediction(Y+1:hei,1:X);
+RB = prediction(Y+1:hei,X+1:wid);
+
+end
+
+function Q = ssim(prediction,GT)
+% ssim computes the region similarity between foreground maps and ground
+% truth(as proposed in "Structure-measure: A new way to evaluate foreground 
+% maps" [Deng-Ping Fan et. al - ICCV 2017])
+% Usage:
+%   Q = ssim(prediction,GT)
+% Input:
+%   prediction - Binary/Non binary foreground map with values in the range
+%                [0 1]. Type: double.
+%   GT - Binary ground truth. Type: logical.
+% Output:
+%   Q - The region similarity score
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+dGT = double(GT);
+
+[hei,wid] = size(prediction);
+N = wid*hei;
+
+%Compute the mean of SM,GT
+x = mean2(prediction);
+y = mean2(dGT);
+
+%Compute the variance of SM,GT
+sigma_x2 = sum(sum((prediction - x).^2))./(N - 1 + eps);%sigma_x2 = var(prediction(:))
+sigma_y2 = sum(sum((dGT - y).^2))./(N - 1 + eps);       %sigma_y2 = var(dGT(:));      
+
+%Compute the covariance between SM and GT
+sigma_xy = sum(sum((prediction - x).*(dGT - y)))./(N - 1 + eps);
+
+alpha = 4 * x * y * sigma_xy;
+beta = (x.^2 + y.^2).*(sigma_x2 + sigma_y2);
+
+if(alpha ~= 0)
+    Q = alpha./(beta + eps);
+elseif(alpha == 0 && beta == 0)
+    Q = 1.0;
+else
+    Q = 0;
+end
+
+end

tools/Curve_BenchCode/StructureMeasure.m

@@ -0,0 +1,42 @@
+function Q = StructureMeasure(prediction,GT)
+% StructureMeasure computes the similarity between the foreground map and
+% ground truth(as proposed in "Structure-measure: A new way to evaluate
+% foreground maps" [Deng-Ping Fan et. al - ICCV 2017])
+% Usage:
+%   Q = StructureMeasure(prediction,GT)
+% Input:
+%   prediction - Binary/Non binary foreground map with values in the range
+%                [0 1]. Type: double.
+%   GT - Binary ground truth. Type: logical. 
+% Output:
+%   Q - The computed similarity score
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+% Check input
+if (~isa(prediction,'double'))
+    error('The prediction should be double type...');
+end
+if ((max(prediction(:))>1) || min(prediction(:))<0)
+    error('The prediction should be in the range of [0 1]...');
+end
+if (~islogical(GT))
+    error('GT should be logical type...');
+end
+
+y = mean2(GT);
+
+if (y==0)% if the GT is completely black
+    x = mean2(prediction);
+    Q = 1.0 - x; %only calculate the area of intersection
+elseif(y==1)%if the GT is completely white
+    x = mean2(prediction);
+    Q = x; %only calcualte the area of intersection
+else
+    alpha = 0.5;
+    Q = alpha*S_object(prediction,GT)+(1-alpha)*S_region(prediction,GT);
+    if (Q<0)
+      Q=0;
+    end
+end
+
+end