Merge pull request #1155 from XuanWang-Amos/OverrideTargetResult

Allow custom target status based on targetResult

pb/custom_evaluator/custom_evaluator.proto

@@ -52,6 +52,10 @@ message TestResultComparison {
     //
     // NOTE: Only supported for string and numerical values
     string test_result_error_field = 4;
+
+    // This will enable the target status comparation. The value of the status
+    // will be used to evaluate.
+    bool target_status = 5;
   }
 }
 
@@ -104,5 +108,8 @@ message Comparison {
 
     // For operations EQ, NE, LT, LE, GT, GE
     double numerical_value = 3;
+
+    // For operations EQ
+    testgrid.test_status.TestStatus target_status_value = 4;
   }
 }

pkg/updater/eval.go

@@ -69,6 +69,10 @@ func numNE(a, b float64) bool {
 	return a != b
 }
 
+func targetStatusEQ(a, b tspb.TestStatus) bool {
+	return a == b
+}
+
 // TestResult defines the interface for accessing data about the result.
 type TestResult interface {
 	// Properties for the test result.
@@ -83,6 +87,11 @@ type TestResult interface {
 	Exceptions() []string
 }
 
+// TargetResult defines the interface for accessing data about the target/suite result.
+type TargetResult interface {
+	TargetStatus() tspb.TestStatus
+}
+
 // CustomStatus evaluates the result according to the rules.
 //
 // Returns nil if no rule matches, otherwise returns the overridden status.
@@ -95,6 +104,18 @@ func CustomStatus(rules []*evalpb.Rule, testResult TestResult) *tspb.TestStatus
 	return nil
 }
 
+// CustomTargetStatus evaluates the result according to the rules.
+//
+// Returns nil if no rule matches, otherwise returns the overridden status.
+func CustomTargetStatus(rules []*evalpb.Rule, targetResult TargetResult) *tspb.TestStatus {
+	for _, rule := range rules {
+		if got := evalTargetProperties(rule, targetResult); got != nil {
+			return got
+		}
+	}
+	return nil
+}
+
 type jUnitTestResult struct {
 	Result *junit.Result
 }
@@ -231,3 +252,28 @@ func evalProperties(rule *evalpb.Rule, testResult TestResult) *tspb.TestStatus {
 	want := rule.GetComputedStatus()
 	return &want
 }
+
+func evalTargetProperties(rule *evalpb.Rule, targetResult TargetResult) *tspb.TestStatus {
+	for _, cmp := range rule.TestResultComparisons {
+		if cmp.Comparison == nil {
+			return nil
+		}
+		// Only EQ is supported
+		if cmp.Comparison.Op != evalpb.Comparison_OP_EQ {
+			return nil
+		}
+		// Only target_status is supported
+		if f := cmp.GetTargetStatus(); f == true {
+			getSts := targetResult.TargetStatus
+			stscmp := targetStatusEQ
+			stsval := cmp.Comparison.GetTargetStatusValue()
+			if !stscmp(getSts(), stsval) {
+				return nil
+			}
+		} else {
+			return nil
+		}
+	}
+	want := rule.GetComputedStatus()
+	return &want
+}

pkg/updater/eval_test.go

@@ -371,3 +371,116 @@ func TestCustomStatus(t *testing.T) {
 		})
 	}
 }
+
+type fakeTargetResult struct {
+	targetStatus tspb.TestStatus
+}
+
+func (r fakeTargetResult) TargetStatus() tspb.TestStatus {
+	return r.targetStatus
+}
+
+func makeTargetResult(status tspb.TestStatus) *fakeTargetResult {
+	return &fakeTargetResult{
+		targetStatus: status,
+	}
+}
+
+func TestCustomTargetStatus(t *testing.T) {
+	abort := tspb.TestStatus_CATEGORIZED_ABORT
+	cases := []struct {
+		name  string
+		rules []*evalpb.Rule
+		tgr   TargetResult
+		want  *tspb.TestStatus
+	}{
+		{
+			name: "empty",
+		},
+		{
+			name: "target status equal",
+			rules: []*evalpb.Rule{
+				{
+					ComputedStatus: tspb.TestStatus_CATEGORIZED_ABORT,
+					TestResultComparisons: []*evalpb.TestResultComparison{
+						{
+							TestResultInfo: &evalpb.TestResultComparison_TargetStatus{
+								TargetStatus: true,
+							},
+							Comparison: &evalpb.Comparison{
+								Op: evalpb.Comparison_OP_EQ,
+								ComparisonValue: &evalpb.Comparison_TargetStatusValue{
+									TargetStatusValue: tspb.TestStatus_TOOL_FAIL,
+								},
+							},
+						},
+					},
+				},
+			},
+			tgr:  makeTargetResult(tspb.TestStatus_TOOL_FAIL),
+			want: &abort,
+		},
+		{
+			name: "target status not equal",
+			rules: []*evalpb.Rule{
+				{
+					ComputedStatus: tspb.TestStatus_TIMED_OUT,
+					TestResultComparisons: []*evalpb.TestResultComparison{
+						{
+							TestResultInfo: &evalpb.TestResultComparison_TargetStatus{
+								TargetStatus: true,
+							},
+							Comparison: &evalpb.Comparison{
+								Op: evalpb.Comparison_OP_EQ,
+								ComparisonValue: &evalpb.Comparison_TargetStatusValue{
+									TargetStatusValue: tspb.TestStatus_CATEGORIZED_ABORT,
+								},
+							},
+						},
+					},
+				},
+			},
+			tgr:  makeTargetResult(tspb.TestStatus_TOOL_FAIL),
+			want: nil,
+		},
+		{
+			name: "target status comparation not enabled",
+			rules: []*evalpb.Rule{
+				{
+					ComputedStatus: tspb.TestStatus_CATEGORIZED_ABORT,
+					TestResultComparisons: []*evalpb.TestResultComparison{
+						{
+							TestResultInfo: &evalpb.TestResultComparison_PropertyKey{
+								PropertyKey: "want",
+							},
+							Comparison: &evalpb.Comparison{
+								Op: evalpb.Comparison_OP_EQ,
+								ComparisonValue: &evalpb.Comparison_TargetStatusValue{
+									TargetStatusValue: tspb.TestStatus_TOOL_FAIL,
+								},
+							},
+						},
+					},
+				},
+			},
+			tgr:  makeTargetResult(tspb.TestStatus_TOOL_FAIL),
+			want: nil,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := CustomTargetStatus(tc.rules, tc.tgr)
+			switch {
+			case got == nil:
+				if tc.want != nil {
+					t.Errorf("CustomTargetStatus() got nil, want %v", tc.want)
+				}
+			case tc.want == nil:
+				t.Errorf("CustomTargetStatus() should be nil, not %v", got)
+			case *tc.want != *got:
+				t.Errorf("CustomTargetStatus() got %v, want %v", *got, *tc.want)
+			}
+		})
+	}
+}