feat: gator test --deny-only flag

Signed-off-by: Steeve Chailloux <[email protected]>

cmd/gator/test/gatortest_test.go

@@ -6,6 +6,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/open-policy-agent/frameworks/constraint/pkg/types"
 	"github.com/open-policy-agent/gatekeeper/v3/pkg/gator/test"
+	"github.com/open-policy-agent/gatekeeper/v3/pkg/util"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
 
@@ -97,3 +98,53 @@ func Test_formatOutput(t *testing.T) {
 		})
 	}
 }
+
+// Test_enforcableFailures makes sure that the `gator test` is able to detect
+// denied constraints when if found some.
+func Test_enforcableFailures(t *testing.T) {
+	testCases := []struct{
+		name string
+		input []*test.GatorResult
+		expectedOutput bool
+	}{
+		{
+			name: "don't fail on warn action",
+			input: []*test.GatorResult{{
+				Result: types.Result{
+					EnforcementAction: string(util.Warn),
+				},
+			}},
+			expectedOutput: false,
+		},
+		{
+			name: "fail on deny action",
+			input: []*test.GatorResult{{
+				Result: types.Result{
+					EnforcementAction: string(util.Deny),
+				},
+			}},
+			expectedOutput: true,
+		},
+		{
+			name: "fail if at least one scoped deny action",
+			input: []*test.GatorResult{{
+				Result: types.Result{
+					EnforcementAction: string(util.Dryrun),
+					ScopedEnforcementActions: []string{
+						string(util.Scoped),
+						string(util.Deny),
+					},
+				},
+			}},
+			expectedOutput: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			if ok := enforceableFailures(tc.input); ok != tc.expectedOutput {
+					t.Fatalf("unexpected output: %v", ok)
+			}
+		})
+	}
+}

cmd/gator/test/test.go

@@ -53,6 +53,7 @@ var (
 	flagImages       []string
 	flagTempDir      string
 	flagEnableK8sCel bool
+	flagDenyOnly     bool
 )
 
 const (
@@ -76,6 +77,7 @@ func init() {
 	Cmd.Flags().BoolVarP(&flagEnableK8sCel, "enable-k8s-native-validation", "", true, "enable the validating admission policy driver")
 	Cmd.Flags().StringArrayVarP(&flagImages, flagNameImage, "i", []string{}, "a URL to an OCI image containing policies. Can be specified multiple times.")
 	Cmd.Flags().StringVarP(&flagTempDir, flagNameTempDir, "d", "", fmt.Sprintf("Specifies the temporary directory to download and unpack images to, if using the --%s flag. Optional.", flagNameImage))
+	Cmd.Flags().BoolVarP(&flagDenyOnly, "deny-only", "", false, "output only denied constraints")
 }
 
 func run(_ *cobra.Command, _ []string) {
@@ -98,13 +100,20 @@ func run(_ *cobra.Command, _ []string) {
 	// Whether or not we return non-zero depends on whether we have a `deny`
 	// enforcementAction on one of the violated constraints
 	exitCode := 0
-	if enforceableFailure(results) {
+	if enforceableFailures(results) {
 		exitCode = 1
 	}
 	os.Exit(exitCode)
 }
 
-func formatOutput(flagOutput string, results []*test.GatorResult, stats []*instrumentation.StatsEntry) string {
+func formatOutput(flagOutput string, allResults []*test.GatorResult, stats []*instrumentation.StatsEntry) string {
+	var results []*test.GatorResult
+	for _, result := range allResults {
+		if flagDenyOnly && !enforceableFailure(result) {
+			continue
+		}
+		results = append(results, result)
+	}
 	switch strings.ToLower(flagOutput) {
 	case stringJSON:
 		var jsonB []byte
@@ -204,17 +213,23 @@ func formatOutput(flagOutput string, results []*test.GatorResult, stats []*instr
 	return ""
 }
 
-func enforceableFailure(results []*test.GatorResult) bool {
+func enforceableFailures(results []*test.GatorResult) bool {
 	for _, result := range results {
-		if result.EnforcementAction == string(util.Deny) {
+		if enforceableFailure(result) {
 			return true
 		}
-		for _, action := range result.ScopedEnforcementActions {
-			if action == string(util.Deny) {
-				return true
-			}
-		}
 	}
+	return false
+}
 
+func enforceableFailure(result *test.GatorResult) bool {
+	if result.EnforcementAction == string(util.Deny) {
+		return true
+	}
+	for _, action := range result.ScopedEnforcementActions {
+		if action == string(util.Deny) {
+			return true
+		}
+	}
 	return false
 }

website/docs/gator.md

@@ -86,6 +86,8 @@ cat my-manifest.yaml | gator test --image=localhost:5000/gator/template-library:
   --image=localhost:5000/gator/constraints:v1
 ```
 
+The `--deny-only` will only output violations about denied constraints, not the ones using `warn` enforcement action.
+
 #### Exit Codes
 
 `gator test` will return a `0` exit status when the objects, Templates, and