From 72edcc90d97a18888b32dc7bb703ce1fcc95f450 Mon Sep 17 00:00:00 2001 From: Luiz Carvalho Date: Wed, 9 Oct 2024 14:48:41 -0400 Subject: [PATCH] Rudimentary OPA evaluator This commit introduces a new policy evaluator that is based on OPA directly, instead of via conftest. There are plenty of TODOs and bugs. It is a proof of concept in order to help us decide whether or not moving away from conftest is feasible. Signed-off-by: Luiz Carvalho --- .vscode/launch.json | 10 +- cmd/validate/image.go | 4 +- internal/evaluator/common.go | 265 ++++++++++++ internal/evaluator/conftest_evaluator.go | 198 +-------- internal/evaluator/conftest_evaluator_test.go | 2 +- internal/evaluator/opa_evaluator.go | 399 ++++++++++++++++++ internal/opa/rule/rule.go | 17 + 7 files changed, 696 insertions(+), 199 deletions(-) create mode 100644 internal/evaluator/common.go create mode 100644 internal/evaluator/opa_evaluator.go diff --git a/.vscode/launch.json b/.vscode/launch.json index 07cb29c89..e87ec9208 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -96,20 +96,26 @@ "request": "launch", "mode": "auto", "program": "${workspaceFolder}/main.go", + "env": { + "EC_DEBUG": "true" + }, "args": [ "validate", "image", "--public-key", "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEZP/0htjhVt2y0ohjgtIIgICOtQtA\nnaYJRuLprwIv6FDhZ5yFjYUEtsmoNcW7rx2KM6FOXGsCX3BNc7qhHELT+g==\n-----END PUBLIC KEY-----", "--policy", - "github.com/enterprise-contract/config//slsa3", + "github.com/enterprise-contract/config//redhat", "--image", "quay.io/konflux-ci/ec-golden-image:latest", "--ignore-rekor", "--output", "data=data.yaml", "--output", - "text" + "text", + "--debug", + "--workers", + "1" ] }, { diff --git a/cmd/validate/image.go b/cmd/validate/image.go index 2538ada80..dbcfe8d93 100644 --- a/cmd/validate/image.go +++ b/cmd/validate/image.go @@ -44,7 +44,9 @@ import ( type imageValidationFunc func(context.Context, app.SnapshotComponent, *app.SnapshotSpec, policy.Policy, []evaluator.Evaluator, bool) (*output.Output, error) -var newConftestEvaluator = evaluator.NewConftestEvaluator +// TODO: Make this configurable via some sort of env var. +// var newConftestEvaluator = evaluator.NewConftestEvaluator +var newConftestEvaluator = evaluator.NewOPAEvaluator func validateImageCmd(validate imageValidationFunc) *cobra.Command { data := struct { diff --git a/internal/evaluator/common.go b/internal/evaluator/common.go new file mode 100644 index 000000000..cb7707798 --- /dev/null +++ b/internal/evaluator/common.go @@ -0,0 +1,265 @@ +package evaluator + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "path/filepath" + "strings" + "time" + + "github.com/enterprise-contract/ec-cli/internal/opa" + "github.com/enterprise-contract/ec-cli/internal/opa/rule" + "github.com/enterprise-contract/ec-cli/internal/policy/source" + "github.com/enterprise-contract/ec-cli/internal/utils" + "github.com/open-policy-agent/opa/ast" + log "github.com/sirupsen/logrus" + "github.com/spf13/afero" +) + +type contextKey string + +const ( + capabilitiesKey contextKey = "ec.evaluator.capabilities" + effectiveTimeKey contextKey = "ec.evaluator.effective_time" +) + +type testRunner interface { + Run(context.Context, []string) ([]Outcome, Data, error) +} + +// TODO: Come up with more specific name for this file, maybe multiple files? + +// createDataDirectory creates the base content in the data directory +func createDataDirectory(ctx context.Context, dataDir string, policy ConfigProvider) error { + fs := utils.FS(ctx) + exists, err := afero.DirExists(fs, dataDir) + if err != nil { + return err + } + if !exists { + log.Debugf("Data dir '%s' does not exist, will create.", dataDir) + _ = fs.MkdirAll(dataDir, 0755) + } + + if err := createConfigJSON(ctx, dataDir, policy); err != nil { + return err + } + + return nil +} + +// createConfigJSON creates the config.json file with the provided configuration +// in the data directory +func createConfigJSON(ctx context.Context, dataDir string, p ConfigProvider) error { + if p == nil { + return nil + } + configFilePath := filepath.Join(dataDir, "config.json") + + config := map[string]interface{}{ + "config": map[string]interface{}{}, + } + + pc := &struct { + WhenNs int64 `json:"when_ns"` + }{} + + // Now that the future deny logic is handled in the ec-cli and not in rego, + // this field is used only for the checking the effective times in the + // acceptable bundles list. Always set it, even when we are using the current + // time, so that a consistent current time is used everywhere. + pc.WhenNs = p.EffectiveTime().UnixNano() + + opts, err := p.SigstoreOpts() + if err != nil { + return err + } + + // Add the policy config we just prepared + config["config"] = map[string]interface{}{ + "policy": pc, + "default_sigstore_opts": opts, + } + + configJSON, err := json.MarshalIndent(config, "", " ") + if err != nil { + return err + } + + fs := utils.FS(ctx) + // Check to see if the data.json file exists + exists, err := afero.Exists(fs, configFilePath) + if err != nil { + return err + } + // if so, remove it + if exists { + if err := fs.Remove(configFilePath); err != nil { + return err + } + } + // write our jsonData content to the data.json file in the data directory under the workDir + log.Debugf("Writing config data to %s: %#v", configFilePath, string(configJSON)) + if err := afero.WriteFile(fs, configFilePath, configJSON, 0444); err != nil { + return err + } + + return nil +} + +func collectPolicyRules(ctx context.Context, sources []source.PolicySource, workDir string) (map[string]rule.Info, error) { + // hold all rule annotations from all policy sources + // NOTE: emphasis on _all rules from all sources_; meaning that if two rules + // exist with the same code in two separate sources the collected rule + // information is not deterministic + rules := policyRules{} + // Download all sources + for _, s := range sources { + dir, err := s.GetPolicy(ctx, workDir, false) + if err != nil { + log.Debugf("Unable to download source from %s!", s.PolicyUrl()) + // TODO do we want to download other policies instead of erroring out? + return nil, err + } + annotations := []*ast.AnnotationsRef{} + fs := utils.FS(ctx) + // We only want to inspect the directory of policy subdirs, not config or data subdirs. + if s.Subdir() == "policy" { + annotations, err = opa.InspectDir(fs, dir) + if err != nil { + errMsg := err + if err.Error() == "no rego files found in policy subdirectory" { + // Let's try to give some more robust messaging to the user. + policyURL, err := url.Parse(s.PolicyUrl()) + if err != nil { + return nil, errMsg + } + // Do we have a prefix at the end of the URL path? + // If not, this means we aren't trying to access a specific file. + // TODO: Determine if we want to check for a .git suffix as well? + pos := strings.LastIndex(policyURL.Path, ".") + if pos == -1 { + // Are we accessing a GitHub or GitLab URL? If so, are we beginning with 'https' or 'http'? + if (policyURL.Host == "github.com" || policyURL.Host == "gitlab.com") && (policyURL.Scheme == "https" || policyURL.Scheme == "http") { + log.Debug("Git Hub or GitLab, http transport, and no file extension, this could be a problem.") + errMsg = fmt.Errorf("%s.\nYou've specified a %s URL with an %s:// scheme.\nDid you mean: %s instead?", errMsg, policyURL.Hostname(), policyURL.Scheme, fmt.Sprint(policyURL.Host+policyURL.RequestURI())) + } + } + } + return nil, errMsg + } + } + + for _, a := range annotations { + if a.Annotations == nil { + continue + } + if err := rules.collect(a); err != nil { + return nil, err + } + } + } + + return rules, nil +} + +type policyRules map[string]rule.Info + +func (r *policyRules) collect(a *ast.AnnotationsRef) error { + if a.Annotations == nil { + return nil + } + + info := rule.RuleInfo(a) + + if info.ShortName == "" { + // no short name matching with the code from Metadata will not be + // deterministic + return nil + } + + code := info.Code + + if _, ok := (*r)[code]; ok { + return fmt.Errorf("found a second rule with the same code: `%s`", code) + } + + (*r)[code] = info + return nil +} + +func addRuleMetadata(ctx context.Context, result *Result, rules policyRules) { + code, ok := (*result).Metadata[metadataCode].(string) + if ok { + addMetadataToResults(ctx, result, rules[code]) + } +} + +func addMetadataToResults(ctx context.Context, r *Result, rule rule.Info) { + // Note that r.Metadata already includes some fields that we get from + // the real conftest violation and warning results, (as provided by + // lib.result_helper in the ec-policies rego). Here we augment it with + // other fields from rule.Metadata, which we get by opa-inspecting the + // rego source. + + if r.Metadata == nil { + return + } + // normalize collection to []string + if v, ok := r.Metadata[metadataCollections]; ok { + switch vals := v.(type) { + case []any: + col := make([]string, 0, len(vals)) + for _, c := range vals { + col = append(col, fmt.Sprint(c)) + } + r.Metadata[metadataCollections] = col + case []string: + // all good, mainly left for documentation of the normalization + default: + // remove unsupported collections attribute + delete(r.Metadata, metadataCollections) + } + } + + if rule.Title != "" { + r.Metadata[metadataTitle] = rule.Title + } + if rule.EffectiveOn != "" { + r.Metadata[metadataEffectiveOn] = rule.EffectiveOn + } + if rule.Severity != "" { + r.Metadata[metadataSeverity] = rule.Severity + } + if rule.Description != "" { + r.Metadata[metadataDescription] = rule.Description + } + if rule.Solution != "" { + r.Metadata[metadataSolution] = rule.Solution + } + if len(rule.Collections) > 0 { + r.Metadata[metadataCollections] = rule.Collections + } + if len(rule.DependsOn) > 0 { + r.Metadata[metadataDependsOn] = rule.DependsOn + } + + // If the rule has been effective for a long time, we'll consider + // the effective_on date not relevant and not bother including it + if effectiveTime, ok := ctx.Value(effectiveTimeKey).(time.Time); ok { + if effectiveOnString, ok := r.Metadata[metadataEffectiveOn].(string); ok { + effectiveOnTime, err := time.Parse(effectiveOnFormat, effectiveOnString) + if err == nil { + if effectiveOnTime.Before(effectiveTime.Add(effectiveOnTimeout)) { + delete(r.Metadata, metadataEffectiveOn) + } + } else { + log.Warnf("Invalid %q value %q", metadataEffectiveOn, rule.EffectiveOn) + } + } + } else { + log.Warnf("Could not get effectiveTime from context") + } +} diff --git a/internal/evaluator/conftest_evaluator.go b/internal/evaluator/conftest_evaluator.go index 6b5c26ef8..1c1dcd2b3 100644 --- a/internal/evaluator/conftest_evaluator.go +++ b/internal/evaluator/conftest_evaluator.go @@ -39,19 +39,14 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "github.com/enterprise-contract/ec-cli/internal/opa" - "github.com/enterprise-contract/ec-cli/internal/opa/rule" "github.com/enterprise-contract/ec-cli/internal/policy" "github.com/enterprise-contract/ec-cli/internal/policy/source" "github.com/enterprise-contract/ec-cli/internal/tracing" "github.com/enterprise-contract/ec-cli/internal/utils" ) -type contextKey string - const ( - runnerKey contextKey = "ec.evaluator.runner" - capabilitiesKey contextKey = "ec.evaluator.capabilities" - effectiveTimeKey contextKey = "ec.evaluator.effective_time" + conftestRunnerKey contextKey = "ec.conftest.evaluator.runner" ) // trim removes all failure, warning, success or skipped results that depend on @@ -162,10 +157,6 @@ func excludeDirectives(code string, rawTerm any) string { return fmt.Sprintf("%s%s", prefix, strings.Join(output, ", ")) } -type testRunner interface { - Run(context.Context, []string) ([]Outcome, Data, error) -} - const ( effectiveOnFormat = "2006-01-02T15:04:05Z" effectiveOnTimeout = -90 * 24 * time.Hour // keep effective_on metadata up to 90 days @@ -314,7 +305,7 @@ func NewConftestEvaluatorWithNamespace(ctx context.Context, policySources []sour c.policyDir = filepath.Join(c.workDir, "policy") c.dataDir = filepath.Join(c.workDir, "data") - if err := c.createDataDirectory(ctx); err != nil { + if err := createDataDirectory(ctx, c.dataDir, c.policy); err != nil { return nil, err } @@ -339,39 +330,9 @@ func (c conftestEvaluator) CapabilitiesPath() string { return path.Join(c.workDir, "capabilities.json") } -type policyRules map[string]rule.Info - -func (r *policyRules) collect(a *ast.AnnotationsRef) error { - if a.Annotations == nil { - return nil - } - - info := rule.RuleInfo(a) - - if info.ShortName == "" { - // no short name matching with the code from Metadata will not be - // deterministic - return nil - } - - code := info.Code - - if _, ok := (*r)[code]; ok { - return fmt.Errorf("found a second rule with the same code: `%s`", code) - } - - (*r)[code] = info - return nil -} - func (c conftestEvaluator) Evaluate(ctx context.Context, target EvaluationTarget) ([]Outcome, Data, error) { var results []Outcome - if trace.IsEnabled() { - region := trace.StartRegion(ctx, "ec:conftest-evaluate") - defer region.End() - } - // hold all rule annotations from all policy sources // NOTE: emphasis on _all rules from all sources_; meaning that if two rules // exist with the same code in two separate sources the collected rule @@ -426,7 +387,7 @@ func (c conftestEvaluator) Evaluate(ctx context.Context, target EvaluationTarget var r testRunner var ok bool - if r, ok = ctx.Value(runnerKey).(testRunner); r == nil || !ok { + if r, ok = ctx.Value(conftestRunnerKey).(testRunner); r == nil || !ok { // should there be a namespace defined or not allNamespaces := true @@ -627,159 +588,6 @@ func (c conftestEvaluator) computeSuccesses(result Outcome, rules policyRules, t return successes } -func addRuleMetadata(ctx context.Context, result *Result, rules policyRules) { - code, ok := (*result).Metadata[metadataCode].(string) - if ok { - addMetadataToResults(ctx, result, rules[code]) - } -} - -func addMetadataToResults(ctx context.Context, r *Result, rule rule.Info) { - // Note that r.Metadata already includes some fields that we get from - // the real conftest violation and warning results, (as provided by - // lib.result_helper in the ec-policies rego). Here we augment it with - // other fields from rule.Metadata, which we get by opa-inspecting the - // rego source. - - if r.Metadata == nil { - return - } - // normalize collection to []string - if v, ok := r.Metadata[metadataCollections]; ok { - switch vals := v.(type) { - case []any: - col := make([]string, 0, len(vals)) - for _, c := range vals { - col = append(col, fmt.Sprint(c)) - } - r.Metadata[metadataCollections] = col - case []string: - // all good, mainly left for documentation of the normalization - default: - // remove unsupported collections attribute - delete(r.Metadata, metadataCollections) - } - } - - if rule.Title != "" { - r.Metadata[metadataTitle] = rule.Title - } - if rule.EffectiveOn != "" { - r.Metadata[metadataEffectiveOn] = rule.EffectiveOn - } - if rule.Severity != "" { - r.Metadata[metadataSeverity] = rule.Severity - } - if rule.Description != "" { - r.Metadata[metadataDescription] = rule.Description - } - if rule.Solution != "" { - r.Metadata[metadataSolution] = rule.Solution - } - if len(rule.Collections) > 0 { - r.Metadata[metadataCollections] = rule.Collections - } - if len(rule.DependsOn) > 0 { - r.Metadata[metadataDependsOn] = rule.DependsOn - } - - // If the rule has been effective for a long time, we'll consider - // the effective_on date not relevant and not bother including it - if effectiveTime, ok := ctx.Value(effectiveTimeKey).(time.Time); ok { - if effectiveOnString, ok := r.Metadata[metadataEffectiveOn].(string); ok { - effectiveOnTime, err := time.Parse(effectiveOnFormat, effectiveOnString) - if err == nil { - if effectiveOnTime.Before(effectiveTime.Add(effectiveOnTimeout)) { - delete(r.Metadata, metadataEffectiveOn) - } - } else { - log.Warnf("Invalid %q value %q", metadataEffectiveOn, rule.EffectiveOn) - } - } - } else { - log.Warnf("Could not get effectiveTime from context") - } -} - -// createConfigJSON creates the config.json file with the provided configuration -// in the data directory -func createConfigJSON(ctx context.Context, dataDir string, p ConfigProvider) error { - if p == nil { - return nil - } - configFilePath := filepath.Join(dataDir, "config.json") - - config := map[string]interface{}{ - "config": map[string]interface{}{}, - } - - pc := &struct { - WhenNs int64 `json:"when_ns"` - }{} - - // Now that the future deny logic is handled in the ec-cli and not in rego, - // this field is used only for the checking the effective times in the - // acceptable bundles list. Always set it, even when we are using the current - // time, so that a consistent current time is used everywhere. - pc.WhenNs = p.EffectiveTime().UnixNano() - - opts, err := p.SigstoreOpts() - if err != nil { - return err - } - - // Add the policy config we just prepared - config["config"] = map[string]interface{}{ - "policy": pc, - "default_sigstore_opts": opts, - } - - configJSON, err := json.MarshalIndent(config, "", " ") - if err != nil { - return err - } - - fs := utils.FS(ctx) - // Check to see if the data.json file exists - exists, err := afero.Exists(fs, configFilePath) - if err != nil { - return err - } - // if so, remove it - if exists { - if err := fs.Remove(configFilePath); err != nil { - return err - } - } - // write our jsonData content to the data.json file in the data directory under the workDir - log.Debugf("Writing config data to %s: %#v", configFilePath, string(configJSON)) - if err := afero.WriteFile(fs, configFilePath, configJSON, 0444); err != nil { - return err - } - - return nil -} - -// createDataDirectory creates the base content in the data directory -func (c *conftestEvaluator) createDataDirectory(ctx context.Context) error { - fs := utils.FS(ctx) - dataDir := c.dataDir - exists, err := afero.DirExists(fs, dataDir) - if err != nil { - return err - } - if !exists { - log.Debugf("Data dir '%s' does not exist, will create.", dataDir) - _ = fs.MkdirAll(dataDir, 0755) - } - - if err := createConfigJSON(ctx, dataDir, c.policy); err != nil { - return err - } - - return nil -} - // createCapabilitiesFile writes the default OPA capabilities a file. func (c *conftestEvaluator) createCapabilitiesFile(ctx context.Context) error { fs := utils.FS(ctx) diff --git a/internal/evaluator/conftest_evaluator_test.go b/internal/evaluator/conftest_evaluator_test.go index 6202d92b7..a4d940028 100644 --- a/internal/evaluator/conftest_evaluator_test.go +++ b/internal/evaluator/conftest_evaluator_test.go @@ -62,7 +62,7 @@ func (m *mockTestRunner) Run(ctx context.Context, inputs []string) ([]Outcome, D } func withTestRunner(ctx context.Context, clnt testRunner) context.Context { - return context.WithValue(ctx, runnerKey, clnt) + return context.WithValue(ctx, conftestRunnerKey, clnt) } type testPolicySource struct{} diff --git a/internal/evaluator/opa_evaluator.go b/internal/evaluator/opa_evaluator.go new file mode 100644 index 000000000..1e2df1ea8 --- /dev/null +++ b/internal/evaluator/opa_evaluator.go @@ -0,0 +1,399 @@ +package evaluator + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + + "github.com/MakeNowJust/heredoc" + "github.com/enterprise-contract/ec-cli/internal/opa/rule" + "github.com/enterprise-contract/ec-cli/internal/policy/source" + "github.com/enterprise-contract/ec-cli/internal/utils" + ecc "github.com/enterprise-contract/enterprise-contract-controller/api/v1alpha1" + "github.com/open-policy-agent/opa/rego" + log "github.com/sirupsen/logrus" + "github.com/spf13/afero" + "sigs.k8s.io/yaml" +) + +const ( + opaRunnerKey contextKey = "ec.opa.evaluator.runner" +) + +type opaEvaluator struct { + policySources []source.PolicySource + workDir string + dataDir string + policyDir string + policy ConfigProvider + include *Criteria + exclude *Criteria + fs afero.Fs +} + +func NewOPAEvaluator(ctx context.Context, policySources []source.PolicySource, p ConfigProvider, source ecc.Source) (Evaluator, error) { + o := opaEvaluator{ + policySources: policySources, + policy: p, + fs: utils.FS(ctx), + } + + o.include, o.exclude = computeIncludeExclude(source, p) + + var err error + if o.workDir, err = utils.CreateWorkDir(o.fs); err != nil { + return nil, fmt.Errorf("creating work dir: %w", err) + } + log.Debugf("Created work dir %s", o.workDir) + + o.dataDir = filepath.Join(o.workDir, "data") + if err := createDataDirectory(ctx, o.dataDir, o.policy); err != nil { + return nil, fmt.Errorf("creating data dir: %w", err) + } + log.Debugf("Created data dir: %s", o.dataDir) + + o.policyDir = filepath.Join(o.workDir, "policy") + + // TODO: Handle capabilities. Probably doesn't need to be done via a file. + + log.Debug("opaEvaluator created") + return o, nil +} + +func (o opaEvaluator) Destroy() { + // TODO: Remove any working directories +} + +func (o opaEvaluator) CapabilitiesPath() string { + // TODO: This should probably not be part of the Evaluator interface. + return "" +} + +func (o opaEvaluator) Evaluate(ctx context.Context, target EvaluationTarget) ([]Outcome, Data, error) { + var results []Outcome + + rules, err := collectPolicyRules(ctx, o.policySources, o.workDir) + if err != nil { + return nil, nil, fmt.Errorf("collecting policy rules: %w", err) + } + + rules = o.selectPolicyRules(rules, target.Target) + + var r testRunner + var ok bool + if r, ok = ctx.Value(opaRunnerKey).(testRunner); r == nil || !ok { + r = &opaRunner{ + rules: rules, + loadPaths: []string{o.dataDir, o.policyDir}, + } + } + + log.Debugf("runner: %#v", r) + log.Debugf("inputs: %#v", target.Inputs) + + runResults, data, err := r.Run(ctx, target.Inputs) + if err != nil { + return nil, nil, fmt.Errorf("test runner: %w", err) + } + + effectiveTime := o.policy.EffectiveTime() + ctx = context.WithValue(ctx, effectiveTimeKey, effectiveTime) + + // Track how many rules have been processed. This is used later on to determine if anything + // at all was processed. + totalRules := 0 + + for i, result := range runResults { + log.Debugf("Evaluation result at %d: %#v", i, result) + warnings := []Result{} + failures := []Result{} + successes := []Result{} + + for i := range result.Warnings { + warning := result.Warnings[i] + // TODO: Hmm maybe rule metadata should already be done by the opa runner? + addRuleMetadata(ctx, &warning, rules) + + // TODO: Consider moving this to opaRunner + if getSeverity(warning) == severityFailure { + failures = append(failures, warning) + } else { + warnings = append(warnings, warning) + } + } + + for i := range result.Failures { + failure := result.Failures[i] + addRuleMetadata(ctx, &failure, rules) + + // TODO: Consider moving this to opaRunner + if getSeverity(failure) == severityWarning || !isResultEffective(failure, effectiveTime) { + warnings = append(warnings, failure) + } else { + failures = append(failures, failure) + } + } + + for i := range result.Successes { + success := result.Successes[i] + addRuleMetadata(ctx, &success, rules) + successes = append(successes, success) + } + + result.Warnings = warnings + result.Failures = failures + result.Successes = successes + + totalRules += len(result.Warnings) + len(result.Failures) + len(result.Successes) + + results = append(results, result) + } + + // TODO: Implement this. + trim(&results) + + // If no rules were checked, then we have effectively failed, because no tests were actually + // ran due to input error, etc. + if totalRules == 0 { + log.Error("no successes, warnings, or failures, check input") + return nil, nil, fmt.Errorf("no successes, warnings, or failures, check input") + } + + return results, data, nil +} + +func (o opaEvaluator) selectPolicyRules(allRules map[string]rule.Info, target string) map[string]rule.Info { + relevantRules := make(map[string]rule.Info) + for key, rule := range allRules { + matchers := []string{ + // foo.bar + rule.Code, + // foo + rule.CodePackage, + // foo.* + fmt.Sprintf("%s.*", rule.CodePackage), + // * + "*", + } + for _, collection := range rule.Collections { + matchers = append(matchers, fmt.Sprintf("@%s", collection)) + } + + includeScore := scoreMatches(matchers, o.include.get(target)) + excludeScore := scoreMatches(matchers, o.exclude.get(target)) + if includeScore > excludeScore { + relevantRules[key] = rule + } + } + return relevantRules +} + +type opaRunner struct { + rules map[string]rule.Info + loadPaths []string +} + +// TODO: We probably don't need a runner and can just use the evaluator directly. +func (o *opaRunner) Run(ctx context.Context, fileList []string) ([]Outcome, Data, error) { + // TODO: Make this better. We probably only ever want to support a single file at a time. + inputPath := fileList[0] + rawInput, err := os.ReadFile(inputPath) + if err != nil { + return nil, nil, fmt.Errorf("reading input file: %w", err) + } + var input = map[string]any{} + if err := yaml.Unmarshal(rawInput, &input); err != nil { + return nil, nil, fmt.Errorf("unmarshaling input file: %w", err) + } + + rulesInfo := make(map[string]rule.Info, 0) + for _, r := range o.rules { + rulesInfo[r.Code] = r + } + + if len(o.rules) != len(rulesInfo) { + return nil, nil, fmt.Errorf("hmm duplicated rules? TODO") + } + + // TODO: Probably better off using a template? + entrypoint := heredoc.Doc(` + package __evaluator__ + + import rego.v1 + `) + handled := map[string]bool{} + for _, r := range rulesInfo { + // Some rules may have the same path, e.g. multiple `deny` in the same package. We only + // need to query them once. + if handled[r.Path] { + continue + } + handled[r.Path] = true + entrypoint += heredoc.Doc(fmt.Sprintf(` + results contains result if { + some result in %s + } + `, r.Path)) + } + log.Debugf("Entrypoint: \n%s", entrypoint) + + // TODO: This is a hack... This should be done only once per source group. + if err := os.WriteFile(path.Join(o.loadPaths[1], "entrypoint.rego"), []byte(entrypoint), 0644); err != nil { + return nil, nil, err + } + + query := "data.__evaluator__.results" + + options := []func(r *rego.Rego){ + rego.Input(input), + rego.Query(query), + // TODO: Data doesn't seem to be getting loaded properly + rego.Load(o.loadPaths, nil), + // TODO: rego.Compiler? + } + + regoInstance := rego.New(options...) + opaResultSet, err := regoInstance.Eval(ctx) + if err != nil { + return nil, nil, fmt.Errorf("evaluating rego instance: %w", err) + } + + var seenRules = map[string]bool{} + + var outcomes []Outcome + for _, opaResult := range opaResultSet { + for _, expression := range opaResult.Expressions { + // Rego rules that are intended for evaluation should return a slice of values. + // For example, deny[msg] or violation[{"msg": msg}]. + // + // When an expression does not have a slice of values, the expression did not + // evaluate to true, and no message was returned. + expressionValues, _ := expression.Value.([]interface{}) + for _, v := range expressionValues { + var result Result + switch val := v.(type) { + case map[string]interface{}: + if result, err = newResult(val); err != nil { + return nil, nil, fmt.Errorf("processing OPA result: %w", err) + } + default: + continue + // TODO: Support policies that return other types. + } + + code := ExtractStringFromMetadata(result, metadataCode) + ruleInfo, found := rulesInfo[code] + if !found { + // TODO: Error? Just log? + return nil, nil, fmt.Errorf("hmm TODO unknown rule? %q", code) + } + + seenRules[code] = true + + outcome := Outcome{ + FileName: "", // TODO: what should this be? + Namespace: ruleInfo.Package, + } + + switch ruleInfo.Severity { + case "warning": + outcome.Warnings = append(outcome.Warnings, result) + case "failure": + outcome.Failures = append(outcome.Failures, result) + } + + outcomes = append(outcomes, outcome) + } + } + } + + // Now compute the results + // any rule left DID NOT get metadata added so it's a success + // this depends on the delete in addMetadata + for code, rule := range rulesInfo { + if seenRules[code] { + continue + } + + // TODO: Some duplication from conftest_evaluator. + success := Result{ + Message: "Pass", + Metadata: map[string]interface{}{ + metadataCode: code, + }, + } + + if rule.Title != "" { + success.Metadata[metadataTitle] = rule.Title + } + + if rule.Description != "" { + success.Metadata[metadataDescription] = rule.Description + } + + if len(rule.Collections) > 0 { + success.Metadata[metadataCollections] = rule.Collections + } + + if len(rule.DependsOn) > 0 { + success.Metadata[metadataDependsOn] = rule.DependsOn + } + + // TODO: Is this needed? + // if !c.isResultIncluded(success, target) { + // log.Debugf("Skipping result success: %#v", success) + // continue + // } + + if rule.EffectiveOn != "" { + success.Metadata[metadataEffectiveOn] = rule.EffectiveOn + } + + // Let's omit the solution text here because if the rule is passing + // already then the user probably doesn't care about the solution. + + outcomes = append(outcomes, Outcome{ + FileName: "", // TODO: what should this be? + Namespace: rule.Package, + Successes: []Result{success}, + }) + } + + return outcomes, nil, nil +} + +/* +TODO: Remove this whole block. + +Some things we need to consider: + +1. If OPA doesn't distinguish between data and policy files, we may want to add some checks that + only *.rego files are read from policy sources, and only non-*.rego files are read from data + sources. That seems important, right? +2. Need to handle effective on, severity, and term properly. +3. Need to expose the actual data. +*/ + +func newResult(metadata map[string]interface{}) (Result, error) { + if _, ok := metadata["msg"]; !ok { + return Result{}, fmt.Errorf("rule missing msg field: %v", metadata) + } + if _, ok := metadata["msg"].(string); !ok { + return Result{}, fmt.Errorf("msg field must be string: %v", metadata) + } + + result := Result{ + Message: metadata["msg"].(string), + Metadata: make(map[string]interface{}), + } + + for k, v := range metadata { + if k != "msg" { + result.Metadata[k] = v + } + } + + return result, nil +} diff --git a/internal/opa/rule/rule.go b/internal/opa/rule/rule.go index 5b17b7fda..b50e428ad 100644 --- a/internal/opa/rule/rule.go +++ b/internal/opa/rule/rule.go @@ -259,6 +259,20 @@ func dependsOn(a *ast.AnnotationsRef) []string { } } +func path(a *ast.AnnotationsRef) string { + return a.Path.String() +} + +func severity(a *ast.AnnotationsRef) string { + // TODO: Return constant values. Remove duplication from conftest_evaluator + name := a.Path[len(a.Path)-1].String() + name = strings.Trim(name, `"`) + if strings.HasPrefix(name, "deny") { + return "failure" + } + return "warning" +} + type RuleKind string const ( @@ -281,6 +295,7 @@ type Info struct { ShortName string Solution string Title string + Path string } func RuleInfo(a *ast.AnnotationsRef) Info { @@ -297,5 +312,7 @@ func RuleInfo(a *ast.AnnotationsRef) Info { Package: packageName(a), ShortName: shortName(a), Title: title(a), + Path: path(a), + Severity: severity(a), } }