Add EOPA as an engine option (#1000)

And extend capabilities to allow fetching from remote URLs.

README.md

@@ -566,6 +566,28 @@ capabilities:
           type: object
 ```
 
+### Loading Capabilities from URLs
+
+Starting with Regal version TODO, Regal can load capabilities from URLs with
+the `http`, or `https` schemes using the `capabilities.from.url` config key.
+For example, to load capabilities from `https://example.org/capabilities.json`,
+this configuration could be used:
+
+```yaml
+capabilities:
+  from:
+    url: https://example.org/capabilities.json
+```
+
+### Supported Engines
+
+Regal includes capabilities files for the following engines:
+
+| Engine | Website | Description |
+|--------|---------|-------------|
+| `opa`  | [OPA website](https://www.openpolicyagent.org/) | Open Policy Agent |
+| `eopa` | [Enterprise OPA website](https://www.styra.com/enterprise-opa/) | Styra Enterprise OPA |
+
 ## Exit Codes
 
 Exit codes are used to indicate the result of the `lint` command. The `--fail-level` provided for `regal lint` may be

build/do.rq

@@ -27,6 +27,19 @@ package script
 
 import future.keywords
 
+# Path to Regal's project root. (./build/do.rq/../.. => ./)
+regal_root := rq.abs(rq.dir(rq.dir(rq.scriptpath())))
+
+build_dir := rq.joinpath([regal_root, "build"])
+
+# Root of the directory hierarchy where engine-specific capabilities JSON for
+# embedding into the Regal binary should be placed.
+embedded_caps_dir := rq.joinpath([regal_root, "internal", "capabilities", "embedded"])
+
+# Directory where EOPA-specific JSON capabilities for embedding should be
+# placed.
+eopa_caps_dir := rq.joinpath([embedded_caps_dir, "eopa"])
+
 main contains do[what]  if some what in rq.args()
 main contains job[what] if some what in rq.args()
 main contains job.tasks if {
@@ -93,6 +106,14 @@ job.test {
 	test
 }
 
+# METADATA
+# title: fetch
+# description: Fetch third-party artifacts, such as capabilities JSON files for engines.
+job.fetch {
+	fetch_engine_caps
+}
+
+
 # METADATA
 # title: lint
 # description: Run `regal lint` on the Regal bundle
@@ -141,6 +162,7 @@ test {
 
 e2e {
 	run("go test -tags e2e ./e2e")
+	run("go test -tags integration ./internal/capabilities")
 }
 
 lint {
@@ -163,6 +185,137 @@ write_readme {
 	run("./regal table --write-to-readme bundle")
 }
 
+fetch_engine_caps {
+	fetch_eopa_caps
+}
+
+fetch_eopa_caps {
+
+	# git ls-remote --tags output looks like this:
+	#
+	#	...
+	#	bab5bdd4bb2de45c7d1bcf5d7b5df3c61a131f38        refs/tags/v1.0.0
+	#	bab5bdd4bb2de45c7d1bcf5d7b5df3c61a131f38        refs/tags/v1.0.1
+	#	9ace37dda9859a1afd2fcd6a87dd7f1955be4dc4        refs/tags/v1.10.1
+	#	...
+	#
+	# so .stdout will look like this:
+	#
+	#	...
+	#	[
+	#		"bab5bdd4bb2de45c7d1bcf5d7b5df3c61a131f38\trefs",
+	#		"tags",
+	#		"v1.0.0"
+	#	],
+	#	[
+	#		"bab5bdd4bb2de45c7d1bcf5d7b5df3c61a131f38\trefs",
+	#		"tags",
+	#		"v1.0.1"
+	#	],
+	#	[
+	#		"bab5bdd4bb2de45c7d1bcf5d7b5df3c61a131f38\trefs",
+	#		"tags",
+	#		"v1.1.0"
+	#	],
+	#	...
+
+	print("fetching tags for enterprise-opa repository")
+
+	eopa_tags_result := rq.run([
+			"git",
+			"ls-remote",
+			"--tags",
+			"https://github.com/styrainc/enterprise-opa"
+		], {
+			"stdout_spec": {
+				"format": "raw",
+				"options": {
+					"raw.fs": "/",
+					"raw.rs": "[\n\r]"
+				},
+			}
+		}
+	)
+
+	error_nonzero(eopa_tags_result, "failed to fetch tags from GitHub")
+
+	# We assume that tags and capabilities files are 1:1, but some EOPA
+	# release tags in the past did not correctly get capabilities files, so
+	# we eliminate them from consideration.
+
+	known_bad_tags := {
+		"v1.15.0",   # tag missing capabilities file (misnamed v0.15.0)
+		"v1.4.1",    # tag missing capabilities file
+		"v1.5.0",    # tag missing capabilities file
+	}
+
+	# Note that we use the `not startswith` to explicitly drop any
+	# pre-1.0.0 tags. There is little reason anyone would want them, and if
+	# they really have a need, they can manually download them and use the
+	# file locally.
+
+	eopa_tags := {
+		t
+		|
+		some r in eopa_tags_result.stdout
+		t := r[2]
+		not known_bad_tags[t]
+		not startswith(t, "v0.")
+	}
+
+	# Get a directory listing for the capabilities directory, filtering for
+	# only nonzero size files with JSON extensions. The size check is to
+	# avoid long-tail edge cases where we crashed after opening the file
+	# for writing but before committing any content.
+	eopa_caps_tree := {
+		p: f
+		|
+		f := rq.tree(eopa_caps_dir, {})[p]
+		f.size != 0
+		f.ext == "json"
+	}
+
+	# Determine which capabilities files are missing, what URL they
+	# should be fetched from, and where they should end up on disk.
+	missing_locally := {
+		{"local": p, "remote": r}
+
+		|
+
+		# construct the local path we expect the caps to exist at
+		t := eopa_tags[_]
+		p := rq.joinpath([eopa_caps_dir, sprintf("%s.json", [t])])
+
+		not eopa_caps_tree[p]
+
+		# construct the URL to fetch the content from
+		r := rq.template("https://raw.githubusercontent.com/StyraInc/enterprise-opa/main/capabilities/{{.tag}}.json", {"tag": t})
+	}
+
+	print(sprintf("fetching %d capabilities files missing locally", [count(missing_locally)]))
+
+	# Download the capabilities from the constructed URLs.
+	new_caps := {
+		{"local": m.local, "content": c}
+		|
+		m := missing_locally[_]
+		print("\tfetching ", m.remote)
+		resp := http.send({"url": m.remote, "method": "GET"})
+
+		{ rq.error(sprintf("non-200 status code '%d' for URL '%s'", [resp.status_code, m.remote])) | resp.status_code != 200 }
+
+		c := resp.raw_body
+	}
+
+	# Commit the retrieved content to disk.
+	{
+		rq.write(cap.content, {"format": "raw", "file_path": cap.local})
+		|
+		some cap in new_caps
+	}
+}
+
+
 fmt_all {
 	gci
 	gofumpt
@@ -226,3 +379,10 @@ github(what, j) {
 } else := true
 
 is_github if rq.env().GITHUB_ACTION
+
+error_nonzero(run_result, message) if {
+	run_result.exitcode != 0
+	rq.error(sprintf("%s\nstdout:%s\nstderr:\n%s\n", [message, run_result.stdout, run_result.stderr]))
+} else {
+	true
+}

go.mod

@@ -5,6 +5,7 @@ go 1.22.5
 require (
 	dario.cat/mergo v1.0.1
 	github.com/anderseknert/roast v0.1.0
+	github.com/coreos/go-semver v0.3.1
 	github.com/fatih/color v1.17.0
 	github.com/fsnotify/fsnotify v1.7.0
 	github.com/gobwas/glob v0.2.3

go.sum

@@ -23,6 +23,8 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
+github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

internal/ast/rule.go

@@ -53,7 +53,9 @@ func GetRuleDetail(rule *ast.Rule) string {
 	case ast.Call:
 		name := v[0].String()
 
-		if builtin, ok := rego.BuiltIns[name]; ok {
+		bis := rego.GetBuiltins()
+
+		if builtin, ok := bis[name]; ok {
 			retType := builtin.Decl.NamedResult().String()
 
 			detail += fmt.Sprintf(" (%s)", simplifyType(retType))