perf: improving cel env setup performance

expressions/parser.go

@@ -50,6 +50,29 @@ func NewParser(options ...Option) (*Parser, error) {
 	return parser, nil
 }
 
+// Extend creates a new parser with the same environment configuration but extended with additional options
+func (p *Parser) Extend(options ...Option) (*Parser, error) {
+	env, err := p.CelEnv.Extend([]cel.EnvOption{}...)
+	if err != nil {
+		return nil, err
+	}
+
+	newParser := &Parser{
+		CelEnv:             env,
+		Provider:           p.Provider,
+		ExpectedReturnType: p.ExpectedReturnType,
+	}
+
+	for _, opt := range options {
+		if err := opt(newParser); err != nil {
+			return nil, err
+		}
+	}
+
+	return newParser, nil
+}
+
+// Validate validates an expression and returns a list of validation errors
 func (p *Parser) Validate(expression *parser.Expression) ([]*errorhandling.ValidationError, error) {
 	expr := expression.String()
 	ast, issues := p.CelEnv.Compile(expr)

expressions/parser_test.go

@@ -0,0 +1,80 @@
+package expressions_test
+
+import (
+	"testing"
+
+	"github.com/teamkeel/keel/expressions"
+	"github.com/teamkeel/keel/expressions/options"
+	"github.com/test-go/testify/require"
+)
+
+func TestNewOptions(t *testing.T) {
+	opt1 := options.WithVariable("first", "Text", false)
+
+	parser, err := expressions.NewParser(opt1)
+	require.NoError(t, err)
+
+	expr1 := "first"
+	_, issues := parser.CelEnv.Compile(expr1)
+	require.Len(t, issues.Errors(), 0)
+
+	opt2 := options.WithVariable("second", "Text", false)
+
+	err = opt2(parser)
+	require.NoError(t, err)
+
+	_, issues = parser.CelEnv.Compile(expr1)
+	require.Len(t, issues.Errors(), 0)
+
+	expr2 := "second"
+	_, issues = parser.CelEnv.Compile(expr2)
+	require.Len(t, issues.Errors(), 0)
+}
+
+func TestExtendsVariables(t *testing.T) {
+	opt1 := options.WithVariable("first", "Text", false)
+
+	parser1, err := expressions.NewParser(opt1)
+	require.NoError(t, err)
+
+	expr1 := "first"
+	_, issues := parser1.CelEnv.Compile(expr1)
+	require.Len(t, issues.Errors(), 0)
+
+	opt2 := options.WithVariable("second", "Text", false)
+
+	parser2, err := parser1.Extend(opt2)
+	require.NoError(t, err)
+
+	_, issues = parser2.CelEnv.Compile(expr1)
+	require.Len(t, issues.Errors(), 0)
+
+	expr2 := "second"
+	_, issues = parser2.CelEnv.Compile(expr2)
+	require.Len(t, issues.Errors(), 0)
+
+	// "second" should not exist in parser1
+	_, issues = parser1.CelEnv.Compile(expr2)
+	require.Len(t, issues.Errors(), 1)
+}
+
+func TestExtendsTypeProvider(t *testing.T) {
+	parser1, err := expressions.NewParser()
+	require.NoError(t, err)
+
+	expr := "ctx.identity"
+	_, issues := parser1.CelEnv.Compile(expr)
+	require.Len(t, issues.Errors(), 1)
+
+	opt2 := options.WithCtx()
+
+	parser2, err := parser1.Extend(opt2)
+	require.NoError(t, err)
+
+	_, issues = parser2.CelEnv.Compile(expr)
+	require.Len(t, issues.Errors(), 0)
+
+	// "ctx.identity" should not exist in parser1
+	_, issues = parser1.CelEnv.Compile(expr)
+	require.Len(t, issues.Errors(), 1)
+}

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/fatih/camelcase v1.0.0
 	github.com/goccy/go-yaml v1.15.13
 	github.com/golang-jwt/jwt/v4 v4.5.1
-	github.com/google/cel-go v0.21.0
+	github.com/google/cel-go v0.23.2
 	github.com/hashicorp/go-version v1.7.0
 	github.com/iancoleman/strcase v0.3.0
 	github.com/jackc/pgx/v5 v5.7.2
@@ -64,6 +64,7 @@ require (
 )
 
 require (
+	cel.dev/expr v0.19.1 // indirect
 	github.com/PaesslerAG/gval v1.2.4 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect

go.sum

@@ -1,3 +1,5 @@
+cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4=
+cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw=
 github.com/99designs/gqlgen v0.17.63 h1:HCdaYDPd9HqUXRchEvmE3EFzELRwLlaJ8DBuyC8Cqto=
 github.com/99designs/gqlgen v0.17.63/go.mod h1:sVCM2iwIZisJjTI/DEC3fpH+HFgxY1496ZJ+jbT9IjA=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
@@ -116,8 +118,8 @@ github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQg
 github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
-github.com/google/cel-go v0.21.0 h1:cl6uW/gxN+Hy50tNYvI691+sXxioCnstFzLp2WO4GCI=
-github.com/google/cel-go v0.21.0/go.mod h1:rHUlWCcBKgyEk+eV03RPdZUekPp6YcJwV0FxuUksYxc=
+github.com/google/cel-go v0.23.2 h1:UdEe3CvQh3Nv+E/j9r1Y//WO0K0cSyD7/y0bzyLIMI4=
+github.com/google/cel-go v0.23.2/go.mod h1:52Pb6QsDbC5kvgxvZhiL9QX1oZEkcUF/ZqaPx1J5Wwo=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=

schema/attributes/computed.go

@@ -1,25 +1,60 @@
 package attributes
 
 import (
+	"encoding/hex"
+
 	"github.com/iancoleman/strcase"
 	"github.com/teamkeel/keel/expressions"
 	"github.com/teamkeel/keel/expressions/options"
 	"github.com/teamkeel/keel/schema/parser"
 	"github.com/teamkeel/keel/schema/validation/errorhandling"
 )
 
-func ValidateComputedExpression(schema []*parser.AST, model *parser.ModelNode, field *parser.FieldNode, expression *parser.Expression) ([]*errorhandling.ValidationError, error) {
+var computed = make(map[string]*expressions.Parser)
+
+func defaultComputed(schema []*parser.AST) (*expressions.Parser, error) {
+	var contents string
+	for _, s := range schema {
+		contents += s.Raw + "\n"
+	}
+	key := hex.EncodeToString([]byte(contents))
+
+	if parser, exists := computed[key]; exists {
+		return parser, nil
+	}
+
 	opts := []expressions.Option{
+		options.WithCtx(),
 		options.WithSchemaTypes(schema),
-		options.WithVariable(strcase.ToLowerCamel(model.Name.Value), model.Name.Value, false),
 		options.WithComparisonOperators(),
 		options.WithLogicalOperators(),
 		options.WithArithmeticOperators(),
 		options.WithFunctions(),
+		options.WithReturnTypeAssertion(parser.FieldTypeBoolean, false),
+	}
+
+	parser, err := expressions.NewParser(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	computed[key] = parser
+
+	return parser, nil
+}
+
+func ValidateComputedExpression(schema []*parser.AST, model *parser.ModelNode, field *parser.FieldNode, expression *parser.Expression) ([]*errorhandling.ValidationError, error) {
+	parser, err := defaultComputed(schema)
+	if err != nil {
+		return nil, err
+	}
+
+	opts := []expressions.Option{
+		options.WithVariable(strcase.ToLowerCamel(model.Name.Value), model.Name.Value, false),
 		options.WithReturnTypeAssertion(field.Type.Value, field.Repeated),
 	}
 
-	p, err := expressions.NewParser(opts...)
+	p, err := parser.Extend(opts...)
 	if err != nil {
 		return nil, err
 	}

schema/attributes/permission.go

@@ -1,15 +1,31 @@
 package attributes
 
 import (
+	"encoding/hex"
+
 	"github.com/iancoleman/strcase"
 	"github.com/teamkeel/keel/expressions"
 	"github.com/teamkeel/keel/expressions/options"
+	"github.com/teamkeel/keel/expressions/resolve"
 	"github.com/teamkeel/keel/expressions/typing"
 	"github.com/teamkeel/keel/schema/parser"
 	"github.com/teamkeel/keel/schema/validation/errorhandling"
 )
 
-func ValidatePermissionExpression(schema []*parser.AST, model *parser.ModelNode, action *parser.ActionNode, job *parser.JobNode, expression *parser.Expression) ([]*errorhandling.ValidationError, error) {
+var permissions = make(map[string]*expressions.Parser)
+
+// defaultPermission will cache the base CEL environment for a schema
+func defaultPermission(schema []*parser.AST) (*expressions.Parser, error) {
+	var contents string
+	for _, s := range schema {
+		contents += s.Raw + "\n"
+	}
+	key := hex.EncodeToString([]byte(contents))
+
+	if parser, exists := permissions[key]; exists {
+		return parser, nil
+	}
+
 	opts := []expressions.Option{
 		options.WithCtx(),
 		options.WithSchemaTypes(schema),
@@ -18,15 +34,56 @@ func ValidatePermissionExpression(schema []*parser.AST, model *parser.ModelNode,
 		options.WithReturnTypeAssertion(parser.FieldTypeBoolean, false),
 	}
 
+	parser, err := expressions.NewParser(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	permissions[key] = parser
+
+	return parser, nil
+}
+
+func ValidatePermissionExpression(schema []*parser.AST, model *parser.ModelNode, action *parser.ActionNode, job *parser.JobNode, expression *parser.Expression) ([]*errorhandling.ValidationError, error) {
+	parser, err := defaultPermission(schema)
+	if err != nil {
+		return nil, err
+	}
+
+	opts := []expressions.Option{}
+
+	operands, err := resolve.IdentOperands(expression)
+	if err != nil {
+		return nil, err
+	}
+
 	if action != nil {
-		opts = append(opts, options.WithActionInputs(schema, action))
+		for _, operand := range operands {
+			for _, input := range action.Inputs {
+				if operand.Fragments[0] == input.Name() {
+					opts = append(opts, options.WithActionInputs(schema, action))
+					break
+				}
+			}
+		}
 	}
 
 	if model != nil {
-		opts = append(opts, options.WithVariable(strcase.ToLowerCamel(model.Name.Value), model.Name.Value, false))
+		for _, operand := range operands {
+			if operand.Fragments[0] == strcase.ToLowerCamel(model.Name.Value) {
+				opts = append(opts, options.WithVariable(strcase.ToLowerCamel(model.Name.Value), model.Name.Value, false))
+				break
+			}
+		}
 	}
 
-	p, err := expressions.NewParser(opts...)
+	// If there are no options to add, we can just validate without extending the parser
+	// This is a performance optimization
+	if len(opts) == 0 {
+		return parser.Validate(expression)
+	}
+
+	p, err := parser.Extend(opts...)
 	if err != nil {
 		return nil, err
 	}

schema/attributes/where.go

@@ -1,6 +1,8 @@
 package attributes
 
 import (
+	"encoding/hex"
+
 	"github.com/iancoleman/strcase"
 	"github.com/teamkeel/keel/expressions"
 	"github.com/teamkeel/keel/expressions/options"
@@ -9,20 +11,52 @@ import (
 	"github.com/teamkeel/keel/schema/validation/errorhandling"
 )
 
-func ValidateWhereExpression(schema []*parser.AST, action *parser.ActionNode, expression *parser.Expression) ([]*errorhandling.ValidationError, error) {
-	model := query.ActionModel(schema, action.Name.Value)
+var wheres = make(map[string]*expressions.Parser)
+
+// defaultWhere will cache the base CEL environment for a schema
+func defaultWhere(schema []*parser.AST) (*expressions.Parser, error) {
+	var contents string
+	for _, s := range schema {
+		contents += s.Raw + "\n"
+	}
+	key := hex.EncodeToString([]byte(contents))
+
+	if parser, exists := wheres[key]; exists {
+		return parser, nil
+	}
 
 	opts := []expressions.Option{
 		options.WithCtx(),
 		options.WithSchemaTypes(schema),
-		options.WithActionInputs(schema, action),
-		options.WithVariable(strcase.ToLowerCamel(model.Name.Value), model.Name.Value, false),
 		options.WithComparisonOperators(),
 		options.WithLogicalOperators(),
 		options.WithReturnTypeAssertion(parser.FieldTypeBoolean, false),
 	}
 
-	p, err := expressions.NewParser(opts...)
+	parser, err := expressions.NewParser(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	wheres[key] = parser
+
+	return parser, nil
+}
+
+func ValidateWhereExpression(schema []*parser.AST, action *parser.ActionNode, expression *parser.Expression) ([]*errorhandling.ValidationError, error) {
+	parser, err := defaultWhere(schema)
+	if err != nil {
+		return nil, err
+	}
+
+	model := query.ActionModel(schema, action.Name.Value)
+
+	opts := []expressions.Option{
+		options.WithActionInputs(schema, action),
+		options.WithVariable(strcase.ToLowerCamel(model.Name.Value), model.Name.Value, false),
+	}
+
+	p, err := parser.Extend(opts...)
 	if err != nil {
 		return nil, err
 	}

schema/parser/parser.go

@@ -16,6 +16,7 @@ import (
 type AST struct {
 	node.Node
 
+	Raw                  string
 	Declarations         []*DeclarationNode `@@*`
 	EnvironmentVariables []string
 	Secrets              []string
@@ -379,5 +380,7 @@ func Parse(s *reader.SchemaFile) (*AST, error) {
 		return schema, err
 	}
 
+	schema.Raw = s.Contents
+
 	return schema, nil
 }