From 21f92eb13d0c141ba5326796810952f1ba169ec0 Mon Sep 17 00:00:00 2001 From: Thibault Koechlin Date: Tue, 15 Oct 2024 16:32:45 +0200 Subject: [PATCH 01/13] make MatchedRule a proper type --- pkg/types/appsec_event.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/types/appsec_event.go b/pkg/types/appsec_event.go index dc81c63b344..b41ebc9e904 100644 --- a/pkg/types/appsec_event.go +++ b/pkg/types/appsec_event.go @@ -18,7 +18,9 @@ len(evt.Waf.ByTagRx("*CVE*").ByConfidence("high").ByAction("block")) > 1 */ -type MatchedRules []map[string]interface{} +type MatchedRules []MatchedRule + +type MatchedRule map[string]interface{} type AppsecEvent struct { HasInBandMatches, HasOutBandMatches bool From 08ab38c700894cc26e70b81a791eee44ebeba1c6 Mon Sep 17 00:00:00 2001 From: Thibault Koechlin Date: Tue, 15 Oct 2024 17:18:43 +0200 Subject: [PATCH 02/13] switch to proper alert context --- pkg/acquisition/modules/appsec/utils.go | 120 +++++------------------- 1 file changed, 24 insertions(+), 96 deletions(-) diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index 4fb1a979d14..517720efc8a 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -3,8 +3,6 @@ package appsecacquisition import ( "fmt" "net" - "slices" - "strconv" "time" "github.com/oschwald/geoip2-golang" @@ -22,27 +20,27 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -var appsecMetaKeys = []string{ - "id", - "name", - "method", - "uri", - "matched_zones", - "msg", -} +// var appsecMetaKeys = []string{ +// "id", +// "name", +// "method", +// "uri", +// "matched_zones", +// "msg", +// } -func appendMeta(meta models.Meta, key string, value string) models.Meta { - if value == "" { - return meta - } +// func appendMeta(meta models.Meta, key string, value string) models.Meta { +// if value == "" { +// return meta +// } - meta = append(meta, &models.MetaItems0{ - Key: key, - Value: value, - }) +// meta = append(meta, &models.MetaItems0{ +// Key: key, +// Value: value, +// }) - return meta -} +// return meta +// } func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { // if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI @@ -60,6 +58,7 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { Scope: ptr.Of(types.Ip), } + //GeoIP enrich asndata, err := exprhelpers.GeoIPASNEnrich(sourceIP) if err != nil { @@ -88,6 +87,7 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { source.Range = record.String() } + // Build overflow evt.Overflow.Sources = make(map[string]models.Source) evt.Overflow.Sources[sourceIP] = source @@ -95,83 +95,11 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { alert.Capacity = ptr.Of(int32(1)) alert.Events = make([]*models.Event, len(evt.Appsec.GetRuleIDs())) - now := ptr.Of(time.Now().UTC().Format(time.RFC3339)) - - tmpAppsecContext := make(map[string][]string) - - for _, matched_rule := range inEvt.Appsec.MatchedRules { - evtRule := models.Event{} - - evtRule.Timestamp = now - - evtRule.Meta = make(models.Meta, 0) - - for _, key := range appsecMetaKeys { - if tmpAppsecContext[key] == nil { - tmpAppsecContext[key] = make([]string, 0) - } - - switch value := matched_rule[key].(type) { - case string: - evtRule.Meta = appendMeta(evtRule.Meta, key, value) - - if value != "" && !slices.Contains(tmpAppsecContext[key], value) { - tmpAppsecContext[key] = append(tmpAppsecContext[key], value) - } - case int: - val := strconv.Itoa(value) - evtRule.Meta = appendMeta(evtRule.Meta, key, val) - - if val != "" && !slices.Contains(tmpAppsecContext[key], val) { - tmpAppsecContext[key] = append(tmpAppsecContext[key], val) - } - case []string: - for _, v := range value { - evtRule.Meta = appendMeta(evtRule.Meta, key, v) - - if v != "" && !slices.Contains(tmpAppsecContext[key], v) { - tmpAppsecContext[key] = append(tmpAppsecContext[key], v) - } - } - case []int: - for _, v := range value { - val := strconv.Itoa(v) - evtRule.Meta = appendMeta(evtRule.Meta, key, val) - - if val != "" && !slices.Contains(tmpAppsecContext[key], val) { - tmpAppsecContext[key] = append(tmpAppsecContext[key], val) - } - } - default: - val := fmt.Sprintf("%v", value) - evtRule.Meta = appendMeta(evtRule.Meta, key, val) - - if val != "" && !slices.Contains(tmpAppsecContext[key], val) { - tmpAppsecContext[key] = append(tmpAppsecContext[key], val) - } - } - } - - alert.Events = append(alert.Events, &evtRule) - } - - metas := make([]*models.MetaItems0, 0) - - for key, values := range tmpAppsecContext { - if len(values) == 0 { - continue - } - - valueStr, err := alertcontext.TruncateContext(values, alertcontext.MaxContextValueLen) - if err != nil { - log.Warning(err.Error()) - } - - meta := models.MetaItems0{ - Key: key, - Value: valueStr, + metas, errors := alertcontext.AppsecEventToContext(inEvt.Appsec) + if len(errors) > 0 { + for _, err := range errors { + log.Errorf("failed to generate appsec context: %s", err) } - metas = append(metas, &meta) } alert.Meta = metas From 7edf1b5197d6487c77f5ddb6e9e7c75d7a696ede Mon Sep 17 00:00:00 2001 From: Thibault Koechlin Date: Tue, 15 Oct 2024 17:19:03 +0200 Subject: [PATCH 03/13] AppsecEventToAlertContext --- pkg/alertcontext/alertcontext.go | 119 +++++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 31 deletions(-) diff --git a/pkg/alertcontext/alertcontext.go b/pkg/alertcontext/alertcontext.go index 16ebc6d0ac2..18afa5298b4 100644 --- a/pkg/alertcontext/alertcontext.go +++ b/pkg/alertcontext/alertcontext.go @@ -30,7 +30,7 @@ type Context struct { func ValidateContextExpr(key string, expressions []string) error { for _, expression := range expressions { - _, err := expr.Compile(expression, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) + _, err := expr.Compile(expression, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}, "match": types.MatchedRule{}})...) if err != nil { return fmt.Errorf("compilation of '%s' failed: %w", expression, err) } @@ -72,7 +72,7 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error { } for _, value := range values { - valueCompiled, err := expr.Compile(value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) + valueCompiled, err := expr.Compile(value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}, "match": types.MatchedRule{}})...) if err != nil { return fmt.Errorf("compilation of '%s' context value failed: %w", value, err) } @@ -116,43 +116,87 @@ func TruncateContext(values []string, contextValueLen int) (string, error) { return ret, nil } -func EventToContext(events []types.Event) (models.Meta, []error) { +func EvalAlertContextRules(evt *types.Event, match *types.MatchedRule, tmpContext map[string][]string) []error { + var errors []error - metas := make([]*models.MetaItems0, 0) - tmpContext := make(map[string][]string) + for key, values := range alertContext.ContextToSendCompiled { + if _, ok := tmpContext[key]; !ok { + tmpContext[key] = make([]string, 0) + } - for _, evt := range events { - for key, values := range alertContext.ContextToSendCompiled { - if _, ok := tmpContext[key]; !ok { - tmpContext[key] = make([]string, 0) + for _, value := range values { + var val string + + output, err := expr.Run(value, map[string]interface{}{"match": match, "expr": evt}) + if err != nil { + errors = append(errors, fmt.Errorf("failed to get value for %s: %w", key, err)) + continue + } + //Need to support back []int and []string + switch out := output.(type) { + case string: + val = out + case int: + val = strconv.Itoa(out) + default: + errors = append(errors, fmt.Errorf("unexpected return type for %s: %T", key, output)) + continue } - for _, value := range values { - var val string - - output, err := expr.Run(value, map[string]interface{}{"evt": evt}) - if err != nil { - errors = append(errors, fmt.Errorf("failed to get value for %s: %w", key, err)) - continue - } - - switch out := output.(type) { - case string: - val = out - case int: - val = strconv.Itoa(out) - default: - errors = append(errors, fmt.Errorf("unexpected return type for %s: %T", key, output)) - continue - } - - if val != "" && !slices.Contains(tmpContext[key], val) { - tmpContext[key] = append(tmpContext[key], val) - } + if val != "" && !slices.Contains(tmpContext[key], val) { + tmpContext[key] = append(tmpContext[key], val) } } } + return errors +} + +// Iterate over the individual appsec matched rules to create the needed alert context. +func AppsecEventToContext(event types.AppsecEvent) (models.Meta, []error) { + var errors []error + + metas := make([]*models.MetaItems0, 0) + tmpContext := make(map[string][]string) + + for _, matched_rule := range event.MatchedRules { + tmpErrors := EvalAlertContextRules(&types.Event{}, &matched_rule, tmpContext) + errors = append(errors, tmpErrors...) + } + + for key, values := range tmpContext { + if len(values) == 0 { + continue + } + + valueStr, err := TruncateContext(values, alertContext.ContextValueLen) + if err != nil { + log.Warning(err.Error()) + } + + meta := models.MetaItems0{ + Key: key, + Value: valueStr, + } + metas = append(metas, &meta) + } + + ret := models.Meta(metas) + + return ret, errors +} + +// Iterate over the individual events to create the needed alert context. +func EventToContext(events []types.Event) (models.Meta, []error) { + var errors []error + + metas := make([]*models.MetaItems0, 0) + tmpContext := make(map[string][]string) + + for _, evt := range events { + tmpErrors := EvalAlertContextRules(&evt, &types.MatchedRule{}, tmpContext) + errors = append(errors, tmpErrors...) + } for key, values := range tmpContext { if len(values) == 0 { @@ -175,3 +219,16 @@ func EventToContext(events []types.Event) (models.Meta, []error) { return ret, errors } + +func appendMeta(meta models.Meta, key string, value string) models.Meta { + if value == "" { + return meta + } + + meta = append(meta, &models.MetaItems0{ + Key: key, + Value: value, + }) + + return meta +} From 9ac415fbf0a6e1c158fd79fff845d530d0b8baca Mon Sep 17 00:00:00 2001 From: Thibault Koechlin Date: Wed, 16 Oct 2024 16:24:17 +0200 Subject: [PATCH 04/13] generate empty event object --- pkg/types/event.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/types/event.go b/pkg/types/event.go index e016d0294c4..1746a3648a9 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -47,6 +47,13 @@ type Event struct { Meta map[string]string `yaml:"Meta,omitempty" json:"Meta,omitempty"` } +func EmptyEvent() *Event { + return &Event{Type: LOG, + Parsed: make(map[string]string), + Enriched: make(map[string]string), + Meta: make(map[string]string)} +} + func (e *Event) SetMeta(key string, value string) bool { if e.Meta == nil { e.Meta = make(map[string]string) From 6a03eef6ca0f6e7052b86380b42529291d732502 Mon Sep 17 00:00:00 2001 From: Thibault Koechlin Date: Wed, 16 Oct 2024 16:24:32 +0200 Subject: [PATCH 05/13] generate empty MatchedRule object --- pkg/types/appsec_event.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/types/appsec_event.go b/pkg/types/appsec_event.go index b41ebc9e904..cef8d6a6ba5 100644 --- a/pkg/types/appsec_event.go +++ b/pkg/types/appsec_event.go @@ -47,6 +47,10 @@ const ( Kind Field = "kind" ) +func EmptyMatchedRule() *MatchedRule { + return &MatchedRule{} +} + func (w AppsecEvent) GetVar(varName string) string { if w.Vars == nil { return "" From a8866d870439f43635f5eee0d5171a6cb4f386bc Mon Sep 17 00:00:00 2001 From: Thibault Koechlin Date: Wed, 16 Oct 2024 16:25:54 +0200 Subject: [PATCH 06/13] cleanup, support more ''complex'' types --- pkg/alertcontext/alertcontext.go | 141 ++++++++++++++++++------------- 1 file changed, 80 insertions(+), 61 deletions(-) diff --git a/pkg/alertcontext/alertcontext.go b/pkg/alertcontext/alertcontext.go index 18afa5298b4..32cf67d90b0 100644 --- a/pkg/alertcontext/alertcontext.go +++ b/pkg/alertcontext/alertcontext.go @@ -3,6 +3,7 @@ package alertcontext import ( "encoding/json" "fmt" + "net/http" "slices" "strconv" @@ -30,7 +31,10 @@ type Context struct { func ValidateContextExpr(key string, expressions []string) error { for _, expression := range expressions { - _, err := expr.Compile(expression, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}, "match": types.MatchedRule{}})...) + _, err := expr.Compile(expression, exprhelpers.GetExprOptions(map[string]interface{}{ + "evt": &types.Event{}, + "match": &types.MatchedRule{}, + "req": &http.Request{}})...) if err != nil { return fmt.Errorf("compilation of '%s' failed: %w", expression, err) } @@ -72,7 +76,10 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error { } for _, value := range values { - valueCompiled, err := expr.Compile(value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}, "match": types.MatchedRule{}})...) + valueCompiled, err := expr.Compile(value, exprhelpers.GetExprOptions(map[string]interface{}{ + "evt": &types.Event{}, + "match": &types.MatchedRule{}, + "req": &http.Request{}})...) if err != nil { return fmt.Errorf("compilation of '%s' context value failed: %w", value, err) } @@ -85,6 +92,32 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error { return nil } +// Truncate the context map to fit in the context value length +func TruncateContextMap(contextMap map[string][]string, contextValueLen int) ([]*models.MetaItems0, []error) { + metas := make([]*models.MetaItems0, 0) + errors := make([]error, 0) + + for key, values := range contextMap { + if len(values) == 0 { + continue + } + + valueStr, err := TruncateContext(values, alertContext.ContextValueLen) + if err != nil { + errors = append(errors, fmt.Errorf("error truncating content for %s: %w", key, err)) + continue + } + + meta := models.MetaItems0{ + Key: key, + Value: valueStr, + } + metas = append(metas, &meta) + } + return metas, errors +} + +// Truncate an individual []string to fit in the context value length func TruncateContext(values []string, contextValueLen int) (string, error) { valueByte, err := json.Marshal(values) if err != nil { @@ -116,11 +149,24 @@ func TruncateContext(values []string, contextValueLen int) (string, error) { return ret, nil } -func EvalAlertContextRules(evt *types.Event, match *types.MatchedRule, tmpContext map[string][]string) []error { +func EvalAlertContextRules(evt *types.Event, match *types.MatchedRule, request *http.Request, tmpContext map[string][]string) []error { var errors []error + //if we're evaluating context for appsec event, match and request will be present. + //otherwise, only evt will be. + if evt == nil { + evt = types.EmptyEvent() + } + if match == nil { + match = types.EmptyMatchedRule() + } + if request == nil { + request = &http.Request{} + } + for key, values := range alertContext.ContextToSendCompiled { + if _, ok := tmpContext[key]; !ok { tmpContext[key] = make([]string, 0) } @@ -128,24 +174,40 @@ func EvalAlertContextRules(evt *types.Event, match *types.MatchedRule, tmpContex for _, value := range values { var val string - output, err := expr.Run(value, map[string]interface{}{"match": match, "expr": evt}) + output, err := expr.Run(value, map[string]interface{}{"match": match, "evt": evt, "req": request}) if err != nil { errors = append(errors, fmt.Errorf("failed to get value for %s: %w", key, err)) continue } - //Need to support back []int and []string switch out := output.(type) { case string: val = out + if val != "" && !slices.Contains(tmpContext[key], val) { + tmpContext[key] = append(tmpContext[key], val) + } + case []string: + for _, v := range out { + if v != "" && !slices.Contains(tmpContext[key], v) { + tmpContext[key] = append(tmpContext[key], v) + } + } case int: val = strconv.Itoa(out) + if val != "" && !slices.Contains(tmpContext[key], val) { + tmpContext[key] = append(tmpContext[key], val) + } + case []int: + for _, v := range out { + val = strconv.Itoa(v) + if val != "" && !slices.Contains(tmpContext[key], val) { + tmpContext[key] = append(tmpContext[key], val) + } + } default: - errors = append(errors, fmt.Errorf("unexpected return type for %s: %T", key, output)) - continue - } - - if val != "" && !slices.Contains(tmpContext[key], val) { - tmpContext[key] = append(tmpContext[key], val) + val := fmt.Sprintf("%v", output) + if val != "" && !slices.Contains(tmpContext[key], val) { + tmpContext[key] = append(tmpContext[key], val) + } } } } @@ -153,33 +215,18 @@ func EvalAlertContextRules(evt *types.Event, match *types.MatchedRule, tmpContex } // Iterate over the individual appsec matched rules to create the needed alert context. -func AppsecEventToContext(event types.AppsecEvent) (models.Meta, []error) { +func AppsecEventToContext(event types.AppsecEvent, request *http.Request) (models.Meta, []error) { var errors []error - metas := make([]*models.MetaItems0, 0) tmpContext := make(map[string][]string) for _, matched_rule := range event.MatchedRules { - tmpErrors := EvalAlertContextRules(&types.Event{}, &matched_rule, tmpContext) + tmpErrors := EvalAlertContextRules(nil, &matched_rule, request, tmpContext) errors = append(errors, tmpErrors...) } - for key, values := range tmpContext { - if len(values) == 0 { - continue - } - - valueStr, err := TruncateContext(values, alertContext.ContextValueLen) - if err != nil { - log.Warning(err.Error()) - } - - meta := models.MetaItems0{ - Key: key, - Value: valueStr, - } - metas = append(metas, &meta) - } + metas, truncErrors := TruncateContextMap(tmpContext, alertContext.ContextValueLen) + errors = append(errors, truncErrors...) ret := models.Meta(metas) @@ -190,45 +237,17 @@ func AppsecEventToContext(event types.AppsecEvent) (models.Meta, []error) { func EventToContext(events []types.Event) (models.Meta, []error) { var errors []error - metas := make([]*models.MetaItems0, 0) tmpContext := make(map[string][]string) for _, evt := range events { - tmpErrors := EvalAlertContextRules(&evt, &types.MatchedRule{}, tmpContext) + tmpErrors := EvalAlertContextRules(&evt, nil, nil, tmpContext) errors = append(errors, tmpErrors...) } - for key, values := range tmpContext { - if len(values) == 0 { - continue - } - - valueStr, err := TruncateContext(values, alertContext.ContextValueLen) - if err != nil { - log.Warning(err.Error()) - } - - meta := models.MetaItems0{ - Key: key, - Value: valueStr, - } - metas = append(metas, &meta) - } + metas, truncErrors := TruncateContextMap(tmpContext, alertContext.ContextValueLen) + errors = append(errors, truncErrors...) ret := models.Meta(metas) return ret, errors } - -func appendMeta(meta models.Meta, key string, value string) models.Meta { - if value == "" { - return meta - } - - meta = append(meta, &models.MetaItems0{ - Key: key, - Value: value, - }) - - return meta -} From 740fa98e77e533ddf5128f9b47cb5afbe5a5c889 Mon Sep 17 00:00:00 2001 From: Thibault Koechlin Date: Wed, 16 Oct 2024 16:26:10 +0200 Subject: [PATCH 07/13] cleanup --- pkg/acquisition/modules/appsec/utils.go | 27 +++---------------------- 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index 517720efc8a..63c55931032 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -3,6 +3,7 @@ package appsecacquisition import ( "fmt" "net" + "net/http" "time" "github.com/oschwald/geoip2-golang" @@ -20,29 +21,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -// var appsecMetaKeys = []string{ -// "id", -// "name", -// "method", -// "uri", -// "matched_zones", -// "msg", -// } - -// func appendMeta(meta models.Meta, key string, value string) models.Meta { -// if value == "" { -// return meta -// } - -// meta = append(meta, &models.MetaItems0{ -// Key: key, -// Value: value, -// }) - -// return meta -// } - -func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { +func AppsecEventGeneration(inEvt types.Event, request *http.Request) (*types.Event, error) { // if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI if !inEvt.Appsec.HasInBandMatches { return nil, nil @@ -95,7 +74,7 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { alert.Capacity = ptr.Of(int32(1)) alert.Events = make([]*models.Event, len(evt.Appsec.GetRuleIDs())) - metas, errors := alertcontext.AppsecEventToContext(inEvt.Appsec) + metas, errors := alertcontext.AppsecEventToContext(inEvt.Appsec, request) if len(errors) > 0 { for _, err := range errors { log.Errorf("failed to generate appsec context: %s", err) From 6fea5550c3aefef5ab3d510b2f55e67de2d1f46c Mon Sep 17 00:00:00 2001 From: Thibault Koechlin Date: Wed, 16 Oct 2024 16:26:22 +0200 Subject: [PATCH 08/13] pass http request --- pkg/acquisition/modules/appsec/appsec_runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/acquisition/modules/appsec/appsec_runner.go b/pkg/acquisition/modules/appsec/appsec_runner.go index de34b62d704..90d23f63543 100644 --- a/pkg/acquisition/modules/appsec/appsec_runner.go +++ b/pkg/acquisition/modules/appsec/appsec_runner.go @@ -249,7 +249,7 @@ func (r *AppsecRunner) handleInBandInterrupt(request *appsec.ParsedRequest) { // Should the in band match trigger an overflow ? if r.AppsecRuntime.Response.SendAlert { - appsecOvlfw, err := AppsecEventGeneration(evt) + appsecOvlfw, err := AppsecEventGeneration(evt, request.HTTPRequest) if err != nil { r.logger.Errorf("unable to generate appsec event : %s", err) return @@ -293,7 +293,7 @@ func (r *AppsecRunner) handleOutBandInterrupt(request *appsec.ParsedRequest) { // Should the match trigger an overflow ? if r.AppsecRuntime.Response.SendAlert { - appsecOvlfw, err := AppsecEventGeneration(evt) + appsecOvlfw, err := AppsecEventGeneration(evt, request.HTTPRequest) if err != nil { r.logger.Errorf("unable to generate appsec event : %s", err) return From 4d9bff4f1edc15ec64c0f242d3f9ce6a35517a85 Mon Sep 17 00:00:00 2001 From: Thibault Koechlin Date: Thu, 17 Oct 2024 10:35:41 +0200 Subject: [PATCH 09/13] extract the geoip enrich --- pkg/acquisition/modules/appsec/utils.go | 63 +++++++++++++++---------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index 63c55931032..f29eca51596 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -21,49 +21,62 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -func AppsecEventGeneration(inEvt types.Event, request *http.Request) (*types.Event, error) { - // if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI - if !inEvt.Appsec.HasInBandMatches { - return nil, nil - } +func AppsecEventGenerationGeoIPEnrich(src *models.Source) error { - evt := types.Event{} - evt.Type = types.APPSEC - evt.Process = true - sourceIP := inEvt.Parsed["source_ip"] - source := models.Source{ - Value: &sourceIP, - IP: sourceIP, - Scope: ptr.Of(types.Ip), + if src == nil || src.Scope == nil || *src.Scope != types.Ip { + return fmt.Errorf("source is nil or not an IP") } //GeoIP enrich - asndata, err := exprhelpers.GeoIPASNEnrich(sourceIP) + asndata, err := exprhelpers.GeoIPASNEnrich(src.IP) if err != nil { - log.Errorf("Unable to enrich ip '%s' for ASN: %s", sourceIP, err) + return err } else if asndata != nil { record := asndata.(*geoip2.ASN) - source.AsName = record.AutonomousSystemOrganization - source.AsNumber = fmt.Sprintf("%d", record.AutonomousSystemNumber) + src.AsName = record.AutonomousSystemOrganization + src.AsNumber = fmt.Sprintf("%d", record.AutonomousSystemNumber) } - cityData, err := exprhelpers.GeoIPEnrich(sourceIP) + cityData, err := exprhelpers.GeoIPEnrich(src.IP) if err != nil { - log.Errorf("Unable to enrich ip '%s' for geo data: %s", sourceIP, err) + return err } else if cityData != nil { record := cityData.(*geoip2.City) - source.Cn = record.Country.IsoCode - source.Latitude = float32(record.Location.Latitude) - source.Longitude = float32(record.Location.Longitude) + src.Cn = record.Country.IsoCode + src.Latitude = float32(record.Location.Latitude) + src.Longitude = float32(record.Location.Longitude) } - rangeData, err := exprhelpers.GeoIPRangeEnrich(sourceIP) + rangeData, err := exprhelpers.GeoIPRangeEnrich(src.IP) if err != nil { - log.Errorf("Unable to enrich ip '%s' for range: %s", sourceIP, err) + return err } else if rangeData != nil { record := rangeData.(*net.IPNet) - source.Range = record.String() + src.Range = record.String() + } + return nil +} + +func AppsecEventGeneration(inEvt types.Event, request *http.Request) (*types.Event, error) { + // if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI + if !inEvt.Appsec.HasInBandMatches { + return nil, nil + } + + evt := types.Event{} + evt.Type = types.APPSEC + evt.Process = true + sourceIP := inEvt.Parsed["source_ip"] + source := models.Source{ + Value: &sourceIP, + IP: sourceIP, + Scope: ptr.Of(types.Ip), + } + + // Enrich source with GeoIP data + if err := AppsecEventGenerationGeoIPEnrich(&source); err != nil { + log.Errorf("unable to enrich source with GeoIP data : %s", err) } // Build overflow From b6c1124921a0f90ff3fb92cd752914b8a905dc32 Mon Sep 17 00:00:00 2001 From: Thibault Koechlin Date: Thu, 17 Oct 2024 10:50:34 +0200 Subject: [PATCH 10/13] lint --- pkg/acquisition/modules/appsec/utils.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index f29eca51596..b4b66897516 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -1,6 +1,7 @@ package appsecacquisition import ( + "errors" "fmt" "net" "net/http" @@ -24,7 +25,7 @@ import ( func AppsecEventGenerationGeoIPEnrich(src *models.Source) error { if src == nil || src.Scope == nil || *src.Scope != types.Ip { - return fmt.Errorf("source is nil or not an IP") + return errors.New("source is nil or not an IP") } //GeoIP enrich From ac577d798f9008c0304a8a5213df94e12832eebd Mon Sep 17 00:00:00 2001 From: Thibault Koechlin Date: Mon, 21 Oct 2024 14:08:56 +0200 Subject: [PATCH 11/13] add tests --- pkg/alertcontext/alertcontext_test.go | 164 ++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/pkg/alertcontext/alertcontext_test.go b/pkg/alertcontext/alertcontext_test.go index c111d1bbcfb..15506c99b2c 100644 --- a/pkg/alertcontext/alertcontext_test.go +++ b/pkg/alertcontext/alertcontext_test.go @@ -2,13 +2,16 @@ package alertcontext import ( "fmt" + "net/http" "testing" + "github.com/davecgh/go-spew/spew" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/go-cs-lib/ptr" ) func TestNewAlertContext(t *testing.T) { @@ -200,3 +203,164 @@ func TestEventToContext(t *testing.T) { assert.ElementsMatch(t, test.expectedResult, metas) } } + +func TestValidateContextExpr(t *testing.T) { + tests := []struct { + name string + key string + exprs []string + expectedErr *string + }{ + { + name: "basic config", + key: "source_ip", + exprs: []string{ + "evt.Parsed.source_ip", + }, + expectedErr: nil, + }, + { + name: "basic config with non existent field", + key: "source_ip", + exprs: []string{ + "evt.invalid.source_ip", + }, + expectedErr: ptr.Of("compilation of 'evt.invalid.source_ip' failed: type types.Event has no field invalid"), + }, + } + for _, test := range tests { + fmt.Printf("Running test '%s'\n", test.name) + err := ValidateContextExpr(test.key, test.exprs) + if test.expectedErr == nil { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, *test.expectedErr) + } + } +} + +func TestAppsecEventToContext(t *testing.T) { + + tests := []struct { + name string + contextToSend map[string][]string + match types.AppsecEvent + req *http.Request + expectedResult models.Meta + expectedErrLen int + }{ + { + name: "basic test on match", + contextToSend: map[string][]string{ + "id": {"match.id"}, + }, + match: types.AppsecEvent{ + MatchedRules: types.MatchedRules{ + { + "id": "test", + }, + }, + }, + req: &http.Request{}, + expectedResult: []*models.MetaItems0{ + { + Key: "id", + Value: "[\"test\"]", + }, + }, + expectedErrLen: 0, + }, + { + name: "basic test on req", + contextToSend: map[string][]string{ + "ua": {"req.UserAgent()"}, + }, + match: types.AppsecEvent{ + MatchedRules: types.MatchedRules{ + { + "id": "test", + }, + }, + }, + req: &http.Request{ + Header: map[string][]string{ + "User-Agent": {"test"}, + }, + }, + expectedResult: []*models.MetaItems0{ + { + Key: "ua", + Value: "[\"test\"]", + }, + }, + expectedErrLen: 0, + }, + { + name: "test on req -> []string", + contextToSend: map[string][]string{ + "foobarxx": {"req.Header.Values('Foobar')"}, + }, + match: types.AppsecEvent{ + MatchedRules: types.MatchedRules{ + { + "id": "test", + }, + }, + }, + req: &http.Request{ + Header: map[string][]string{ + "User-Agent": {"test"}, + "Foobar": {"test1", "test2"}, + }, + }, + expectedResult: []*models.MetaItems0{ + { + Key: "foobarxx", + Value: "[\"test1\",\"test2\"]", + }, + }, + expectedErrLen: 0, + }, + { + name: "test on type int", + contextToSend: map[string][]string{ + "foobarxx": {"len(req.Header.Values('Foobar'))"}, + }, + match: types.AppsecEvent{ + MatchedRules: types.MatchedRules{ + { + "id": "test", + }, + }, + }, + req: &http.Request{ + Header: map[string][]string{ + "User-Agent": {"test"}, + "Foobar": {"test1", "test2"}, + }, + }, + expectedResult: []*models.MetaItems0{ + { + Key: "foobarxx", + Value: "[\"2\"]", + }, + }, + expectedErrLen: 0, + }, + } + + for _, test := range tests { + //reset cache + alertContext = Context{} + //compile + if err := NewAlertContext(test.contextToSend, 100); err != nil { + t.Fatalf("failed to compile %s: %s", test.name, err) + } + //run + + metas, errors := AppsecEventToContext(test.match, test.req) + fmt.Printf(spew.Sdump(metas)) + assert.Len(t, errors, test.expectedErrLen) + assert.ElementsMatch(t, test.expectedResult, metas) + } +} From ed00b70e716fbe8ca7f37574bfef142bf24ae3e9 Mon Sep 17 00:00:00 2001 From: Thibault Koechlin Date: Mon, 21 Oct 2024 14:52:49 +0200 Subject: [PATCH 12/13] oops --- pkg/alertcontext/alertcontext_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/alertcontext/alertcontext_test.go b/pkg/alertcontext/alertcontext_test.go index 15506c99b2c..dc752ba8b09 100644 --- a/pkg/alertcontext/alertcontext_test.go +++ b/pkg/alertcontext/alertcontext_test.go @@ -5,7 +5,6 @@ import ( "net/http" "testing" - "github.com/davecgh/go-spew/spew" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -359,7 +358,6 @@ func TestAppsecEventToContext(t *testing.T) { //run metas, errors := AppsecEventToContext(test.match, test.req) - fmt.Printf(spew.Sdump(metas)) assert.Len(t, errors, test.expectedErrLen) assert.ElementsMatch(t, test.expectedResult, metas) } From 37c044754369f2737d8c945f312c2ff7eadbf6b6 Mon Sep 17 00:00:00 2001 From: Thibault Koechlin Date: Wed, 30 Oct 2024 18:22:44 +0100 Subject: [PATCH 13/13] fix comments --- pkg/alertcontext/alertcontext.go | 4 ++-- pkg/types/appsec_event.go | 2 +- pkg/types/event.go | 10 ++++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/alertcontext/alertcontext.go b/pkg/alertcontext/alertcontext.go index 32cf67d90b0..0c60dea4292 100644 --- a/pkg/alertcontext/alertcontext.go +++ b/pkg/alertcontext/alertcontext.go @@ -156,10 +156,10 @@ func EvalAlertContextRules(evt *types.Event, match *types.MatchedRule, request * //if we're evaluating context for appsec event, match and request will be present. //otherwise, only evt will be. if evt == nil { - evt = types.EmptyEvent() + evt = types.NewEvent() } if match == nil { - match = types.EmptyMatchedRule() + match = types.NewMatchedRule() } if request == nil { request = &http.Request{} diff --git a/pkg/types/appsec_event.go b/pkg/types/appsec_event.go index cef8d6a6ba5..11d70ad368d 100644 --- a/pkg/types/appsec_event.go +++ b/pkg/types/appsec_event.go @@ -47,7 +47,7 @@ const ( Kind Field = "kind" ) -func EmptyMatchedRule() *MatchedRule { +func NewMatchedRule() *MatchedRule { return &MatchedRule{} } diff --git a/pkg/types/event.go b/pkg/types/event.go index 1746a3648a9..6d275aedf95 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -47,11 +47,13 @@ type Event struct { Meta map[string]string `yaml:"Meta,omitempty" json:"Meta,omitempty"` } -func EmptyEvent() *Event { +func NewEvent() *Event { return &Event{Type: LOG, - Parsed: make(map[string]string), - Enriched: make(map[string]string), - Meta: make(map[string]string)} + Parsed: make(map[string]string), + Enriched: make(map[string]string), + Meta: make(map[string]string), + Unmarshaled: make(map[string]interface{}), + } } func (e *Event) SetMeta(key string, value string) bool {