From ec05de3c180a9c302e47713fa2b2254ec1a892e5 Mon Sep 17 00:00:00 2001 From: mahanth <22050509+gnmahanth@users.noreply.github.com> Date: Thu, 27 Jun 2024 16:45:15 +0530 Subject: [PATCH] add human readable error messages in integrations (#2225) --- .../elasticsearch/elasticsearch.go | 16 ++++---- .../pkg/integration/errors/errors.go | 40 +++++++++++++++++++ .../google-chronicle/googlechronicle.go | 17 +++----- .../http-endpoint/http-endpoint.go | 16 ++++---- deepfence_server/pkg/integration/jira/jira.go | 4 +- .../pkg/integration/pagerduty/pagerduty.go | 13 +++--- deepfence_server/pkg/integration/s3/s3.go | 18 ++++----- .../pkg/integration/slack/slack.go | 9 +++-- .../pkg/integration/sumologic/sumologic.go | 18 +++------ .../pkg/integration/teams/teams.go | 2 +- 10 files changed, 89 insertions(+), 64 deletions(-) create mode 100644 deepfence_server/pkg/integration/errors/errors.go diff --git a/deepfence_server/pkg/integration/elasticsearch/elasticsearch.go b/deepfence_server/pkg/integration/elasticsearch/elasticsearch.go index 823a0d5384..f93920bcef 100644 --- a/deepfence_server/pkg/integration/elasticsearch/elasticsearch.go +++ b/deepfence_server/pkg/integration/elasticsearch/elasticsearch.go @@ -10,6 +10,7 @@ import ( "github.com/rs/zerolog/log" + intgerr "github.com/deepfence/ThreatMapper/deepfence_server/pkg/integration/errors" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" ) @@ -45,6 +46,7 @@ func (e ElasticSearch) SendNotification(ctx context.Context, message []map[strin endpointURL := strings.TrimRight(e.Config.EndpointURL, "/") req, err := http.NewRequest(http.MethodPost, endpointURL+"/_bulk", bytes.NewBuffer([]byte(payloadMsg))) if err != nil { + log.Error().Err(err).Msg("error on create http request") span.EndWithErr(err) return err } @@ -58,17 +60,13 @@ func (e ElasticSearch) SendNotification(ctx context.Context, message []map[strin // Make the HTTP request. resp, err := utils.GetHTTPClient().Do(req) if err != nil { + log.Error().Err(err).Msg("error on http request") span.EndWithErr(err) - return err + return intgerr.CheckHTTPError(err) } defer resp.Body.Close() - // Check the response status code. - if resp.StatusCode != http.StatusOK { - return err - } - - return nil + return intgerr.CheckResponseCode(resp, http.StatusOK) } func (e ElasticSearch) IsValidCredential(ctx context.Context) (bool, error) { @@ -96,8 +94,8 @@ func (e ElasticSearch) IsValidCredential(ctx context.Context) (bool, error) { // Check the status code if resp.StatusCode != http.StatusOK { - log.Error().Msgf("Elasticsearch index validation failed. Status code: %d", resp.StatusCode) - return false, fmt.Errorf("Elasticsearch index validation failed. Status code: %d", resp.StatusCode) + log.Error().Msgf("elasticsearch index validation failed. Status code: %d", resp.StatusCode) + return false, fmt.Errorf("elasticsearch index validation failed. Status code: %d", resp.StatusCode) } return true, nil diff --git a/deepfence_server/pkg/integration/errors/errors.go b/deepfence_server/pkg/integration/errors/errors.go new file mode 100644 index 0000000000..2e57fffd53 --- /dev/null +++ b/deepfence_server/pkg/integration/errors/errors.go @@ -0,0 +1,40 @@ +package errors + +import ( + "errors" + "net" + "net/http" + "syscall" +) + +type HTTPError struct { + StatusCode int + Message string +} + +func NewHTTPError(code int, message string) *HTTPError { + return &HTTPError{StatusCode: code, Message: message} +} + +func (he *HTTPError) Error() string { + return he.Message +} + +func CheckResponseCode(resp *http.Response, code int) error { + if resp.StatusCode != code { + return NewHTTPError(resp.StatusCode, resp.Status) + } + return nil +} + +func CheckHTTPError(err error) error { + + if errors.Is(err, syscall.ECONNREFUSED) { + return NewHTTPError(0, "Connection Refused") + } + if netError, ok := err.(net.Error); ok && netError.Timeout() { + return NewHTTPError(0, "Connection Timeout") + } + + return nil +} diff --git a/deepfence_server/pkg/integration/google-chronicle/googlechronicle.go b/deepfence_server/pkg/integration/google-chronicle/googlechronicle.go index 0dd356f7a0..dba2421d5e 100644 --- a/deepfence_server/pkg/integration/google-chronicle/googlechronicle.go +++ b/deepfence_server/pkg/integration/google-chronicle/googlechronicle.go @@ -6,6 +6,7 @@ import ( "encoding/json" "net/http" + intgerr "github.com/deepfence/ThreatMapper/deepfence_server/pkg/integration/errors" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" @@ -34,10 +35,9 @@ func (g GoogleChronicle) SendNotification(ctx context.Context, message []map[str return err } - // send message to this elasticsearch using http - // Set up the HTTP request. req, err = http.NewRequest("POST", g.Config.URL, bytes.NewBuffer(payload)) if err != nil { + log.Error().Err(err).Msg("error on create http request") span.EndWithErr(err) return err } @@ -46,26 +46,19 @@ func (g GoogleChronicle) SendNotification(ctx context.Context, message []map[str req.Header.Set("Authorization", g.Config.AuthKey) } - if err != nil { - return err - } req.Header.Set("Content-Type", "application/json") // Make the HTTP request. client := utils.GetHTTPClient() resp, err := client.Do(req) if err != nil { + log.Error().Err(err).Msg("error on http request") span.EndWithErr(err) - return err + return intgerr.CheckHTTPError(err) } defer resp.Body.Close() - // Check the response status code. - if resp.StatusCode != http.StatusOK { - return err - } - - return nil + return intgerr.CheckResponseCode(resp, http.StatusOK) } // todo diff --git a/deepfence_server/pkg/integration/http-endpoint/http-endpoint.go b/deepfence_server/pkg/integration/http-endpoint/http-endpoint.go index ef3f2546d0..672a62d28f 100644 --- a/deepfence_server/pkg/integration/http-endpoint/http-endpoint.go +++ b/deepfence_server/pkg/integration/http-endpoint/http-endpoint.go @@ -6,6 +6,7 @@ import ( "encoding/json" "net/http" + intgerr "github.com/deepfence/ThreatMapper/deepfence_server/pkg/integration/errors" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" @@ -39,8 +40,9 @@ func (h HTTPEndpoint) SendNotification(ctx context.Context, message []map[string // send message to this http url using http // Set up the HTTP request. - req, err = http.NewRequest("POST", h.Config.URL, bytes.NewBuffer(payload)) + req, err = http.NewRequest(http.MethodPost, h.Config.URL, bytes.NewBuffer(payload)) if err != nil { + log.Error().Err(err).Msg("error on create http request") span.EndWithErr(err) return err } @@ -55,17 +57,13 @@ func (h HTTPEndpoint) SendNotification(ctx context.Context, message []map[string client := utils.GetHTTPClient() resp, err := client.Do(req) if err != nil { + log.Error().Err(err).Msg("error on http request") span.EndWithErr(err) - return err + return intgerr.CheckHTTPError(err) } defer resp.Body.Close() - // Check the response status code. - if resp.StatusCode != http.StatusOK { - return err - } - - return nil + return intgerr.CheckResponseCode(resp, http.StatusOK) } func (h HTTPEndpoint) IsValidCredential(ctx context.Context) (bool, error) { @@ -83,6 +81,7 @@ func (h HTTPEndpoint) IsValidCredential(ctx context.Context) (bool, error) { // Set up the HTTP request. req, err := http.NewRequest("POST", h.Config.URL, bytes.NewBuffer(payloadBytes)) if err != nil { + log.Error().Err(err).Msg("error on create http request") return false, nil } @@ -96,6 +95,7 @@ func (h HTTPEndpoint) IsValidCredential(ctx context.Context) (bool, error) { client := utils.GetHTTPClient() resp, err := client.Do(req) if err != nil { + log.Error().Err(err).Msg("error on http request") return false, err } defer resp.Body.Close() diff --git a/deepfence_server/pkg/integration/jira/jira.go b/deepfence_server/pkg/integration/jira/jira.go index 0417db1994..ecf545bf82 100644 --- a/deepfence_server/pkg/integration/jira/jira.go +++ b/deepfence_server/pkg/integration/jira/jira.go @@ -45,7 +45,7 @@ func (j Jira) SendNotification(ctx context.Context, message []map[string]interfa client, err := jira.NewClient(auth.Client(), strings.TrimSpace(j.Config.JiraSiteURL)) if err != nil { - log.Error().Msgf(err.Error()) + log.Error().Err(err).Msgf("error create jira client") span.EndWithErr(err) return err } @@ -117,7 +117,7 @@ func (j Jira) SendNotification(ctx context.Context, message []map[string]interfa log.Error().Msgf(err.Error()) body, err := io.ReadAll(resp.Body) if err != nil { - log.Error().Msgf(err.Error()) + log.Error().Err(err).Msgf("error adding jira issue attachment") } log.Error().Msgf("jira attachment error reponse: %s", string(body)) span.EndWithErr(err) diff --git a/deepfence_server/pkg/integration/pagerduty/pagerduty.go b/deepfence_server/pkg/integration/pagerduty/pagerduty.go index 38d8f0d7e4..7b6a65e8c3 100644 --- a/deepfence_server/pkg/integration/pagerduty/pagerduty.go +++ b/deepfence_server/pkg/integration/pagerduty/pagerduty.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/PagerDuty/go-pagerduty" + intgerr "github.com/deepfence/ThreatMapper/deepfence_server/pkg/integration/errors" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" @@ -97,15 +98,11 @@ func createPagerDutyEvent(pagerDutyAPIToken string, event pagerduty.V2Event) err client := utils.GetHTTPClient() resp, err := client.Do(req) if err != nil { - return err + return intgerr.CheckHTTPError(err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusAccepted { - return fmt.Errorf("unexpected response status: %s", resp.Status) - } - - return nil + return intgerr.CheckResponseCode(resp, http.StatusOK) } func (p PagerDuty) FormatMessage(message []map[string]interface{}) string { @@ -133,7 +130,7 @@ func IsValidCreds(p PagerDuty) (bool, error) { var req *http.Request var err error - req, err = http.NewRequest("POST", url, nil) + req, err = http.NewRequest(http.MethodPost, url, nil) if err != nil { return false, err } @@ -150,7 +147,7 @@ func IsValidCreds(p PagerDuty) (bool, error) { } defer resp.Body.Close() - if resp.StatusCode == 200 { + if resp.StatusCode == http.StatusOK { return true, nil } // todo: check response body for error message like invalid api key or something diff --git a/deepfence_server/pkg/integration/s3/s3.go b/deepfence_server/pkg/integration/s3/s3.go index 3d2a26840c..7d4ec69f07 100644 --- a/deepfence_server/pkg/integration/s3/s3.go +++ b/deepfence_server/pkg/integration/s3/s3.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "fmt" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" "github.com/deepfence/ThreatMapper/deepfence_utils/log" @@ -51,7 +50,8 @@ func (s S3) SendNotification(ctx context.Context, message []map[string]interface sess, err = session.NewSession(&awsConfig) if err != nil { - return fmt.Errorf("error creating session: %v", err) + log.Error().Err(err).Msg("Failed to create AWS session") + return err } } else { @@ -60,7 +60,7 @@ func (s S3) SendNotification(ctx context.Context, message []map[string]interface Credentials: credentials.NewStaticCredentials(s.Config.AWSAccessKey, s.Config.AWSSecretKey, ""), }) if err != nil { - fmt.Println("Failed to create AWS session", err) + log.Error().Err(err).Msg("Failed to create AWS session") return err } } @@ -75,7 +75,7 @@ func (s S3) SendNotification(ctx context.Context, message []map[string]interface s.Buffer.Reset() gzWriter, err := gzip.NewWriterLevel(s.Buffer, gzip.DefaultCompression) if err != nil { - fmt.Println("Failed to get the gzip writer", err) + log.Error().Err(err).Msg("Failed to get the gzip writer") span.EndWithErr(err) return err } @@ -92,12 +92,12 @@ func (s S3) SendNotification(ctx context.Context, message []map[string]interface Key: aws.String(s.Config.S3FolderName + "/" + utils.GetDatetimeNow() + ".json"), }) if err != nil { - fmt.Println("Failed to upload JSON data to S3", err) + log.Error().Err(err).Msg("Failed to upload JSON data to S3") span.EndWithErr(err) return err } - fmt.Println("JSON data uploaded successfully") + log.Info().Msg("JSON data uploaded successfully") return nil } @@ -118,7 +118,7 @@ func (s S3) IsValidCredential(ctx context.Context) (bool, error) { sess, err = session.NewSession(&awsConfig) if err != nil { - fmt.Printf("error creating session: %v", err) + log.Error().Err(err).Msg("error creating aws session") return false, err } } else { @@ -127,7 +127,7 @@ func (s S3) IsValidCredential(ctx context.Context) (bool, error) { Credentials: credentials.NewStaticCredentials(s.Config.AWSAccessKey, s.Config.AWSSecretKey, ""), }) if err != nil { - fmt.Println("Failed to create AWS session", err) + log.Error().Err(err).Msg("error creating aws session") return false, err } } @@ -136,7 +136,7 @@ func (s S3) IsValidCredential(ctx context.Context) (bool, error) { _, err = svc.ListBuckets(nil) if err != nil { - fmt.Println("Failed to list buckets", err) + log.Error().Err(err).Msg("Failed to list buckets") return false, err } diff --git a/deepfence_server/pkg/integration/slack/slack.go b/deepfence_server/pkg/integration/slack/slack.go index 223bd5fa49..879ccc49ea 100644 --- a/deepfence_server/pkg/integration/slack/slack.go +++ b/deepfence_server/pkg/integration/slack/slack.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" + intgerr "github.com/deepfence/ThreatMapper/deepfence_server/pkg/integration/errors" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" @@ -149,8 +150,9 @@ func (s Slack) SendNotification(ctx context.Context, message []map[string]interf // send message to this webhookURL using http // Set up the HTTP request. - req, err := http.NewRequest("POST", s.Config.WebhookURL, bytes.NewBuffer(payloadBytes)) + req, err := http.NewRequest(http.MethodPost, s.Config.WebhookURL, bytes.NewBuffer(payloadBytes)) if err != nil { + log.Error().Err(err).Msg("error create http request") span.EndWithErr(err) return err } @@ -160,8 +162,9 @@ func (s Slack) SendNotification(ctx context.Context, message []map[string]interf client := utils.GetHTTPClient() resp, err := client.Do(req) if err != nil { + log.Error().Err(err).Msg("error on http request") span.EndWithErr(err) - return err + return intgerr.CheckHTTPError(err) } // Check the response status code. @@ -210,7 +213,7 @@ func (s Slack) IsValidCredential(ctx context.Context) (bool, error) { // send message to this webhookURL using http // Set up the HTTP request. - req, err := http.NewRequest("POST", s.Config.WebhookURL, bytes.NewBuffer(payloadBytes)) + req, err := http.NewRequest(http.MethodPost, s.Config.WebhookURL, bytes.NewBuffer(payloadBytes)) if err != nil { log.Error().Msg(err.Error()) return false, err diff --git a/deepfence_server/pkg/integration/sumologic/sumologic.go b/deepfence_server/pkg/integration/sumologic/sumologic.go index ca57a763e9..918aef605a 100644 --- a/deepfence_server/pkg/integration/sumologic/sumologic.go +++ b/deepfence_server/pkg/integration/sumologic/sumologic.go @@ -7,6 +7,7 @@ import ( "errors" "net/http" + intgerr "github.com/deepfence/ThreatMapper/deepfence_server/pkg/integration/errors" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" @@ -49,14 +50,14 @@ func (s SumoLogic) SendNotification(ctx context.Context, data []map[string]inter msg, err := s.FormatMessage(data) if err != nil { log.Error().Msgf("%v", err) - return nil + return err } // Create a new request to send the JSON data to Sumo Logic - req, err := http.NewRequest("POST", s.Config.HTTPEndpoint, bytes.NewBuffer(msg.Bytes())) + req, err := http.NewRequest(http.MethodPost, s.Config.HTTPEndpoint, bytes.NewBuffer(msg.Bytes())) if err != nil { log.Error().Msgf("Failed to create HTTP request: %v", err) - return nil + return err } req.Header.Set("Content-Type", "application/json") @@ -64,18 +65,11 @@ func (s SumoLogic) SendNotification(ctx context.Context, data []map[string]inter resp, err := client.Do(req) if err != nil { log.Error().Msgf("Failed to send data to Sumo Logic: %v", err) - return nil + return intgerr.CheckHTTPError(err) } defer resp.Body.Close() - // Check the response status code - if resp.StatusCode != http.StatusOK { - log.Error().Msgf("Failed to send data to Sumo Logic: %v", resp.Status) - return nil - } - - log.Debug().Msg("Data sent to Sumo Logic successfully") - return nil + return intgerr.CheckResponseCode(resp, http.StatusOK) } // todo diff --git a/deepfence_server/pkg/integration/teams/teams.go b/deepfence_server/pkg/integration/teams/teams.go index aed6b4f08f..09a8643356 100644 --- a/deepfence_server/pkg/integration/teams/teams.go +++ b/deepfence_server/pkg/integration/teams/teams.go @@ -10,9 +10,9 @@ import ( "strings" "sync" + "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" - "github.com/rs/zerolog/log" ) const BatchSize = 5