add human readable error messages in integrations (#2225)

deepfence_server/pkg/integration/elasticsearch/elasticsearch.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/rs/zerolog/log"
 
+	intgerr "github.com/deepfence/ThreatMapper/deepfence_server/pkg/integration/errors"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/telemetry"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/utils"
 )
@@ -45,6 +46,7 @@ func (e ElasticSearch) SendNotification(ctx context.Context, message []map[strin
 	endpointURL := strings.TrimRight(e.Config.EndpointURL, "/")
 	req, err := http.NewRequest(http.MethodPost, endpointURL+"/_bulk", bytes.NewBuffer([]byte(payloadMsg)))
 	if err != nil {
+		log.Error().Err(err).Msg("error on create http request")
 		span.EndWithErr(err)
 		return err
 	}
@@ -58,17 +60,13 @@ func (e ElasticSearch) SendNotification(ctx context.Context, message []map[strin
 	// Make the HTTP request.
 	resp, err := utils.GetHTTPClient().Do(req)
 	if err != nil {
+		log.Error().Err(err).Msg("error on http request")
 		span.EndWithErr(err)
-		return err
+		return intgerr.CheckHTTPError(err)
 	}
 	defer resp.Body.Close()
 
-	// Check the response status code.
-	if resp.StatusCode != http.StatusOK {
-		return err
-	}
-
-	return nil
+	return intgerr.CheckResponseCode(resp, http.StatusOK)
 }
 
 func (e ElasticSearch) IsValidCredential(ctx context.Context) (bool, error) {
@@ -96,8 +94,8 @@ func (e ElasticSearch) IsValidCredential(ctx context.Context) (bool, error) {
 
 	// Check the status code
 	if resp.StatusCode != http.StatusOK {
-		log.Error().Msgf("Elasticsearch index validation failed. Status code: %d", resp.StatusCode)
-		return false, fmt.Errorf("Elasticsearch index validation failed. Status code: %d", resp.StatusCode)
+		log.Error().Msgf("elasticsearch index validation failed. Status code: %d", resp.StatusCode)
+		return false, fmt.Errorf("elasticsearch index validation failed. Status code: %d", resp.StatusCode)
 	}
 
 	return true, nil

deepfence_server/pkg/integration/errors/errors.go

@@ -0,0 +1,40 @@
+package errors
+
+import (
+	"errors"
+	"net"
+	"net/http"
+	"syscall"
+)
+
+type HTTPError struct {
+	StatusCode int
+	Message    string
+}
+
+func NewHTTPError(code int, message string) *HTTPError {
+	return &HTTPError{StatusCode: code, Message: message}
+}
+
+func (he *HTTPError) Error() string {
+	return he.Message
+}
+
+func CheckResponseCode(resp *http.Response, code int) error {
+	if resp.StatusCode != code {
+		return NewHTTPError(resp.StatusCode, resp.Status)
+	}
+	return nil
+}
+
+func CheckHTTPError(err error) error {
+
+	if errors.Is(err, syscall.ECONNREFUSED) {
+		return NewHTTPError(0, "Connection Refused")
+	}
+	if netError, ok := err.(net.Error); ok && netError.Timeout() {
+		return NewHTTPError(0, "Connection Timeout")
+	}
+
+	return nil
+}

deepfence_server/pkg/integration/google-chronicle/googlechronicle.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"net/http"
 
+	intgerr "github.com/deepfence/ThreatMapper/deepfence_server/pkg/integration/errors"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/telemetry"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/utils"
@@ -34,10 +35,9 @@ func (g GoogleChronicle) SendNotification(ctx context.Context, message []map[str
 		return err
 	}
 
-	// send message to this elasticsearch using http
-	// Set up the HTTP request.
 	req, err = http.NewRequest("POST", g.Config.URL, bytes.NewBuffer(payload))
 	if err != nil {
+		log.Error().Err(err).Msg("error on create http request")
 		span.EndWithErr(err)
 		return err
 	}
@@ -46,26 +46,19 @@ func (g GoogleChronicle) SendNotification(ctx context.Context, message []map[str
 		req.Header.Set("Authorization", g.Config.AuthKey)
 	}
 
-	if err != nil {
-		return err
-	}
 	req.Header.Set("Content-Type", "application/json")
 
 	// Make the HTTP request.
 	client := utils.GetHTTPClient()
 	resp, err := client.Do(req)
 	if err != nil {
+		log.Error().Err(err).Msg("error on http request")
 		span.EndWithErr(err)
-		return err
+		return intgerr.CheckHTTPError(err)
 	}
 	defer resp.Body.Close()
 
-	// Check the response status code.
-	if resp.StatusCode != http.StatusOK {
-		return err
-	}
-
-	return nil
+	return intgerr.CheckResponseCode(resp, http.StatusOK)
 }
 
 // todo

deepfence_server/pkg/integration/http-endpoint/http-endpoint.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"net/http"
 
+	intgerr "github.com/deepfence/ThreatMapper/deepfence_server/pkg/integration/errors"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/telemetry"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/utils"
@@ -39,8 +40,9 @@ func (h HTTPEndpoint) SendNotification(ctx context.Context, message []map[string
 
 	// send message to this http url using http
 	// Set up the HTTP request.
-	req, err = http.NewRequest("POST", h.Config.URL, bytes.NewBuffer(payload))
+	req, err = http.NewRequest(http.MethodPost, h.Config.URL, bytes.NewBuffer(payload))
 	if err != nil {
+		log.Error().Err(err).Msg("error on create http request")
 		span.EndWithErr(err)
 		return err
 	}
@@ -55,17 +57,13 @@ func (h HTTPEndpoint) SendNotification(ctx context.Context, message []map[string
 	client := utils.GetHTTPClient()
 	resp, err := client.Do(req)
 	if err != nil {
+		log.Error().Err(err).Msg("error on http request")
 		span.EndWithErr(err)
-		return err
+		return intgerr.CheckHTTPError(err)
 	}
 	defer resp.Body.Close()
 
-	// Check the response status code.
-	if resp.StatusCode != http.StatusOK {
-		return err
-	}
-
-	return nil
+	return intgerr.CheckResponseCode(resp, http.StatusOK)
 }
 
 func (h HTTPEndpoint) IsValidCredential(ctx context.Context) (bool, error) {
@@ -83,6 +81,7 @@ func (h HTTPEndpoint) IsValidCredential(ctx context.Context) (bool, error) {
 	// Set up the HTTP request.
 	req, err := http.NewRequest("POST", h.Config.URL, bytes.NewBuffer(payloadBytes))
 	if err != nil {
+		log.Error().Err(err).Msg("error on create http request")
 		return false, nil
 	}
 
@@ -96,6 +95,7 @@ func (h HTTPEndpoint) IsValidCredential(ctx context.Context) (bool, error) {
 	client := utils.GetHTTPClient()
 	resp, err := client.Do(req)
 	if err != nil {
+		log.Error().Err(err).Msg("error on http request")
 		return false, err
 	}
 	defer resp.Body.Close()

deepfence_server/pkg/integration/jira/jira.go

@@ -45,7 +45,7 @@ func (j Jira) SendNotification(ctx context.Context, message []map[string]interfa
 
 	client, err := jira.NewClient(auth.Client(), strings.TrimSpace(j.Config.JiraSiteURL))
 	if err != nil {
-		log.Error().Msgf(err.Error())
+		log.Error().Err(err).Msgf("error create jira client")
 		span.EndWithErr(err)
 		return err
 	}
@@ -117,7 +117,7 @@ func (j Jira) SendNotification(ctx context.Context, message []map[string]interfa
 		log.Error().Msgf(err.Error())
 		body, err := io.ReadAll(resp.Body)
 		if err != nil {
-			log.Error().Msgf(err.Error())
+			log.Error().Err(err).Msgf("error adding jira issue attachment")
 		}
 		log.Error().Msgf("jira attachment error reponse: %s", string(body))
 		span.EndWithErr(err)

deepfence_server/pkg/integration/pagerduty/pagerduty.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/PagerDuty/go-pagerduty"
+	intgerr "github.com/deepfence/ThreatMapper/deepfence_server/pkg/integration/errors"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/telemetry"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/utils"
@@ -97,15 +98,11 @@ func createPagerDutyEvent(pagerDutyAPIToken string, event pagerduty.V2Event) err
 	client := utils.GetHTTPClient()
 	resp, err := client.Do(req)
 	if err != nil {
-		return err
+		return intgerr.CheckHTTPError(err)
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode != http.StatusAccepted {
-		return fmt.Errorf("unexpected response status: %s", resp.Status)
-	}
-
-	return nil
+	return intgerr.CheckResponseCode(resp, http.StatusOK)
 }
 
 func (p PagerDuty) FormatMessage(message []map[string]interface{}) string {
@@ -133,7 +130,7 @@ func IsValidCreds(p PagerDuty) (bool, error) {
 	var req *http.Request
 	var err error
 
-	req, err = http.NewRequest("POST", url, nil)
+	req, err = http.NewRequest(http.MethodPost, url, nil)
 	if err != nil {
 		return false, err
 	}
@@ -150,7 +147,7 @@ func IsValidCreds(p PagerDuty) (bool, error) {
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode == 200 {
+	if resp.StatusCode == http.StatusOK {
 		return true, nil
 	}
 	// todo: check response body for error message like invalid api key or something

deepfence_server/pkg/integration/s3/s3.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"fmt"
 
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
@@ -51,7 +50,8 @@ func (s S3) SendNotification(ctx context.Context, message []map[string]interface
 
 		sess, err = session.NewSession(&awsConfig)
 		if err != nil {
-			return fmt.Errorf("error creating session: %v", err)
+			log.Error().Err(err).Msg("Failed to create AWS session")
+			return err
 		}
 
 	} else {
@@ -60,7 +60,7 @@ func (s S3) SendNotification(ctx context.Context, message []map[string]interface
 			Credentials: credentials.NewStaticCredentials(s.Config.AWSAccessKey, s.Config.AWSSecretKey, ""),
 		})
 		if err != nil {
-			fmt.Println("Failed to create AWS session", err)
+			log.Error().Err(err).Msg("Failed to create AWS session")
 			return err
 		}
 	}
@@ -75,7 +75,7 @@ func (s S3) SendNotification(ctx context.Context, message []map[string]interface
 	s.Buffer.Reset()
 	gzWriter, err := gzip.NewWriterLevel(s.Buffer, gzip.DefaultCompression)
 	if err != nil {
-		fmt.Println("Failed to get the gzip writer", err)
+		log.Error().Err(err).Msg("Failed to get the gzip writer")
 		span.EndWithErr(err)
 		return err
 	}
@@ -92,12 +92,12 @@ func (s S3) SendNotification(ctx context.Context, message []map[string]interface
 		Key:             aws.String(s.Config.S3FolderName + "/" + utils.GetDatetimeNow() + ".json"),
 	})
 	if err != nil {
-		fmt.Println("Failed to upload JSON data to S3", err)
+		log.Error().Err(err).Msg("Failed to upload JSON data to S3")
 		span.EndWithErr(err)
 		return err
 	}
 
-	fmt.Println("JSON data uploaded successfully")
+	log.Info().Msg("JSON data uploaded successfully")
 	return nil
 }
 
@@ -118,7 +118,7 @@ func (s S3) IsValidCredential(ctx context.Context) (bool, error) {
 
 		sess, err = session.NewSession(&awsConfig)
 		if err != nil {
-			fmt.Printf("error creating session: %v", err)
+			log.Error().Err(err).Msg("error creating aws session")
 			return false, err
 		}
 	} else {
@@ -127,7 +127,7 @@ func (s S3) IsValidCredential(ctx context.Context) (bool, error) {
 			Credentials: credentials.NewStaticCredentials(s.Config.AWSAccessKey, s.Config.AWSSecretKey, ""),
 		})
 		if err != nil {
-			fmt.Println("Failed to create AWS session", err)
+			log.Error().Err(err).Msg("error creating aws session")
 			return false, err
 		}
 	}
@@ -136,7 +136,7 @@ func (s S3) IsValidCredential(ctx context.Context) (bool, error) {
 
 	_, err = svc.ListBuckets(nil)
 	if err != nil {
-		fmt.Println("Failed to list buckets", err)
+		log.Error().Err(err).Msg("Failed to list buckets")
 		return false, err
 	}
 

deepfence_server/pkg/integration/slack/slack.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net/http"
 
+	intgerr "github.com/deepfence/ThreatMapper/deepfence_server/pkg/integration/errors"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/telemetry"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/utils"
@@ -149,8 +150,9 @@ func (s Slack) SendNotification(ctx context.Context, message []map[string]interf
 
 		// send message to this webhookURL using http
 		// Set up the HTTP request.
-		req, err := http.NewRequest("POST", s.Config.WebhookURL, bytes.NewBuffer(payloadBytes))
+		req, err := http.NewRequest(http.MethodPost, s.Config.WebhookURL, bytes.NewBuffer(payloadBytes))
 		if err != nil {
+			log.Error().Err(err).Msg("error create http request")
 			span.EndWithErr(err)
 			return err
 		}
@@ -160,8 +162,9 @@ func (s Slack) SendNotification(ctx context.Context, message []map[string]interf
 		client := utils.GetHTTPClient()
 		resp, err := client.Do(req)
 		if err != nil {
+			log.Error().Err(err).Msg("error on http request")
 			span.EndWithErr(err)
-			return err
+			return intgerr.CheckHTTPError(err)
 		}
 
 		// Check the response status code.
@@ -210,7 +213,7 @@ func (s Slack) IsValidCredential(ctx context.Context) (bool, error) {
 
 	// send message to this webhookURL using http
 	// Set up the HTTP request.
-	req, err := http.NewRequest("POST", s.Config.WebhookURL, bytes.NewBuffer(payloadBytes))
+	req, err := http.NewRequest(http.MethodPost, s.Config.WebhookURL, bytes.NewBuffer(payloadBytes))
 	if err != nil {
 		log.Error().Msg(err.Error())
 		return false, err