From 4e6e6dec65deea66c4c355fe1ce99782a5471dd9 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 2 Jan 2025 12:33:54 +0100 Subject: [PATCH 01/25] lint: explicit error checks (#3388) * errcheck: tests * fflag errcheck * http_test errcheck (avoid duplicate metric registration) --- cmd/crowdsec-cli/clihub/hub.go | 8 +- pkg/acquisition/modules/file/file_test.go | 18 ++-- pkg/acquisition/modules/http/http_test.go | 89 +++++++++++-------- .../modules/journalctl/journalctl_test.go | 4 +- pkg/acquisition/modules/kafka/kafka_test.go | 6 +- .../modules/kinesis/kinesis_test.go | 12 ++- .../modules/kubernetesaudit/k8s_audit_test.go | 6 +- pkg/acquisition/modules/syslog/syslog_test.go | 4 +- pkg/apiclient/alerts_service_test.go | 18 ++-- pkg/apiclient/auth_key_test.go | 6 +- pkg/apiclient/client_http_test.go | 4 +- pkg/apiclient/client_test.go | 24 +++-- pkg/apiclient/decisions_service_test.go | 31 ++++--- pkg/apiserver/alerts_test.go | 3 +- pkg/csprofiles/csprofiles_test.go | 3 +- pkg/cwhub/download_test.go | 6 +- pkg/fflag/features_test.go | 6 +- pkg/parser/whitelist_test.go | 4 +- pkg/setup/install.go | 32 +++++-- 19 files changed, 185 insertions(+), 99 deletions(-) diff --git a/cmd/crowdsec-cli/clihub/hub.go b/cmd/crowdsec-cli/clihub/hub.go index 9571b3d866d..36e851d1b74 100644 --- a/cmd/crowdsec-cli/clihub/hub.go +++ b/cmd/crowdsec-cli/clihub/hub.go @@ -177,11 +177,15 @@ func (cli *cliHub) upgrade(ctx context.Context, yes bool, dryRun bool, force boo for _, itemType := range cwhub.ItemTypes { for _, item := range hub.GetInstalledByType(itemType, true) { - plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)) + if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil { + return err + } } } - plan.AddCommand(hubops.NewDataRefreshCommand(force)) + if err := plan.AddCommand(hubops.NewDataRefreshCommand(force)); err != nil { + return err + } verbose := (cfg.Cscli.Output == "raw") diff --git a/pkg/acquisition/modules/file/file_test.go b/pkg/acquisition/modules/file/file_test.go index a26e44cc9c7..b9c6e65d8ce 100644 --- a/pkg/acquisition/modules/file/file_test.go +++ b/pkg/acquisition/modules/file/file_test.go @@ -333,14 +333,19 @@ force_inotify: true`, testPattern), logLevel: log.DebugLevel, name: "GlobInotifyChmod", afterConfigure: func() { - f, _ := os.Create("test_files/a.log") - f.Close() + f, err := os.Create("test_files/a.log") + require.NoError(t, err) + err = f.Close() + require.NoError(t, err) time.Sleep(1 * time.Second) - os.Chmod("test_files/a.log", 0o000) + err = os.Chmod("test_files/a.log", 0o000) + require.NoError(t, err) }, teardown: func() { - os.Chmod("test_files/a.log", 0o644) - os.Remove("test_files/a.log") + err := os.Chmod("test_files/a.log", 0o644) + require.NoError(t, err) + err = os.Remove("test_files/a.log") + require.NoError(t, err) }, }, { @@ -353,7 +358,8 @@ force_inotify: true`, testPattern), logLevel: log.DebugLevel, name: "InotifyMkDir", afterConfigure: func() { - os.Mkdir("test_files/pouet/", 0o700) + err := os.Mkdir("test_files/pouet/", 0o700) + require.NoError(t, err) }, teardown: func() { os.Remove("test_files/pouet/") diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index 4d99134419f..c76f311d669 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -218,7 +218,7 @@ func TestGetName(t *testing.T) { assert.Equal(t, "http", h.GetName()) } -func SetupAndRunHTTPSource(t *testing.T, h *HTTPSource, config []byte, metricLevel int) (chan types.Event, *tomb.Tomb) { +func SetupAndRunHTTPSource(t *testing.T, h *HTTPSource, config []byte, metricLevel int) (chan types.Event, *prometheus.Registry, *tomb.Tomb) { ctx := context.Background() subLogger := log.WithFields(log.Fields{ "type": "http", @@ -230,16 +230,18 @@ func SetupAndRunHTTPSource(t *testing.T, h *HTTPSource, config []byte, metricLev err = h.StreamingAcquisition(ctx, out, &tomb) require.NoError(t, err) + testRegistry := prometheus.NewPedanticRegistry() for _, metric := range h.GetMetrics() { - prometheus.Register(metric) + err = testRegistry.Register(metric) + require.NoError(t, err) } - return out, &tomb + return out, testRegistry, &tomb } func TestStreamingAcquisitionWrongHTTPMethod(t *testing.T) { h := &HTTPSource{} - _, tomb := SetupAndRunHTTPSource(t, h, []byte(` + _, _, tomb:= SetupAndRunHTTPSource(t, h, []byte(` source: http listen_addr: 127.0.0.1:8080 path: /test @@ -256,12 +258,13 @@ basic_auth: h.Server.Close() tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } func TestStreamingAcquisitionUnknownPath(t *testing.T) { h := &HTTPSource{} - _, tomb := SetupAndRunHTTPSource(t, h, []byte(` + _, _, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http listen_addr: 127.0.0.1:8080 path: /test @@ -278,12 +281,13 @@ basic_auth: h.Server.Close() tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } func TestStreamingAcquisitionBasicAuth(t *testing.T) { h := &HTTPSource{} - _, tomb := SetupAndRunHTTPSource(t, h, []byte(` + _, _, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http listen_addr: 127.0.0.1:8080 path: /test @@ -310,12 +314,13 @@ basic_auth: h.Server.Close() tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } func TestStreamingAcquisitionBadHeaders(t *testing.T) { h := &HTTPSource{} - _, tomb := SetupAndRunHTTPSource(t, h, []byte(` + _, _, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http listen_addr: 127.0.0.1:8080 path: /test @@ -337,12 +342,13 @@ headers: h.Server.Close() tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } func TestStreamingAcquisitionMaxBodySize(t *testing.T) { h := &HTTPSource{} - _, tomb := SetupAndRunHTTPSource(t, h, []byte(` + _, _, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http listen_addr: 127.0.0.1:8080 path: /test @@ -365,12 +371,13 @@ max_body_size: 5`), 0) h.Server.Close() tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } func TestStreamingAcquisitionSuccess(t *testing.T) { h := &HTTPSource{} - out, tomb := SetupAndRunHTTPSource(t, h, []byte(` + out, reg, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http listen_addr: 127.0.0.1:8080 path: /test @@ -396,16 +403,17 @@ headers: err = <-errChan require.NoError(t, err) - assertMetrics(t, h.GetMetrics(), 1) + assertMetrics(t, reg, h.GetMetrics(), 1) h.Server.Close() tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } func TestStreamingAcquisitionCustomStatusCodeAndCustomHeaders(t *testing.T) { h := &HTTPSource{} - out, tomb := SetupAndRunHTTPSource(t, h, []byte(` + out, reg, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http listen_addr: 127.0.0.1:8080 path: /test @@ -435,11 +443,12 @@ custom_headers: err = <-errChan require.NoError(t, err) - assertMetrics(t, h.GetMetrics(), 1) + assertMetrics(t, reg, h.GetMetrics(), 1) h.Server.Close() tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } type slowReader struct { @@ -495,7 +504,7 @@ func assertEvents(out chan types.Event, expected []string, errChan chan error) { func TestStreamingAcquisitionTimeout(t *testing.T) { h := &HTTPSource{} - _, tomb := SetupAndRunHTTPSource(t, h, []byte(` + _, _, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http listen_addr: 127.0.0.1:8080 path: /test @@ -525,12 +534,13 @@ timeout: 1s`), 0) h.Server.Close() tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } func TestStreamingAcquisitionTLSHTTPRequest(t *testing.T) { h := &HTTPSource{} - _, tomb := SetupAndRunHTTPSource(t, h, []byte(` + _, _, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http listen_addr: 127.0.0.1:8080 auth_type: mtls @@ -549,12 +559,13 @@ tls: h.Server.Close() tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } func TestStreamingAcquisitionTLSWithHeadersAuthSuccess(t *testing.T) { h := &HTTPSource{} - out, tomb := SetupAndRunHTTPSource(t, h, []byte(` + out, reg, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http listen_addr: 127.0.0.1:8080 path: /test @@ -600,16 +611,17 @@ tls: err = <-errChan require.NoError(t, err) - assertMetrics(t, h.GetMetrics(), 0) + assertMetrics(t, reg, h.GetMetrics(), 0) h.Server.Close() tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } func TestStreamingAcquisitionMTLS(t *testing.T) { h := &HTTPSource{} - out, tomb := SetupAndRunHTTPSource(t, h, []byte(` + out, reg, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http listen_addr: 127.0.0.1:8080 path: /test @@ -657,16 +669,17 @@ tls: err = <-errChan require.NoError(t, err) - assertMetrics(t, h.GetMetrics(), 0) + assertMetrics(t, reg, h.GetMetrics(), 0) h.Server.Close() tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } func TestStreamingAcquisitionGzipData(t *testing.T) { h := &HTTPSource{} - out, tomb := SetupAndRunHTTPSource(t, h, []byte(` + out, reg, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http listen_addr: 127.0.0.1:8080 path: /test @@ -709,16 +722,17 @@ headers: err = <-errChan require.NoError(t, err) - assertMetrics(t, h.GetMetrics(), 2) + assertMetrics(t, reg, h.GetMetrics(), 2) h.Server.Close() tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } func TestStreamingAcquisitionNDJson(t *testing.T) { h := &HTTPSource{} - out, tomb := SetupAndRunHTTPSource(t, h, []byte(` + out, reg, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http listen_addr: 127.0.0.1:8080 path: /test @@ -747,15 +761,16 @@ headers: err = <-errChan require.NoError(t, err) - assertMetrics(t, h.GetMetrics(), 2) + assertMetrics(t, reg, h.GetMetrics(), 2) h.Server.Close() tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } -func assertMetrics(t *testing.T, metrics []prometheus.Collector, expected int) { - promMetrics, err := prometheus.DefaultGatherer.Gather() +func assertMetrics(t *testing.T, reg *prometheus.Registry, metrics []prometheus.Collector, expected int) { + promMetrics, err := reg.Gather() require.NoError(t, err) isExist := false diff --git a/pkg/acquisition/modules/journalctl/journalctl_test.go b/pkg/acquisition/modules/journalctl/journalctl_test.go index fedbed6b707..48b034f41c6 100644 --- a/pkg/acquisition/modules/journalctl/journalctl_test.go +++ b/pkg/acquisition/modules/journalctl/journalctl_test.go @@ -12,6 +12,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "gopkg.in/tomb.v2" "github.com/crowdsecurity/go-cs-lib/cstest" @@ -268,7 +269,8 @@ journalctl_filter: } tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) output, _ := exec.Command("pgrep", "-x", "journalctl").CombinedOutput() if len(output) != 0 { diff --git a/pkg/acquisition/modules/kafka/kafka_test.go b/pkg/acquisition/modules/kafka/kafka_test.go index d796166a6ca..2f3361c4f6b 100644 --- a/pkg/acquisition/modules/kafka/kafka_test.go +++ b/pkg/acquisition/modules/kafka/kafka_test.go @@ -194,7 +194,8 @@ topic: crowdsecplaintext`), subLogger, configuration.METRICS_NONE) } require.Equal(t, ts.expectedLines, actualLines) tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) }) } } @@ -271,7 +272,8 @@ tls: } require.Equal(t, ts.expectedLines, actualLines) tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) }) } } diff --git a/pkg/acquisition/modules/kinesis/kinesis_test.go b/pkg/acquisition/modules/kinesis/kinesis_test.go index 778dda4a681..3f6d780b192 100644 --- a/pkg/acquisition/modules/kinesis/kinesis_test.go +++ b/pkg/acquisition/modules/kinesis/kinesis_test.go @@ -63,7 +63,8 @@ func GenSubObject(t *testing.T, i int) []byte { var b bytes.Buffer gz := gzip.NewWriter(&b) - gz.Write(body) + _, err = gz.Write(body) + require.NoError(t, err) gz.Close() // AWS actually base64 encodes the data, but it looks like kinesis automatically decodes it at some point // localstack does not do it, so let's just write a raw gzipped stream @@ -198,7 +199,8 @@ stream_name: stream-1-shard`, } tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } } @@ -246,7 +248,8 @@ stream_name: stream-2-shards`, } assert.Equal(t, test.count, c) tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } } @@ -290,7 +293,8 @@ from_subscription: true`, assert.Equal(t, fmt.Sprintf("%d", i), e.Line.Raw) } tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) } } diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit_test.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit_test.go index a086a756e4a..bf8a8cea02c 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit_test.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit_test.go @@ -85,7 +85,8 @@ webhook_path: /k8s-audit`, err = f.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) require.NoError(t, err) - f.StreamingAcquisition(ctx, out, tb) + err = f.StreamingAcquisition(ctx, out, tb) + require.NoError(t, err) time.Sleep(1 * time.Second) tb.Kill(nil) @@ -260,7 +261,8 @@ webhook_path: /k8s-audit`, req := httptest.NewRequest(test.method, "/k8s-audit", strings.NewReader(test.body)) w := httptest.NewRecorder() - f.StreamingAcquisition(ctx, out, tb) + err = f.StreamingAcquisition(ctx, out, tb) + require.NoError(t, err) f.webhookHandler(w, req) diff --git a/pkg/acquisition/modules/syslog/syslog_test.go b/pkg/acquisition/modules/syslog/syslog_test.go index 57fa3e8747b..3008ba5507b 100644 --- a/pkg/acquisition/modules/syslog/syslog_test.go +++ b/pkg/acquisition/modules/syslog/syslog_test.go @@ -10,6 +10,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "gopkg.in/tomb.v2" "github.com/crowdsecurity/go-cs-lib/cstest" @@ -168,7 +169,8 @@ listen_addr: 127.0.0.1`, } assert.Equal(t, ts.expectedLines, actualLines) tomb.Kill(nil) - tomb.Wait() + err = tomb.Wait() + require.NoError(t, err) }) } } diff --git a/pkg/apiclient/alerts_service_test.go b/pkg/apiclient/alerts_service_test.go index 0d1ff41685f..9df633fa8be 100644 --- a/pkg/apiclient/alerts_service_test.go +++ b/pkg/apiclient/alerts_service_test.go @@ -23,7 +23,8 @@ func TestAlertsListAsMachine(t *testing.T) { mux, urlx, teardown := setup() mux.HandleFunc("/watchers/login", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + _, err := w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + assert.NoError(t, err) }) log.Printf("URL is %s", urlx) @@ -202,7 +203,8 @@ func TestAlertsGetAsMachine(t *testing.T) { mux, urlx, teardown := setup() mux.HandleFunc("/watchers/login", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + _, err := w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + assert.NoError(t, err) }) log.Printf("URL is %s", urlx) @@ -368,13 +370,15 @@ func TestAlertsCreateAsMachine(t *testing.T) { mux, urlx, teardown := setup() mux.HandleFunc("/watchers/login", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + _, err := w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + assert.NoError(t, err) }) mux.HandleFunc("/alerts", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, "POST") w.WriteHeader(http.StatusOK) - w.Write([]byte(`["3"]`)) + _, err := w.Write([]byte(`["3"]`)) + assert.NoError(t, err) }) log.Printf("URL is %s", urlx) @@ -408,14 +412,16 @@ func TestAlertsDeleteAsMachine(t *testing.T) { mux, urlx, teardown := setup() mux.HandleFunc("/watchers/login", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + _, err := w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + assert.NoError(t, err) }) mux.HandleFunc("/alerts", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, "DELETE") assert.Equal(t, "ip=1.2.3.4", r.URL.RawQuery) w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"message":"0 deleted alerts"}`)) + _, err := w.Write([]byte(`{"message":"0 deleted alerts"}`)) + assert.NoError(t, err) }) log.Printf("URL is %s", urlx) diff --git a/pkg/apiclient/auth_key_test.go b/pkg/apiclient/auth_key_test.go index f686de6227a..b7cce3e15c9 100644 --- a/pkg/apiclient/auth_key_test.go +++ b/pkg/apiclient/auth_key_test.go @@ -24,10 +24,12 @@ func TestApiAuth(t *testing.T) { if r.Header.Get("X-Api-Key") == "ixu" { assert.Equal(t, "ip=1.2.3.4", r.URL.RawQuery) w.WriteHeader(http.StatusOK) - w.Write([]byte(`null`)) + _, err := w.Write([]byte(`null`)) + assert.NoError(t, err) } else { w.WriteHeader(http.StatusForbidden) - w.Write([]byte(`{"message":"access forbidden"}`)) + _, err := w.Write([]byte(`{"message":"access forbidden"}`)) + assert.NoError(t, err) } }) diff --git a/pkg/apiclient/client_http_test.go b/pkg/apiclient/client_http_test.go index 45cd8410a8e..0d6cf3d993e 100644 --- a/pkg/apiclient/client_http_test.go +++ b/pkg/apiclient/client_http_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" @@ -31,7 +32,8 @@ func TestNewRequestInvalid(t *testing.T) { /*mock login*/ mux.HandleFunc("/watchers/login", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusUnauthorized) - w.Write([]byte(`{"code": 401, "message" : "bad login/password"}`)) + _, err := w.Write([]byte(`{"code": 401, "message" : "bad login/password"}`)) + assert.NoError(t, err) }) mux.HandleFunc("/alerts", func(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/apiclient/client_test.go b/pkg/apiclient/client_test.go index 327bf8fbd9f..c172849c21e 100644 --- a/pkg/apiclient/client_test.go +++ b/pkg/apiclient/client_test.go @@ -101,7 +101,8 @@ func TestNewClientOk(t *testing.T) { /*mock login*/ mux.HandleFunc("/watchers/login", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + _, err := w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + assert.NoError(t, err) }) mux.HandleFunc("/alerts", func(w http.ResponseWriter, r *http.Request) { @@ -138,7 +139,8 @@ func TestNewClientOk_UnixSocket(t *testing.T) { /*mock login*/ mux.HandleFunc("/watchers/login", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + _, err := w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + assert.NoError(t, err) }) mux.HandleFunc("/alerts", func(w http.ResponseWriter, r *http.Request) { @@ -174,7 +176,8 @@ func TestNewClientKo(t *testing.T) { /*mock login*/ mux.HandleFunc("/watchers/login", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusUnauthorized) - w.Write([]byte(`{"code": 401, "message" : "bad login/password"}`)) + _, err := w.Write([]byte(`{"code": 401, "message" : "bad login/password"}`)) + assert.NoError(t, err) }) mux.HandleFunc("/alerts", func(w http.ResponseWriter, r *http.Request) { @@ -200,7 +203,8 @@ func TestNewDefaultClient(t *testing.T) { mux.HandleFunc("/alerts", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusUnauthorized) - w.Write([]byte(`{"code": 401, "message" : "brr"}`)) + _, err := w.Write([]byte(`{"code": 401, "message" : "brr"}`)) + assert.NoError(t, err) }) _, _, err = client.Alerts.List(context.Background(), AlertsListOpts{}) @@ -228,7 +232,8 @@ func TestNewDefaultClient_UnixSocket(t *testing.T) { mux.HandleFunc("/alerts", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusUnauthorized) - w.Write([]byte(`{"code": 401, "message" : "brr"}`)) + _, err := w.Write([]byte(`{"code": 401, "message" : "brr"}`)) + assert.NoError(t, err) }) _, _, err = client.Alerts.List(context.Background(), AlertsListOpts{}) @@ -266,7 +271,8 @@ func TestNewClientRegisterOK(t *testing.T) { mux.HandleFunc("/watchers", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, "POST") w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + _, err := w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + assert.NoError(t, err) }) apiURL, err := url.Parse(urlx + "/") @@ -298,7 +304,8 @@ func TestNewClientRegisterOK_UnixSocket(t *testing.T) { mux.HandleFunc("/watchers", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, "POST") w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + _, err := w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + assert.NoError(t, err) }) apiURL, err := url.Parse(urlx) @@ -331,7 +338,8 @@ func TestNewClientBadAnswer(t *testing.T) { mux.HandleFunc("/watchers", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, "POST") w.WriteHeader(http.StatusUnauthorized) - w.Write([]byte(`bad`)) + _, err := w.Write([]byte(`bad`)) + assert.NoError(t, err) }) apiURL, err := url.Parse(urlx + "/") diff --git a/pkg/apiclient/decisions_service_test.go b/pkg/apiclient/decisions_service_test.go index 942d14689ff..b8bc327a7d7 100644 --- a/pkg/apiclient/decisions_service_test.go +++ b/pkg/apiclient/decisions_service_test.go @@ -31,11 +31,12 @@ func TestDecisionsList(t *testing.T) { assert.Equal(t, "ip=1.2.3.4", r.URL.RawQuery) assert.Equal(t, "ixu", r.Header.Get("X-Api-Key")) w.WriteHeader(http.StatusOK) - w.Write([]byte(`[{"duration":"3h59m55.756182786s","id":4,"origin":"cscli","scenario":"manual 'ban' from '82929df7ee394b73b81252fe3b4e50203yaT2u6nXiaN7Ix9'","scope":"Ip","type":"ban","value":"1.2.3.4"}]`)) + _, err := w.Write([]byte(`[{"duration":"3h59m55.756182786s","id":4,"origin":"cscli","scenario":"manual 'ban' from '82929df7ee394b73b81252fe3b4e50203yaT2u6nXiaN7Ix9'","scope":"Ip","type":"ban","value":"1.2.3.4"}]`)) + assert.NoError(t, err) } else { w.WriteHeader(http.StatusOK) - w.Write([]byte(`null`)) - // no results + _, err := w.Write([]byte(`null`)) + assert.NoError(t, err) } }) @@ -90,10 +91,12 @@ func TestDecisionsStream(t *testing.T) { if r.Method == http.MethodGet { if strings.Contains(r.URL.RawQuery, "startup=true") { w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"deleted":null,"new":[{"duration":"3h59m55.756182786s","id":4,"origin":"cscli","scenario":"manual 'ban' from '82929df7ee394b73b81252fe3b4e50203yaT2u6nXiaN7Ix9'","scope":"Ip","type":"ban","value":"1.2.3.4"}]}`)) + _, err := w.Write([]byte(`{"deleted":null,"new":[{"duration":"3h59m55.756182786s","id":4,"origin":"cscli","scenario":"manual 'ban' from '82929df7ee394b73b81252fe3b4e50203yaT2u6nXiaN7Ix9'","scope":"Ip","type":"ban","value":"1.2.3.4"}]}`)) + assert.NoError(t, err) } else { w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"deleted":null,"new":null}`)) + _, err := w.Write([]byte(`{"deleted":null,"new":null}`)) + assert.NoError(t, err) } } }) @@ -163,10 +166,12 @@ func TestDecisionsStreamV3Compatibility(t *testing.T) { if r.Method == http.MethodGet { if strings.Contains(r.URL.RawQuery, "startup=true") { w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"deleted":[{"scope":"ip","decisions":["1.2.3.5"]}],"new":[{"scope":"ip", "scenario": "manual 'ban' from '82929df7ee394b73b81252fe3b4e50203yaT2u6nXiaN7Ix9'", "decisions":[{"duration":"3h59m55.756182786s","value":"1.2.3.4"}]}]}`)) + _, err := w.Write([]byte(`{"deleted":[{"scope":"ip","decisions":["1.2.3.5"]}],"new":[{"scope":"ip", "scenario": "manual 'ban' from '82929df7ee394b73b81252fe3b4e50203yaT2u6nXiaN7Ix9'", "decisions":[{"duration":"3h59m55.756182786s","value":"1.2.3.4"}]}]}`)) + assert.NoError(t, err) } else { w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"deleted":null,"new":null}`)) + _, err := w.Write([]byte(`{"deleted":null,"new":null}`)) + assert.NoError(t, err) } } }) @@ -227,9 +232,10 @@ func TestDecisionsStreamV3(t *testing.T) { if r.Method == http.MethodGet { w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"deleted":[{"scope":"ip","decisions":["1.2.3.5"]}], + _, err := w.Write([]byte(`{"deleted":[{"scope":"ip","decisions":["1.2.3.5"]}], "new":[{"scope":"ip", "scenario": "manual 'ban' from '82929df7ee394b73b81252fe3b4e50203yaT2u6nXiaN7Ix9'", "decisions":[{"duration":"3h59m55.756182786s","value":"1.2.3.4"}]}], "links": {"blocklists":[{"name":"blocklist1","url":"/v3/blocklist","scope":"ip","remediation":"ban","duration":"24h"}]}}`)) + assert.NoError(t, err) } }) @@ -303,7 +309,8 @@ func TestDecisionsFromBlocklist(t *testing.T) { if r.Method == http.MethodGet { w.WriteHeader(http.StatusOK) - w.Write([]byte("1.2.3.4\r\n1.2.3.5")) + _, err := w.Write([]byte("1.2.3.4\r\n1.2.3.5")) + assert.NoError(t, err) } }) @@ -388,14 +395,16 @@ func TestDeleteDecisions(t *testing.T) { mux, urlx, teardown := setup() mux.HandleFunc("/watchers/login", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + _, err := w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + assert.NoError(t, err) }) mux.HandleFunc("/decisions", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, "DELETE") assert.Equal(t, "ip=1.2.3.4", r.URL.RawQuery) w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"nbDeleted":"1"}`)) + _, err := w.Write([]byte(`{"nbDeleted":"1"}`)) + assert.NoError(t, err) // w.Write([]byte(`{"message":"0 deleted alerts"}`)) }) diff --git a/pkg/apiserver/alerts_test.go b/pkg/apiserver/alerts_test.go index 4c5c6ef129c..c4edb42d475 100644 --- a/pkg/apiserver/alerts_test.go +++ b/pkg/apiserver/alerts_test.go @@ -142,7 +142,8 @@ func TestCreateAlertChannels(t *testing.T) { ctx := context.Background() apiServer, config := NewAPIServer(t, ctx) apiServer.controller.PluginChannel = make(chan csplugin.ProfileAlert) - apiServer.InitController() + err := apiServer.InitController() + require.NoError(t, err) loginResp := LoginToTestAPI(t, ctx, apiServer.router, config) lapi := LAPI{router: apiServer.router, loginResp: loginResp} diff --git a/pkg/csprofiles/csprofiles_test.go b/pkg/csprofiles/csprofiles_test.go index d09bf25d95b..dc3239fe5c1 100644 --- a/pkg/csprofiles/csprofiles_test.go +++ b/pkg/csprofiles/csprofiles_test.go @@ -119,7 +119,8 @@ func TestEvaluateProfile(t *testing.T) { Alert *models.Alert } - exprhelpers.Init(nil) + err := exprhelpers.Init(nil) + require.NoError(t, err) tests := []struct { name string diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go index ec07862abcf..fc0b257a284 100644 --- a/pkg/cwhub/download_test.go +++ b/pkg/cwhub/download_test.go @@ -20,10 +20,12 @@ func TestFetchIndex(t *testing.T) { mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Query().Get("with_content") == "true" { w.WriteHeader(http.StatusOK) - w.Write([]byte(`Hi I'm an index with content`)) + _, err := w.Write([]byte(`Hi I'm an index with content`)) + assert.NoError(t, err) } else { w.WriteHeader(http.StatusOK) - w.Write([]byte(`Hi I'm a regular index`)) + _, err := w.Write([]byte(`Hi I'm a regular index`)) + assert.NoError(t, err) } })) defer mockServer.Close() diff --git a/pkg/fflag/features_test.go b/pkg/fflag/features_test.go index 481e86573e8..144e7049362 100644 --- a/pkg/fflag/features_test.go +++ b/pkg/fflag/features_test.go @@ -376,11 +376,13 @@ func TestGetEnabledFeatures(t *testing.T) { feat1, err := fr.GetFeature("new_standard") require.NoError(t, err) - feat1.Set(true) + err = feat1.Set(true) + require.Error(t, err, "the flag is deprecated") feat2, err := fr.GetFeature("experimental1") require.NoError(t, err) - feat2.Set(true) + err = feat2.Set(true) + require.NoError(t, err) expected := []string{ "experimental1", diff --git a/pkg/parser/whitelist_test.go b/pkg/parser/whitelist_test.go index 02846f17fc1..a3b95b2fa3f 100644 --- a/pkg/parser/whitelist_test.go +++ b/pkg/parser/whitelist_test.go @@ -284,9 +284,9 @@ func TestWhitelistCheck(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var err error node.Whitelist = tt.whitelist - node.CompileWLs() + _, err := node.CompileWLs() + require.NoError(t, err) isWhitelisted := node.CheckIPsWL(tt.event) if !isWhitelisted { isWhitelisted, err = node.CheckExprWL(map[string]interface{}{"evt": tt.event}, tt.event) diff --git a/pkg/setup/install.go b/pkg/setup/install.go index 42634672912..3d1540f23be 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -71,10 +71,14 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub. return fmt.Errorf("collection %s not found", collection) } - plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)) + if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)); err != nil { + return err + } if !downloadOnly { - plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) + if err := plan.AddCommand(hubops.NewEnableCommand(item, forceAction)); err != nil { + return err + } } } @@ -84,10 +88,14 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub. return fmt.Errorf("parser %s not found", parser) } - plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)) + if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)); err != nil { + return err + } if !downloadOnly { - plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) + if err := plan.AddCommand(hubops.NewEnableCommand(item, forceAction)); err != nil { + return err + } } } @@ -97,10 +105,14 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub. return fmt.Errorf("scenario %s not found", scenario) } - plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)) + if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)); err != nil { + return err + } if !downloadOnly { - plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) + if err := plan.AddCommand(hubops.NewEnableCommand(item, forceAction)); err != nil { + return err + } } } @@ -110,10 +122,14 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub. return fmt.Errorf("postoverflow %s not found", postoverflow) } - plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)) + if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)); err != nil { + return err + } if !downloadOnly { - plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) + if err := plan.AddCommand(hubops.NewEnableCommand(item, forceAction)); err != nil { + return err + } } } } From f938b0c602ca524f5897ef0f381778c4e9bb2dcc Mon Sep 17 00:00:00 2001 From: Laurence Jones Date: Thu, 2 Jan 2025 14:21:03 +0000 Subject: [PATCH 02/25] enhancement: Hubtest respect patterndir option set via config.yaml (#3386) * enhancement: Hubtest respect patterndir option set via config.yaml * pass patternDir to Run() --------- Co-authored-by: marco --- cmd/crowdsec-cli/clihubtest/explain.go | 7 +++++-- cmd/crowdsec-cli/clihubtest/run.go | 4 +++- cmd/crowdsec-cli/main.go | 1 - pkg/hubtest/hubtest.go | 4 ++-- pkg/hubtest/hubtest_item.go | 18 +++++++----------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cmd/crowdsec-cli/clihubtest/explain.go b/cmd/crowdsec-cli/clihubtest/explain.go index dbe10fa7ec0..877aec98a37 100644 --- a/cmd/crowdsec-cli/clihubtest/explain.go +++ b/cmd/crowdsec-cli/clihubtest/explain.go @@ -14,9 +14,12 @@ func (cli *cliHubTest) explain(testName string, details bool, skipOk bool) error return fmt.Errorf("can't load test: %+v", err) } + cfg := cli.cfg() + patternDir := cfg.ConfigPaths.PatternDir + err = test.ParserAssert.LoadTest(test.ParserResultFile) if err != nil { - if err = test.Run(); err != nil { + if err = test.Run(patternDir); err != nil { return fmt.Errorf("running test '%s' failed: %+v", test.Name, err) } @@ -27,7 +30,7 @@ func (cli *cliHubTest) explain(testName string, details bool, skipOk bool) error err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile) if err != nil { - if err = test.Run(); err != nil { + if err = test.Run(patternDir); err != nil { return fmt.Errorf("running test '%s' failed: %+v", test.Name, err) } diff --git a/cmd/crowdsec-cli/clihubtest/run.go b/cmd/crowdsec-cli/clihubtest/run.go index 31cceb81884..94a3b0c10f3 100644 --- a/cmd/crowdsec-cli/clihubtest/run.go +++ b/cmd/crowdsec-cli/clihubtest/run.go @@ -42,12 +42,14 @@ func (cli *cliHubTest) run(runAll bool, nucleiTargetHost string, appSecHost stri // set timezone to avoid DST issues os.Setenv("TZ", "UTC") + patternDir := cfg.ConfigPaths.PatternDir + for _, test := range hubPtr.Tests { if cfg.Cscli.Output == "human" { log.Infof("Running test '%s'", test.Name) } - err := test.Run() + err := test.Run(patternDir) if err != nil { log.Errorf("running test '%s' failed: %+v", test.Name, err) } diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 87e9d82fea2..936211be7ff 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -91,7 +91,6 @@ func loadConfigFor(command string) (*csconfig.Config, string, error) { "help", "completion", "version", - "hubtest", } if !slices.Contains(noNeedConfig, command) { diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index ce9efcec601..6e5a11fff10 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -25,8 +25,8 @@ type HubTest struct { NucleiTargetHost string AppSecHost string - HubIndex *cwhub.Hub - Tests []*HubTestItem + HubIndex *cwhub.Hub + Tests []*HubTestItem } const ( diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index e8bc56f650a..bf029b0e835 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -382,7 +382,7 @@ func createDirs(dirs []string) error { return nil } -func (t *HubTestItem) RunWithLogFile() error { +func (t *HubTestItem) RunWithLogFile(patternDir string) error { testPath := filepath.Join(t.HubTestPath, t.Name) if _, err := os.Stat(testPath); os.IsNotExist(err) { return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath) @@ -417,11 +417,9 @@ func (t *HubTestItem) RunWithLogFile() error { return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err) } - crowdsecPatternsFolder := csconfig.DefaultConfigPath("patterns") - // copy template patterns folder to runtime folder - if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil { - return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %w", crowdsecPatternsFolder, t.RuntimePatternsPath, err) + if err = CopyDir(patternDir, t.RuntimePatternsPath); err != nil { + return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %w", patternDir, t.RuntimePatternsPath, err) } // install the hub in the runtime folder @@ -566,7 +564,7 @@ func (t *HubTestItem) RunWithLogFile() error { return nil } -func (t *HubTestItem) Run() error { +func (t *HubTestItem) Run(patternDir string) error { var err error t.Success = false @@ -596,11 +594,9 @@ func (t *HubTestItem) Run() error { return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err) } - crowdsecPatternsFolder := csconfig.DefaultConfigPath("patterns") - // copy template patterns folder to runtime folder - if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil { - return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %w", crowdsecPatternsFolder, t.RuntimePatternsPath, err) + if err = CopyDir(patternDir, t.RuntimePatternsPath); err != nil { + return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %w", patternDir, t.RuntimePatternsPath, err) } // create the appsec-configs dir @@ -634,7 +630,7 @@ func (t *HubTestItem) Run() error { } if t.Config.LogFile != "" { - return t.RunWithLogFile() + return t.RunWithLogFile(patternDir) } else if t.Config.NucleiTemplate != "" { return t.RunWithNucleiTemplate() } From 90f7c56aabe86ea25f2bcf4b2c4109609800c97d Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:41:11 +0100 Subject: [PATCH 03/25] wizard: unattended install w/ new hubops (#3392) --- wizard.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/wizard.sh b/wizard.sh index 6e215365f6c..57311d40cdb 100755 --- a/wizard.sh +++ b/wizard.sh @@ -262,20 +262,26 @@ install_collection() { fi done + local YES="" + if [[ ${SILENT} == "false" ]]; then COLLECTION_TO_INSTALL=($(whiptail --separate-output --ok-button Continue --title "Crowdsec collections" --checklist "Available collections in crowdsec, try to pick one that fits your profile. Collections contains parsers and scenarios to protect your system." 20 120 10 "${HMENU[@]}" 3>&1 1>&2 2>&3)) if [ $? -eq 1 ]; then log_err "user bailed out at collection selection" exit 1; fi; + else + YES="--yes" fi; for collection in "${COLLECTION_TO_INSTALL[@]}"; do log_info "Installing collection '${collection}'" - ${CSCLI_BIN_INSTALLED} collections install "${collection}" --error + # shellcheck disable=SC2248 + ${CSCLI_BIN_INSTALLED} collections install "${collection}" --error ${YES} done - ${CSCLI_BIN_INSTALLED} parsers install "crowdsecurity/whitelists" --error + # shellcheck disable=SC2248 + ${CSCLI_BIN_INSTALLED} parsers install "crowdsecurity/whitelists" --error ${YES} if [[ ${SILENT} == "false" ]]; then whiptail --msgbox "Out of safety, I installed a parser called 'crowdsecurity/whitelists'. This one will prevent private IP addresses from being banned, feel free to remove it any time." 20 50 fi From 5c7b957a34fd4e743f5127e85974c75a51f8df02 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:58:03 +0100 Subject: [PATCH 04/25] cscli: restyle table titles; autocomplete "cscli metrics show" (#3391) * cscli: restyle table titles; autocomplete "cscli metrics show" * lint --- cmd/crowdsec-cli/clialert/table.go | 2 +- cmd/crowdsec-cli/clibouncer/inspect.go | 2 +- cmd/crowdsec-cli/clibouncer/list.go | 3 +- cmd/crowdsec-cli/clihub/items.go | 2 +- cmd/crowdsec-cli/clihub/utils_table.go | 4 +- cmd/crowdsec-cli/cliitem/inspect.go | 2 +- cmd/crowdsec-cli/cliitem/metrics_table.go | 13 +++-- cmd/crowdsec-cli/climachine/inspect.go | 4 +- cmd/crowdsec-cli/climachine/list.go | 3 +- cmd/crowdsec-cli/climetrics/list.go | 3 +- cmd/crowdsec-cli/climetrics/show.go | 15 ++++++ cmd/crowdsec-cli/climetrics/statacquis.go | 6 +-- cmd/crowdsec-cli/climetrics/statalert.go | 6 +-- .../climetrics/statappsecengine.go | 6 +-- cmd/crowdsec-cli/climetrics/statappsecrule.go | 5 +- cmd/crowdsec-cli/climetrics/statbouncer.go | 13 ++--- cmd/crowdsec-cli/climetrics/statbucket.go | 6 +-- cmd/crowdsec-cli/climetrics/statdecision.go | 6 +-- cmd/crowdsec-cli/climetrics/statlapi.go | 6 +-- .../climetrics/statlapibouncer.go | 6 +-- .../climetrics/statlapidecision.go | 6 +-- .../climetrics/statlapimachine.go | 6 +-- cmd/crowdsec-cli/climetrics/statparser.go | 6 +-- cmd/crowdsec-cli/climetrics/statstash.go | 6 +-- cmd/crowdsec-cli/climetrics/statwhitelist.go | 6 +-- cmd/crowdsec-cli/climetrics/store.go | 3 +- .../modules/appsec/appsec_runner_test.go | 15 +++--- pkg/acquisition/modules/http/http.go | 27 ++++++++-- pkg/acquisition/modules/http/http_test.go | 8 +-- pkg/alertcontext/alertcontext_test.go | 10 ++-- pkg/cwhub/item.go | 54 +++++++++---------- pkg/hubops/plan.go | 8 +-- pkg/hubtest/hubtest_item.go | 9 ++-- test/bats/04_nocapi.bats | 2 +- test/bats/08_metrics.bats | 4 +- test/bats/08_metrics_bouncer.bats | 28 +++++++--- test/bats/80_alerts.bats | 2 +- test/bats/cscli-hubtype-list.bats | 2 +- 38 files changed, 185 insertions(+), 130 deletions(-) diff --git a/cmd/crowdsec-cli/clialert/table.go b/cmd/crowdsec-cli/clialert/table.go index 1416e1e435c..4fe7c4b99c6 100644 --- a/cmd/crowdsec-cli/clialert/table.go +++ b/cmd/crowdsec-cli/clialert/table.go @@ -86,7 +86,7 @@ func alertDecisionsTable(out io.Writer, wantColor string, alert *models.Alert) { } if foundActive { - fmt.Printf(" - Active Decisions :\n") + t.Writer.SetTitle("Active Decisions") t.Render() // Send output } } diff --git a/cmd/crowdsec-cli/clibouncer/inspect.go b/cmd/crowdsec-cli/clibouncer/inspect.go index b62344baa9b..9f1d56124d8 100644 --- a/cmd/crowdsec-cli/clibouncer/inspect.go +++ b/cmd/crowdsec-cli/clibouncer/inspect.go @@ -47,7 +47,7 @@ func (cli *cliBouncers) inspectHuman(out io.Writer, bouncer *ent.Bouncer) { t.AppendRow(table.Row{"Feature Flags", ff}) } - io.WriteString(out, t.Render()+"\n") + fmt.Fprint(out, t.Render()) } func (cli *cliBouncers) inspect(bouncer *ent.Bouncer) error { diff --git a/cmd/crowdsec-cli/clibouncer/list.go b/cmd/crowdsec-cli/clibouncer/list.go index a13ca994e1e..4ed22ce752f 100644 --- a/cmd/crowdsec-cli/clibouncer/list.go +++ b/cmd/crowdsec-cli/clibouncer/list.go @@ -37,7 +37,7 @@ func (cli *cliBouncers) listHuman(out io.Writer, bouncers ent.Bouncers) { t.AppendRow(table.Row{b.Name, b.IPAddress, revoked, lastPull, b.Type, b.Version, b.AuthType}) } - io.WriteString(out, t.Render()+"\n") + fmt.Fprintln(out, t.Render()) } func (cli *cliBouncers) listCSV(out io.Writer, bouncers ent.Bouncers) error { @@ -71,7 +71,6 @@ func (cli *cliBouncers) listCSV(out io.Writer, bouncers ent.Bouncers) error { func (cli *cliBouncers) List(ctx context.Context, out io.Writer, db *database.Client) error { // XXX: must use the provided db object, the one in the struct might be nil // (calling List directly skips the PersistentPreRunE) - bouncers, err := db.ListBouncers(ctx) if err != nil { return fmt.Errorf("unable to list bouncers: %w", err) diff --git a/cmd/crowdsec-cli/clihub/items.go b/cmd/crowdsec-cli/clihub/items.go index f63dc4bedd7..730d2208be0 100644 --- a/cmd/crowdsec-cli/clihub/items.go +++ b/cmd/crowdsec-cli/clihub/items.go @@ -105,7 +105,7 @@ func ListItems(out io.Writer, wantColor string, itemTypes []string, items map[st return fmt.Errorf("failed to parse: %w", err) } - out.Write(x) + fmt.Fprint(out, string(x)) case "raw": csvwriter := csv.NewWriter(out) diff --git a/cmd/crowdsec-cli/clihub/utils_table.go b/cmd/crowdsec-cli/clihub/utils_table.go index 4693161005b..b89f8447896 100644 --- a/cmd/crowdsec-cli/clihub/utils_table.go +++ b/cmd/crowdsec-cli/clihub/utils_table.go @@ -20,6 +20,6 @@ func listHubItemTable(out io.Writer, wantColor string, title string, items []*cw t.AppendRow(table.Row{item.Name, status, item.State.LocalVersion, item.State.LocalPath}) } - io.WriteString(out, title+"\n") - io.WriteString(out, t.Render()+"\n") + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) } diff --git a/cmd/crowdsec-cli/cliitem/inspect.go b/cmd/crowdsec-cli/cliitem/inspect.go index 596674aa788..9939de1810e 100644 --- a/cmd/crowdsec-cli/cliitem/inspect.go +++ b/cmd/crowdsec-cli/cliitem/inspect.go @@ -1,8 +1,8 @@ package cliitem import ( - "fmt" "encoding/json" + "fmt" "os" "path/filepath" diff --git a/cmd/crowdsec-cli/cliitem/metrics_table.go b/cmd/crowdsec-cli/cliitem/metrics_table.go index 378394bad85..a41ea0fad39 100644 --- a/cmd/crowdsec-cli/cliitem/metrics_table.go +++ b/cmd/crowdsec-cli/cliitem/metrics_table.go @@ -10,7 +10,6 @@ import ( "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/cstable" ) - func appsecMetricsTable(out io.Writer, wantColor string, itemName string, metrics map[string]int) { t := cstable.NewLight(out, wantColor).Writer t.AppendHeader(table.Row{"Inband Hits", "Outband Hits"}) @@ -20,8 +19,8 @@ func appsecMetricsTable(out io.Writer, wantColor string, itemName string, metric strconv.Itoa(metrics["outband_hits"]), }) - io.WriteString(out, fmt.Sprintf("\n - (AppSec Rule) %s:\n", itemName)) - io.WriteString(out, t.Render()+"\n") + t.SetTitle("(AppSec) " + itemName) + fmt.Fprintln(out, t.Render()) } func scenarioMetricsTable(out io.Writer, wantColor string, itemName string, metrics map[string]int) { @@ -40,8 +39,8 @@ func scenarioMetricsTable(out io.Writer, wantColor string, itemName string, metr strconv.Itoa(metrics["underflow"]), }) - io.WriteString(out, fmt.Sprintf("\n - (Scenario) %s:\n", itemName)) - io.WriteString(out, t.Render()+"\n") + t.SetTitle("(Scenario) " + itemName) + fmt.Fprintln(out, t.Render()) } func parserMetricsTable(out io.Writer, wantColor string, itemName string, metrics map[string]map[string]int) { @@ -65,7 +64,7 @@ func parserMetricsTable(out io.Writer, wantColor string, itemName string, metric } if showTable { - io.WriteString(out, fmt.Sprintf("\n - (Parser) %s:\n", itemName)) - io.WriteString(out, t.Render()+"\n") + t.SetTitle("(Parser) " + itemName) + fmt.Fprintln(out, t.Render()) } } diff --git a/cmd/crowdsec-cli/climachine/inspect.go b/cmd/crowdsec-cli/climachine/inspect.go index b08f2f62794..e973d07e96b 100644 --- a/cmd/crowdsec-cli/climachine/inspect.go +++ b/cmd/crowdsec-cli/climachine/inspect.go @@ -44,7 +44,7 @@ func (cli *cliMachines) inspectHubHuman(out io.Writer, machine *ent.Machine) { t.AppendHeader(table.Row{"Name", "Status", "Version"}) t.SetTitle(itemType) t.AppendRows(rows) - io.WriteString(out, t.Render()+"\n") + fmt.Fprintln(out, t.Render()) } } @@ -80,7 +80,7 @@ func (cli *cliMachines) inspectHuman(out io.Writer, machine *ent.Machine) { t.AppendRow(table.Row{"Collections", coll.Name}) } - io.WriteString(out, t.Render()+"\n") + fmt.Fprintln(out, t.Render()) } func (cli *cliMachines) inspect(machine *ent.Machine) error { diff --git a/cmd/crowdsec-cli/climachine/list.go b/cmd/crowdsec-cli/climachine/list.go index 6bedb2ad807..6fb45166aa2 100644 --- a/cmd/crowdsec-cli/climachine/list.go +++ b/cmd/crowdsec-cli/climachine/list.go @@ -55,7 +55,7 @@ func (cli *cliMachines) listHuman(out io.Writer, machines ent.Machines) { t.AppendRow(table.Row{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, clientinfo.GetOSNameAndVersion(m), m.AuthType, hb}) } - io.WriteString(out, t.Render()+"\n") + fmt.Fprintln(out, t.Render()) } func (cli *cliMachines) listCSV(out io.Writer, machines ent.Machines) error { @@ -90,7 +90,6 @@ func (cli *cliMachines) listCSV(out io.Writer, machines ent.Machines) error { func (cli *cliMachines) List(ctx context.Context, out io.Writer, db *database.Client) error { // XXX: must use the provided db object, the one in the struct might be nil // (calling List directly skips the PersistentPreRunE) - machines, err := db.ListMachines(ctx) if err != nil { return fmt.Errorf("unable to list machines: %w", err) diff --git a/cmd/crowdsec-cli/climetrics/list.go b/cmd/crowdsec-cli/climetrics/list.go index 27fa99710c8..32e2f8e0a80 100644 --- a/cmd/crowdsec-cli/climetrics/list.go +++ b/cmd/crowdsec-cli/climetrics/list.go @@ -3,7 +3,6 @@ package climetrics import ( "encoding/json" "fmt" - "io" "github.com/fatih/color" "github.com/jedib0t/go-pretty/v6/table" @@ -64,7 +63,7 @@ func (cli *cliMetrics) list() error { t.AppendRow(table.Row{metric.Type, metric.Title, metric.Description}) } - io.WriteString(out, t.Render()+"\n") + fmt.Fprintln(out, t.Render()) case "json": x, err := json.MarshalIndent(allMetrics, "", " ") if err != nil { diff --git a/cmd/crowdsec-cli/climetrics/show.go b/cmd/crowdsec-cli/climetrics/show.go index 045959048f6..172d3799435 100644 --- a/cmd/crowdsec-cli/climetrics/show.go +++ b/cmd/crowdsec-cli/climetrics/show.go @@ -4,11 +4,15 @@ import ( "context" "errors" "fmt" + "slices" + "strings" "github.com/fatih/color" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/crowdsecurity/go-cs-lib/maptools" + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" ) @@ -99,6 +103,17 @@ cscli metrics list; cscli metrics list -o json cscli metrics show acquisition parsers scenarios stash -o json`, // Positional args are optional DisableAutoGenTag: true, + ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + ms := NewMetricStore() + ret := []string{} + for _, section := range maptools.SortedKeys(ms) { + if !slices.Contains(args, section) && strings.Contains(section, toComplete) { + ret = append(ret, section) + } + } + + return ret, cobra.ShellCompDirectiveNoFileComp + }, RunE: func(cmd *cobra.Command, args []string) error { args = expandAlias(args) return cli.show(cmd.Context(), args, url, noUnit) diff --git a/cmd/crowdsec-cli/climetrics/statacquis.go b/cmd/crowdsec-cli/climetrics/statacquis.go index 0af2e796f40..da17b1d9480 100644 --- a/cmd/crowdsec-cli/climetrics/statacquis.go +++ b/cmd/crowdsec-cli/climetrics/statacquis.go @@ -1,6 +1,7 @@ package climetrics import ( + "fmt" "io" "github.com/jedib0t/go-pretty/v6/table" @@ -37,8 +38,7 @@ func (s statAcquis) Table(out io.Writer, wantColor string, noUnit bool, showEmpt log.Warningf("while collecting acquis stats: %s", err) } else if numRows > 0 || showEmpty { title, _ := s.Description() - io.WriteString(out, title+":\n") - io.WriteString(out, t.Render()+"\n") - io.WriteString(out, "\n") + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) } } diff --git a/cmd/crowdsec-cli/climetrics/statalert.go b/cmd/crowdsec-cli/climetrics/statalert.go index 942eceaa75c..416b78f0508 100644 --- a/cmd/crowdsec-cli/climetrics/statalert.go +++ b/cmd/crowdsec-cli/climetrics/statalert.go @@ -1,6 +1,7 @@ package climetrics import ( + "fmt" "io" "strconv" @@ -38,8 +39,7 @@ func (s statAlert) Table(out io.Writer, wantColor string, noUnit bool, showEmpty if numRows > 0 || showEmpty { title, _ := s.Description() - io.WriteString(out, title+":\n") - io.WriteString(out, t.Render()+"\n") - io.WriteString(out, "\n") + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) } } diff --git a/cmd/crowdsec-cli/climetrics/statappsecengine.go b/cmd/crowdsec-cli/climetrics/statappsecengine.go index d924375247f..93cc1283c96 100644 --- a/cmd/crowdsec-cli/climetrics/statappsecengine.go +++ b/cmd/crowdsec-cli/climetrics/statappsecengine.go @@ -1,6 +1,7 @@ package climetrics import ( + "fmt" "io" "github.com/jedib0t/go-pretty/v6/table" @@ -34,8 +35,7 @@ func (s statAppsecEngine) Table(out io.Writer, wantColor string, noUnit bool, sh log.Warningf("while collecting appsec stats: %s", err) } else if numRows > 0 || showEmpty { title, _ := s.Description() - io.WriteString(out, title+":\n") - io.WriteString(out, t.Render()+"\n") - io.WriteString(out, "\n") + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) } } diff --git a/cmd/crowdsec-cli/climetrics/statappsecrule.go b/cmd/crowdsec-cli/climetrics/statappsecrule.go index e06a7c2e2b3..8e243aba642 100644 --- a/cmd/crowdsec-cli/climetrics/statappsecrule.go +++ b/cmd/crowdsec-cli/climetrics/statappsecrule.go @@ -40,9 +40,8 @@ func (s statAppsecRule) Table(out io.Writer, wantColor string, noUnit bool, show if numRows, err := metricsToTable(t, appsecEngineRulesStats, keys, noUnit); err != nil { log.Warningf("while collecting appsec rules stats: %s", err) } else if numRows > 0 || showEmpty { - io.WriteString(out, fmt.Sprintf("Appsec '%s' Rules Metrics:\n", appsecEngine)) - io.WriteString(out, t.Render()+"\n") - io.WriteString(out, "\n") + t.SetTitle(fmt.Sprintf("Appsec '%s' Rules Metrics", appsecEngine)) + fmt.Fprintln(out, t.Render()) } } } diff --git a/cmd/crowdsec-cli/climetrics/statbouncer.go b/cmd/crowdsec-cli/climetrics/statbouncer.go index bc0da152d6d..ac79074d506 100644 --- a/cmd/crowdsec-cli/climetrics/statbouncer.go +++ b/cmd/crowdsec-cli/climetrics/statbouncer.go @@ -176,17 +176,20 @@ func (*statBouncer) extractRawMetrics(metrics []*ent.Metric) ([]bouncerMetricIte if item.Name == nil { logWarningOnce(warningsLogged, "missing 'name' field in metrics reported by "+bouncerName) + // no continue - keep checking the rest valid = false } if item.Unit == nil { logWarningOnce(warningsLogged, "missing 'unit' field in metrics reported by "+bouncerName) + valid = false } if item.Value == nil { logWarningOnce(warningsLogged, "missing 'value' field in metrics reported by "+bouncerName) + valid = false } @@ -439,11 +442,8 @@ func (s *statBouncer) bouncerTable(out io.Writer, bouncerName string, wantColor title = fmt.Sprintf("%s since %s", title, s.oldestTS[bouncerName].String()) } - // don't use SetTitle() because it draws the title inside table box - io.WriteString(out, title+":\n") - io.WriteString(out, t.Render()+"\n") - // empty line between tables - io.WriteString(out, "\n") + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) } // Table displays a table of metrics for each bouncer @@ -452,10 +452,11 @@ func (s *statBouncer) Table(out io.Writer, wantColor string, noUnit bool, showEm for _, bouncerName := range maptools.SortedKeys(s.aggOverOrigin) { s.bouncerTable(out, bouncerName, wantColor, noUnit) + found = true } if !found && showEmpty { - io.WriteString(out, "No bouncer metrics found.\n\n") + fmt.Fprintln(out, "No bouncer metrics found.") } } diff --git a/cmd/crowdsec-cli/climetrics/statbucket.go b/cmd/crowdsec-cli/climetrics/statbucket.go index 1882fe21df1..4cddfeb3731 100644 --- a/cmd/crowdsec-cli/climetrics/statbucket.go +++ b/cmd/crowdsec-cli/climetrics/statbucket.go @@ -1,6 +1,7 @@ package climetrics import ( + "fmt" "io" "github.com/jedib0t/go-pretty/v6/table" @@ -35,8 +36,7 @@ func (s statBucket) Table(out io.Writer, wantColor string, noUnit bool, showEmpt log.Warningf("while collecting scenario stats: %s", err) } else if numRows > 0 || showEmpty { title, _ := s.Description() - io.WriteString(out, title+":\n") - io.WriteString(out, t.Render()+"\n") - io.WriteString(out, "\n") + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) } } diff --git a/cmd/crowdsec-cli/climetrics/statdecision.go b/cmd/crowdsec-cli/climetrics/statdecision.go index b862f49ff12..2f27410f56f 100644 --- a/cmd/crowdsec-cli/climetrics/statdecision.go +++ b/cmd/crowdsec-cli/climetrics/statdecision.go @@ -1,6 +1,7 @@ package climetrics import ( + "fmt" "io" "strconv" @@ -53,8 +54,7 @@ func (s statDecision) Table(out io.Writer, wantColor string, noUnit bool, showEm if numRows > 0 || showEmpty { title, _ := s.Description() - io.WriteString(out, title+":\n") - io.WriteString(out, t.Render()+"\n") - io.WriteString(out, "\n") + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) } } diff --git a/cmd/crowdsec-cli/climetrics/statlapi.go b/cmd/crowdsec-cli/climetrics/statlapi.go index 9559eacf0f4..2f460ca5a71 100644 --- a/cmd/crowdsec-cli/climetrics/statlapi.go +++ b/cmd/crowdsec-cli/climetrics/statlapi.go @@ -1,6 +1,7 @@ package climetrics import ( + "fmt" "io" "strconv" @@ -49,8 +50,7 @@ func (s statLapi) Table(out io.Writer, wantColor string, noUnit bool, showEmpty if numRows > 0 || showEmpty { title, _ := s.Description() - io.WriteString(out, title+":\n") - io.WriteString(out, t.Render()+"\n") - io.WriteString(out, "\n") + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) } } diff --git a/cmd/crowdsec-cli/climetrics/statlapibouncer.go b/cmd/crowdsec-cli/climetrics/statlapibouncer.go index 5e5f63a79d3..2ea6b67cd0a 100644 --- a/cmd/crowdsec-cli/climetrics/statlapibouncer.go +++ b/cmd/crowdsec-cli/climetrics/statlapibouncer.go @@ -1,6 +1,7 @@ package climetrics import ( + "fmt" "io" "github.com/jedib0t/go-pretty/v6/table" @@ -35,8 +36,7 @@ func (s statLapiBouncer) Table(out io.Writer, wantColor string, noUnit bool, sho if numRows > 0 || showEmpty { title, _ := s.Description() - io.WriteString(out, title+":\n") - io.WriteString(out, t.Render()+"\n") - io.WriteString(out, "\n") + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) } } diff --git a/cmd/crowdsec-cli/climetrics/statlapidecision.go b/cmd/crowdsec-cli/climetrics/statlapidecision.go index 44f0e8f4b87..3371cb0e8ff 100644 --- a/cmd/crowdsec-cli/climetrics/statlapidecision.go +++ b/cmd/crowdsec-cli/climetrics/statlapidecision.go @@ -1,6 +1,7 @@ package climetrics import ( + "fmt" "io" "strconv" @@ -57,8 +58,7 @@ func (s statLapiDecision) Table(out io.Writer, wantColor string, noUnit bool, sh if numRows > 0 || showEmpty { title, _ := s.Description() - io.WriteString(out, title+":\n") - io.WriteString(out, t.Render()+"\n") - io.WriteString(out, "\n") + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) } } diff --git a/cmd/crowdsec-cli/climetrics/statlapimachine.go b/cmd/crowdsec-cli/climetrics/statlapimachine.go index 0e6693bea82..04fbb98ae8e 100644 --- a/cmd/crowdsec-cli/climetrics/statlapimachine.go +++ b/cmd/crowdsec-cli/climetrics/statlapimachine.go @@ -1,6 +1,7 @@ package climetrics import ( + "fmt" "io" "github.com/jedib0t/go-pretty/v6/table" @@ -35,8 +36,7 @@ func (s statLapiMachine) Table(out io.Writer, wantColor string, noUnit bool, sho if numRows > 0 || showEmpty { title, _ := s.Description() - io.WriteString(out, title+":\n") - io.WriteString(out, t.Render()+"\n") - io.WriteString(out, "\n") + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) } } diff --git a/cmd/crowdsec-cli/climetrics/statparser.go b/cmd/crowdsec-cli/climetrics/statparser.go index 520e68f9adf..bdc9caa8597 100644 --- a/cmd/crowdsec-cli/climetrics/statparser.go +++ b/cmd/crowdsec-cli/climetrics/statparser.go @@ -1,6 +1,7 @@ package climetrics import ( + "fmt" "io" "github.com/jedib0t/go-pretty/v6/table" @@ -36,8 +37,7 @@ func (s statParser) Table(out io.Writer, wantColor string, noUnit bool, showEmpt log.Warningf("while collecting parsers stats: %s", err) } else if numRows > 0 || showEmpty { title, _ := s.Description() - io.WriteString(out, title+":\n") - io.WriteString(out, t.Render()+"\n") - io.WriteString(out, "\n") + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) } } diff --git a/cmd/crowdsec-cli/climetrics/statstash.go b/cmd/crowdsec-cli/climetrics/statstash.go index 2729de931a1..496deaf0535 100644 --- a/cmd/crowdsec-cli/climetrics/statstash.go +++ b/cmd/crowdsec-cli/climetrics/statstash.go @@ -1,6 +1,7 @@ package climetrics import ( + "fmt" "io" "strconv" @@ -52,8 +53,7 @@ func (s statStash) Table(out io.Writer, wantColor string, noUnit bool, showEmpty if numRows > 0 || showEmpty { title, _ := s.Description() - io.WriteString(out, title+":\n") - io.WriteString(out, t.Render()+"\n") - io.WriteString(out, "\n") + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) } } diff --git a/cmd/crowdsec-cli/climetrics/statwhitelist.go b/cmd/crowdsec-cli/climetrics/statwhitelist.go index 7f533b45b4b..a42f653d50d 100644 --- a/cmd/crowdsec-cli/climetrics/statwhitelist.go +++ b/cmd/crowdsec-cli/climetrics/statwhitelist.go @@ -1,6 +1,7 @@ package climetrics import ( + "fmt" "io" "github.com/jedib0t/go-pretty/v6/table" @@ -36,8 +37,7 @@ func (s statWhitelist) Table(out io.Writer, wantColor string, noUnit bool, showE log.Warningf("while collecting parsers stats: %s", err) } else if numRows > 0 || showEmpty { title, _ := s.Description() - io.WriteString(out, title+":\n") - io.WriteString(out, t.Render()+"\n") - io.WriteString(out, "\n") + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) } } diff --git a/cmd/crowdsec-cli/climetrics/store.go b/cmd/crowdsec-cli/climetrics/store.go index 55fab5dbd7f..6c402447901 100644 --- a/cmd/crowdsec-cli/climetrics/store.go +++ b/cmd/crowdsec-cli/climetrics/store.go @@ -262,7 +262,8 @@ func (ms metricStore) Format(out io.Writer, wantColor string, sections []string, if err != nil { return fmt.Errorf("failed to serialize metrics: %w", err) } - out.Write(x) + + fmt.Fprint(out, string(x)) default: return fmt.Errorf("output format '%s' not supported for this command", outputFormat) } diff --git a/pkg/acquisition/modules/appsec/appsec_runner_test.go b/pkg/acquisition/modules/appsec/appsec_runner_test.go index d07fb153186..38549a9106c 100644 --- a/pkg/acquisition/modules/appsec/appsec_runner_test.go +++ b/pkg/acquisition/modules/appsec/appsec_runner_test.go @@ -3,13 +3,15 @@ package appsecacquisition import ( "testing" - "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" ) func TestAppsecRuleLoad(t *testing.T) { log.SetLevel(log.TraceLevel) + tests := []appsecRuleTest{ { name: "simple rule load", @@ -105,21 +107,22 @@ func TestAppsecRuleLoad(t *testing.T) { Or: []appsec_rule.CustomRule{ { - //Name: "rule1", + // Name: "rule1", Zones: []string{"ARGS"}, Match: appsec_rule.Match{Type: "equals", Value: "toto"}, }, { - //Name: "rule1", + // Name: "rule1", Zones: []string{"ARGS"}, Match: appsec_rule.Match{Type: "equals", Value: "tutu"}, }, { - //Name: "rule1", + // Name: "rule1", Zones: []string{"ARGS"}, Match: appsec_rule.Match{Type: "equals", Value: "tata"}, - }, { - //Name: "rule1", + }, + { + // Name: "rule1", Zones: []string{"ARGS"}, Match: appsec_rule.Match{Type: "equals", Value: "titi"}, }, diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 6bb8228f32c..3e4f26915fd 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -16,7 +16,6 @@ import ( "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" - "gopkg.in/tomb.v2" "gopkg.in/yaml.v3" @@ -36,8 +35,8 @@ var linesRead = prometheus.NewCounterVec( []string{"path", "src"}) type HttpConfiguration struct { - //IPFilter []string `yaml:"ip_filter"` - //ChunkSize *int64 `yaml:"chunk_size"` + // IPFilter []string `yaml:"ip_filter"` + // ChunkSize *int64 `yaml:"chunk_size"` ListenAddr string `yaml:"listen_addr"` Path string `yaml:"path"` AuthType string `yaml:"auth_type"` @@ -76,6 +75,7 @@ func (h *HTTPSource) GetUuid() string { func (h *HTTPSource) UnmarshalConfig(yamlConfig []byte) error { h.Config = HttpConfiguration{} + err := yaml.Unmarshal(yamlConfig, &h.Config) if err != nil { return fmt.Errorf("cannot parse %s datasource configuration: %w", dataSourceName, err) @@ -96,6 +96,7 @@ func (hc *HttpConfiguration) Validate() error { if hc.Path == "" { hc.Path = "/" } + if hc.Path[0] != '/' { return errors.New("path must start with /") } @@ -106,9 +107,11 @@ func (hc *HttpConfiguration) Validate() error { if hc.BasicAuth == nil { return errors.New(baseErr + " basic_auth is not provided") } + if hc.BasicAuth.Username == "" { return errors.New(baseErr + " username is not provided") } + if hc.BasicAuth.Password == "" { return errors.New(baseErr + " password is not provided") } @@ -128,6 +131,7 @@ func (hc *HttpConfiguration) Validate() error { if hc.TLS.ServerCert == "" { return errors.New("server_cert is required") } + if hc.TLS.ServerKey == "" { return errors.New("server_key is required") } @@ -156,6 +160,7 @@ func (hc *HttpConfiguration) Validate() error { func (h *HTTPSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { h.logger = logger h.metricsLevel = MetricsLevel + err := h.UnmarshalConfig(yamlConfig) if err != nil { return err @@ -210,6 +215,7 @@ func (hc *HttpConfiguration) NewTLSConfig() (*tls.Config, error) { if err != nil { return nil, fmt.Errorf("failed to load server cert/key: %w", err) } + tlsConfig.Certificates = []tls.Certificate{cert} } @@ -227,6 +233,7 @@ func (hc *HttpConfiguration) NewTLSConfig() (*tls.Config, error) { if caCertPool == nil { caCertPool = x509.NewCertPool() } + caCertPool.AppendCertsFromPEM(caCert) tlsConfig.ClientCAs = caCertPool tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert @@ -241,10 +248,12 @@ func authorizeRequest(r *http.Request, hc *HttpConfiguration) error { if !ok { return errors.New("missing basic auth") } + if username != hc.BasicAuth.Username || password != hc.BasicAuth.Password { return errors.New("invalid basic auth") } } + if hc.AuthType == "headers" { for key, value := range *hc.Headers { if r.Header.Get(key) != value { @@ -252,6 +261,7 @@ func authorizeRequest(r *http.Request, hc *HttpConfiguration) error { } } } + return nil } @@ -280,6 +290,7 @@ func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc * } decoder := json.NewDecoder(reader) + for { var message json.RawMessage @@ -287,7 +298,9 @@ func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc * if err == io.EOF { break } + w.WriteHeader(http.StatusBadRequest) + return fmt.Errorf("failed to decode: %w", err) } @@ -328,11 +341,13 @@ func (h *HTTPSource) RunServer(out chan types.Event, t *tomb.Tomb) error { http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) return } + if err := authorizeRequest(r, &h.Config); err != nil { h.logger.Errorf("failed to authorize request from '%s': %s", r.RemoteAddr, err) http.Error(w, "Unauthorized", http.StatusUnauthorized) return } + err := h.processRequest(w, r, &h.Config, out) if err != nil { h.logger.Errorf("failed to process request from '%s': %s", r.RemoteAddr, err) @@ -344,6 +359,7 @@ func (h *HTTPSource) RunServer(out chan types.Event, t *tomb.Tomb) error { w.Header().Set(key, value) } } + if h.Config.CustomStatusCode != nil { w.WriteHeader(*h.Config.CustomStatusCode) } else { @@ -367,25 +383,30 @@ func (h *HTTPSource) RunServer(out chan types.Event, t *tomb.Tomb) error { if err != nil { return fmt.Errorf("failed to create tls config: %w", err) } + h.logger.Tracef("tls config: %+v", tlsConfig) h.Server.TLSConfig = tlsConfig } t.Go(func() error { defer trace.CatchPanic("crowdsec/acquis/http/server") + if h.Config.TLS != nil { h.logger.Infof("start https server on %s", h.Config.ListenAddr) + err := h.Server.ListenAndServeTLS(h.Config.TLS.ServerCert, h.Config.TLS.ServerKey) if err != nil && err != http.ErrServerClosed { return fmt.Errorf("https server failed: %w", err) } } else { h.logger.Infof("start http server on %s", h.Config.ListenAddr) + err := h.Server.ListenAndServe() if err != nil && err != http.ErrServerClosed { return fmt.Errorf("http server failed: %w", err) } } + return nil }) diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index c76f311d669..b05979c5adf 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -14,13 +14,15 @@ import ( "testing" "time" - "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/crowdsecurity/go-cs-lib/cstest" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/tomb.v2" + + "github.com/crowdsecurity/go-cs-lib/cstest" + + "github.com/crowdsecurity/crowdsec/pkg/types" ) const ( @@ -241,7 +243,7 @@ func SetupAndRunHTTPSource(t *testing.T, h *HTTPSource, config []byte, metricLev func TestStreamingAcquisitionWrongHTTPMethod(t *testing.T) { h := &HTTPSource{} - _, _, tomb:= SetupAndRunHTTPSource(t, h, []byte(` + _, _, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http listen_addr: 127.0.0.1:8080 path: /test diff --git a/pkg/alertcontext/alertcontext_test.go b/pkg/alertcontext/alertcontext_test.go index 284ff451bc2..b1572edd76b 100644 --- a/pkg/alertcontext/alertcontext_test.go +++ b/pkg/alertcontext/alertcontext_test.go @@ -8,9 +8,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/crowdsecurity/go-cs-lib/ptr" + "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/crowdsecurity/go-cs-lib/ptr" ) func TestNewAlertContext(t *testing.T) { @@ -229,6 +230,7 @@ func TestValidateContextExpr(t *testing.T) { } for _, test := range tests { fmt.Printf("Running test '%s'\n", test.name) + err := ValidateContextExpr(test.key, test.exprs) if test.expectedErr == nil { require.NoError(t, err) @@ -348,13 +350,13 @@ func TestAppsecEventToContext(t *testing.T) { } for _, test := range tests { - //reset cache + // reset cache alertContext = Context{} - //compile + // compile if err := NewAlertContext(test.contextToSend, 100); err != nil { t.Fatalf("failed to compile %s: %s", test.name, err) } - //run + // run metas, errors := AppsecEventToContext(test.match, test.req) assert.Len(t, errors, test.expectedErrLen) diff --git a/pkg/cwhub/item.go b/pkg/cwhub/item.go index 8cdb88a18ed..74b1cfa3ebe 100644 --- a/pkg/cwhub/item.go +++ b/pkg/cwhub/item.go @@ -114,37 +114,38 @@ type Dependencies struct { // a group of items of the same type type itemgroup struct { - typeName string - itemNames []string + typeName string + itemNames []string } func (d Dependencies) byType() []itemgroup { - return []itemgroup{ - {PARSERS, d.Parsers}, - {POSTOVERFLOWS, d.PostOverflows}, - {SCENARIOS, d.Scenarios}, - {CONTEXTS, d.Contexts}, - {APPSEC_CONFIGS, d.AppsecConfigs}, - {APPSEC_RULES, d.AppsecRules}, - {COLLECTIONS, d.Collections}, - } + return []itemgroup{ + {PARSERS, d.Parsers}, + {POSTOVERFLOWS, d.PostOverflows}, + {SCENARIOS, d.Scenarios}, + {CONTEXTS, d.Contexts}, + {APPSEC_CONFIGS, d.AppsecConfigs}, + {APPSEC_RULES, d.AppsecRules}, + {COLLECTIONS, d.Collections}, + } } // SubItems iterates over the sub-items in the struct, excluding the ones that were not found in the hub. func (d Dependencies) SubItems(hub *Hub) func(func(*Item) bool) { - return func(yield func(*Item) bool) { - for _, typeGroup := range d.byType() { - for _, name := range typeGroup.itemNames { - s := hub.GetItem(typeGroup.typeName, name) - if s == nil { - continue - } - if !yield(s) { - return - } - } - } - } + return func(yield func(*Item) bool) { + for _, typeGroup := range d.byType() { + for _, name := range typeGroup.itemNames { + s := hub.GetItem(typeGroup.typeName, name) + if s == nil { + continue + } + + if !yield(s) { + return + } + } + } + } } // Item is created from an index file and enriched with local info. @@ -272,6 +273,7 @@ func (i *Item) CurrentDependencies() Dependencies { if errors.Is(err, fs.ErrNotExist) { return i.Dependencies } + if err != nil { // a file might be corrupted, or in development i.hub.logger.Warningf("can't read dependencies for %s, using index", i.FQName()) @@ -285,11 +287,10 @@ func (i *Item) CurrentDependencies() Dependencies { i.hub.logger.Warningf("can't parse dependencies for %s, using index", i.FQName()) return i.Dependencies } - + return d } - func (i *Item) logMissingSubItems() { if !i.HasSubItems() { return @@ -405,7 +406,6 @@ func (i *Item) SafeToRemoveDeps() ([]*Item, error) { return ret, nil } - // descendants returns a list of all (direct or indirect) dependencies of the item's current version. func (i *Item) descendants() ([]*Item, error) { var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error diff --git a/pkg/hubops/plan.go b/pkg/hubops/plan.go index 1535bc41c64..eb99056fab3 100644 --- a/pkg/hubops/plan.go +++ b/pkg/hubops/plan.go @@ -8,8 +8,8 @@ import ( "strings" "github.com/AlecAivazis/survey/v2" - isatty "github.com/mattn/go-isatty" "github.com/fatih/color" + isatty "github.com/mattn/go-isatty" "github.com/crowdsecurity/go-cs-lib/slicetools" @@ -59,13 +59,13 @@ func UniqueKey(c Command) string { type ActionPlan struct { // hold the list of Commands to be executed as part of the action plan. // If a command is skipped (i.e. calling Prepare() returned false), it won't be included in the slice. - commands []Command + commands []Command // Tracks unique commands commandsTracker map[string]struct{} // A reference to the Hub instance, required for dependency lookup. - hub *cwhub.Hub + hub *cwhub.Hub // Indicates whether a reload of the CrowdSec service is required after executing the action plan. ReloadNeeded bool @@ -73,7 +73,7 @@ type ActionPlan struct { func NewActionPlan(hub *cwhub.Hub) *ActionPlan { return &ActionPlan{ - hub: hub, + hub: hub, commandsTracker: make(map[string]struct{}), } } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index bf029b0e835..75895dc729b 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -300,7 +300,7 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { crowdsecDaemon.Start() // wait for the appsec port to be available - if _, err := IsAlive(t.AppSecHost); err != nil { + if _, err = IsAlive(t.AppSecHost); err != nil { crowdsecLog, err2 := os.ReadFile(crowdsecLogFile) if err2 != nil { log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err) @@ -319,7 +319,7 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { } nucleiTargetHost := nucleiTargetParsedURL.Host - if _, err := IsAlive(nucleiTargetHost); err != nil { + if _, err = IsAlive(nucleiTargetHost); err != nil { return fmt.Errorf("target is down: %w", err) } @@ -631,8 +631,11 @@ func (t *HubTestItem) Run(patternDir string) error { if t.Config.LogFile != "" { return t.RunWithLogFile(patternDir) - } else if t.Config.NucleiTemplate != "" { + } + + if t.Config.NucleiTemplate != "" { return t.RunWithNucleiTemplate() } + return fmt.Errorf("log file or nuclei template must be set in '%s'", t.Name) } diff --git a/test/bats/04_nocapi.bats b/test/bats/04_nocapi.bats index d22a6f0a953..8d0018a9a4a 100644 --- a/test/bats/04_nocapi.bats +++ b/test/bats/04_nocapi.bats @@ -76,5 +76,5 @@ teardown() { rune -0 cscli metrics assert_output --partial "Route" assert_output --partial '/v1/watchers/login' - assert_output --partial "Local API Metrics:" + assert_output --partial "Local API Metrics" } diff --git a/test/bats/08_metrics.bats b/test/bats/08_metrics.bats index e260e667524..f3be9c60a95 100644 --- a/test/bats/08_metrics.bats +++ b/test/bats/08_metrics.bats @@ -66,7 +66,7 @@ teardown() { rune -0 cscli metrics assert_output --partial "Route" assert_output --partial '/v1/watchers/login' - assert_output --partial "Local API Metrics:" + assert_output --partial "Local API Metrics" rune -0 cscli metrics -o json rune -0 jq 'keys' <(output) @@ -93,7 +93,7 @@ teardown() { assert_stderr --partial "unknown metrics type: foobar" rune -0 cscli metrics show lapi - assert_output --partial "Local API Metrics:" + assert_output --partial "Local API Metrics" assert_output --regexp "Route.*Method.*Hits" assert_output --regexp "/v1/watchers/login.*POST" diff --git a/test/bats/08_metrics_bouncer.bats b/test/bats/08_metrics_bouncer.bats index c4dfebbab1d..5fb2c543bda 100644 --- a/test/bats/08_metrics_bouncer.bats +++ b/test/bats/08_metrics_bouncer.bats @@ -136,7 +136,10 @@ teardown() { rune -0 cscli metrics show bouncers assert_output - <<-EOT - Bouncer Metrics (testbouncer) since 2024-02-08 13:35:16 +0000 UTC: + +--------------------------+ + | Bouncer Metrics (testbou | + | ncer) since 2024-02-08 1 | + | 3:35:16 +0000 UTC | +--------+-----------------+ | Origin | foo | | | dogyear | pound | @@ -226,7 +229,8 @@ teardown() { rune -0 cscli metrics show bouncers assert_output - <<-EOT - Bouncer Metrics (testbouncer) since 2024-02-08 13:35:16 +0000 UTC: + +-------------------------------------------------------------------------------------------+ + | Bouncer Metrics (testbouncer) since 2024-02-08 13:35:16 +0000 UTC | +----------------------------------+------------------+-------------------+-----------------+ | Origin | active_decisions | dropped | foo | | | IPs | bytes | packets | dogyear | pound | @@ -309,7 +313,8 @@ teardown() { rune -0 cscli metrics show bouncers assert_output - <<-EOT - Bouncer Metrics (testbouncer) since 2024-02-08 13:35:16 +0000 UTC: + +-------------------------------------------------------------------------------------------+ + | Bouncer Metrics (testbouncer) since 2024-02-08 13:35:16 +0000 UTC | +----------------------------------+------------------+-------------------+-----------------+ | Origin | active_decisions | dropped | foo | | | IPs | bytes | packets | dogyear | pound | @@ -365,7 +370,9 @@ teardown() { rune -0 cscli metrics show bouncers assert_output - <<-EOT - Bouncer Metrics (testbouncer) since 2024-02-09 03:40:00 +0000 UTC: + +-----------------------------------------------+ + | Bouncer Metrics (testbouncer) since 2024-02-0 | + | 9 03:40:00 +0000 UTC | +--------------------------+--------+-----------+ | Origin | ima | notagauge | | | second | inch | @@ -417,7 +424,9 @@ teardown() { rune -0 cscli metrics show bouncers assert_output - <<-EOT - Bouncer Metrics (testbouncer) since 2024-02-09 03:40:00 +0000 UTC: + +---------------------------------------------+ + | Bouncer Metrics (testbouncer) since 2024-02 | + | -09 03:40:00 +0000 UTC | +--------------------------+------------------+ | Origin | active_decisions | | | IPs | @@ -502,7 +511,9 @@ teardown() { rune -0 cscli metrics show bouncers assert_output - <<-EOT - Bouncer Metrics (bouncer1) since 2024-02-08 13:35:16 +0000 UTC: + +--------------------------------------------------------------+ + | Bouncer Metrics (bouncer1) since 2024-02-08 13:35:16 +0000 U | + | TC | +----------------------------+---------+-----------------------+ | Origin | dropped | processed | | | bytes | bytes | packets | @@ -512,8 +523,9 @@ teardown() { +----------------------------+---------+-----------+-----------+ | Total | 1.80k | 12.34k | 100 | +----------------------------+---------+-----------+-----------+ - - Bouncer Metrics (bouncer2) since 2024-02-08 10:48:36 +0000 UTC: + +------------------------------------------------+ + | Bouncer Metrics (bouncer2) since 2024-02-08 10 | + | :48:36 +0000 UTC | +----------------------------+-------------------+ | Origin | dropped | | | bytes | packets | diff --git a/test/bats/80_alerts.bats b/test/bats/80_alerts.bats index 6d84c1a1fce..f01e918925c 100644 --- a/test/bats/80_alerts.bats +++ b/test/bats/80_alerts.bats @@ -89,7 +89,7 @@ teardown() { assert_line --regexp "^ - AS *: *$" assert_line --regexp "^ - Begin *: .*$" assert_line --regexp "^ - End *: .*$" - assert_line --regexp "^ - Active Decisions *:$" + assert_line --regexp "^\| Active Decisions *\|$" assert_line --regexp "^.* ID .* scope:value .* action .* expiration .* created_at .*$" assert_line --regexp "^.* Ip:10.20.30.40 .* ban .*$" diff --git a/test/bats/cscli-hubtype-list.bats b/test/bats/cscli-hubtype-list.bats index 14113650c74..b3db4743eb9 100644 --- a/test/bats/cscli-hubtype-list.bats +++ b/test/bats/cscli-hubtype-list.bats @@ -80,7 +80,7 @@ teardown() { # the list should be the same in all formats, and sorted (not case sensitive) list_raw=$(cscli parsers list -o raw -a | tail -n +2 | cut -d, -f1) - list_human=$(cscli parsers list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) + list_human=$(cscli parsers list -o human -a | tail -n +7 | head -n -1 | cut -d' ' -f2) list_json=$(cscli parsers list -o json -a | jq -r '.parsers[].name') # use python to sort because it handles "_" like go From 57d7247a43d2e45f9eb9bdfb73dfe6de5834e821 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 6 Jan 2025 23:55:39 +0100 Subject: [PATCH 05/25] cscli hub: refact/split files; add some doc/examples (#3394) --- cmd/crowdsec-cli/clihub/hub.go | 6 +- cmd/crowdsec-cli/cliitem/appsec.go | 123 ----- cmd/crowdsec-cli/cliitem/cmdinspect.go | 236 ++++++++++ cmd/crowdsec-cli/cliitem/cmdinstall.go | 150 ++++++ cmd/crowdsec-cli/cliitem/cmdremove.go | 151 ++++++ cmd/crowdsec-cli/cliitem/cmdupgrade.go | 106 +++++ cmd/crowdsec-cli/cliitem/collection.go | 41 -- cmd/crowdsec-cli/cliitem/context.go | 41 -- cmd/crowdsec-cli/cliitem/hubappsec.go | 255 ++++++++++ cmd/crowdsec-cli/cliitem/hubcollection.go | 105 +++++ cmd/crowdsec-cli/cliitem/hubcontext.go | 102 ++++ cmd/crowdsec-cli/cliitem/hubparser.go | 105 +++++ cmd/crowdsec-cli/cliitem/hubpostoverflow.go | 102 ++++ cmd/crowdsec-cli/cliitem/hubscenario.go | 78 +++- cmd/crowdsec-cli/cliitem/inspect.go | 57 --- cmd/crowdsec-cli/cliitem/item.go | 492 +------------------- cmd/crowdsec-cli/cliitem/parser.go | 41 -- cmd/crowdsec-cli/cliitem/postoverflow.go | 41 -- cmd/crowdsec-cli/cliitem/suggest.go | 77 --- 19 files changed, 1399 insertions(+), 910 deletions(-) delete mode 100644 cmd/crowdsec-cli/cliitem/appsec.go create mode 100644 cmd/crowdsec-cli/cliitem/cmdinspect.go create mode 100644 cmd/crowdsec-cli/cliitem/cmdinstall.go create mode 100644 cmd/crowdsec-cli/cliitem/cmdremove.go create mode 100644 cmd/crowdsec-cli/cliitem/cmdupgrade.go delete mode 100644 cmd/crowdsec-cli/cliitem/collection.go delete mode 100644 cmd/crowdsec-cli/cliitem/context.go create mode 100644 cmd/crowdsec-cli/cliitem/hubappsec.go create mode 100644 cmd/crowdsec-cli/cliitem/hubcollection.go create mode 100644 cmd/crowdsec-cli/cliitem/hubcontext.go create mode 100644 cmd/crowdsec-cli/cliitem/hubparser.go create mode 100644 cmd/crowdsec-cli/cliitem/hubpostoverflow.go delete mode 100644 cmd/crowdsec-cli/cliitem/inspect.go delete mode 100644 cmd/crowdsec-cli/cliitem/parser.go delete mode 100644 cmd/crowdsec-cli/cliitem/postoverflow.go delete mode 100644 cmd/crowdsec-cli/cliitem/suggest.go diff --git a/cmd/crowdsec-cli/clihub/hub.go b/cmd/crowdsec-cli/clihub/hub.go index 36e851d1b74..0d1e625f715 100644 --- a/cmd/crowdsec-cli/clihub/hub.go +++ b/cmd/crowdsec-cli/clihub/hub.go @@ -213,7 +213,11 @@ func (cli *cliHub) newUpgradeCmd() *cobra.Command { Long: ` Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available. `, - // TODO: Example + Example: `# Upgrade all the collections, scenarios etc. to the latest version in the downloaded index. Update data files too. +cscli hub upgrade + +# Upgrade tainted items as well; force re-download of data files. +cscli hub upgrade --force`, Args: cobra.NoArgs, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { diff --git a/cmd/crowdsec-cli/cliitem/appsec.go b/cmd/crowdsec-cli/cliitem/appsec.go deleted file mode 100644 index 44afa2133bd..00000000000 --- a/cmd/crowdsec-cli/cliitem/appsec.go +++ /dev/null @@ -1,123 +0,0 @@ -package cliitem - -import ( - "fmt" - "os" - - "golang.org/x/text/cases" - "golang.org/x/text/language" - "gopkg.in/yaml.v3" - - "github.com/crowdsecurity/crowdsec/pkg/appsec" - "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewAppsecConfig(cfg configGetter) *cliItem { - return &cliItem{ - cfg: cfg, - name: cwhub.APPSEC_CONFIGS, - singular: "appsec-config", - oneOrMore: "appsec-config(s)", - help: cliHelp{ - example: `cscli appsec-configs list -a -cscli appsec-configs install crowdsecurity/vpatch -cscli appsec-configs inspect crowdsecurity/vpatch -cscli appsec-configs upgrade crowdsecurity/vpatch -cscli appsec-configs remove crowdsecurity/vpatch -`, - }, - installHelp: cliHelp{ - example: `cscli appsec-configs install crowdsecurity/vpatch`, - }, - removeHelp: cliHelp{ - example: `cscli appsec-configs remove crowdsecurity/vpatch`, - }, - upgradeHelp: cliHelp{ - example: `cscli appsec-configs upgrade crowdsecurity/vpatch`, - }, - inspectHelp: cliHelp{ - example: `cscli appsec-configs inspect crowdsecurity/vpatch`, - }, - listHelp: cliHelp{ - example: `cscli appsec-configs list -cscli appsec-configs list -a -cscli appsec-configs list crowdsecurity/vpatch`, - }, - } -} - -func NewAppsecRule(cfg configGetter) *cliItem { - inspectDetail := func(item *cwhub.Item) error { - // Only show the converted rules in human mode - if cfg().Cscli.Output != "human" { - return nil - } - - appsecRule := appsec.AppsecCollectionConfig{} - - yamlContent, err := os.ReadFile(item.State.LocalPath) - if err != nil { - return fmt.Errorf("unable to read file %s: %w", item.State.LocalPath, err) - } - - if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil { - return fmt.Errorf("unable to parse yaml file %s: %w", item.State.LocalPath, err) - } - - for _, ruleType := range appsec_rule.SupportedTypes() { - fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType)) - - for _, rule := range appsecRule.Rules { - convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name) - if err != nil { - return fmt.Errorf("unable to convert rule %s: %w", rule.Name, err) - } - - fmt.Println(convertedRule) - } - - switch ruleType { //nolint:gocritic - case appsec_rule.ModsecurityRuleType: - for _, rule := range appsecRule.SecLangRules { - fmt.Println(rule) - } - } - } - - return nil - } - - return &cliItem{ - cfg: cfg, - name: "appsec-rules", - singular: "appsec-rule", - oneOrMore: "appsec-rule(s)", - help: cliHelp{ - example: `cscli appsec-rules list -a -cscli appsec-rules install crowdsecurity/crs -cscli appsec-rules inspect crowdsecurity/crs -cscli appsec-rules upgrade crowdsecurity/crs -cscli appsec-rules remove crowdsecurity/crs -`, - }, - installHelp: cliHelp{ - example: `cscli appsec-rules install crowdsecurity/crs`, - }, - removeHelp: cliHelp{ - example: `cscli appsec-rules remove crowdsecurity/crs`, - }, - upgradeHelp: cliHelp{ - example: `cscli appsec-rules upgrade crowdsecurity/crs`, - }, - inspectHelp: cliHelp{ - example: `cscli appsec-rules inspect crowdsecurity/crs`, - }, - inspectDetail: inspectDetail, - listHelp: cliHelp{ - example: `cscli appsec-rules list -cscli appsec-rules list -a -cscli appsec-rules list crowdsecurity/crs`, - }, - } -} diff --git a/cmd/crowdsec-cli/cliitem/cmdinspect.go b/cmd/crowdsec-cli/cliitem/cmdinspect.go new file mode 100644 index 00000000000..b5ee0816d72 --- /dev/null +++ b/cmd/crowdsec-cli/cliitem/cmdinspect.go @@ -0,0 +1,236 @@ +package cliitem + +import ( + "cmp" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/hexops/gotextdiff" + "github.com/hexops/gotextdiff/myers" + "github.com/hexops/gotextdiff/span" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff bool, rev bool, noMetrics bool) error { + cfg := cli.cfg() + + if rev && !diff { + return errors.New("--rev can only be used with --diff") + } + + if url != "" { + cfg.Cscli.PrometheusUrl = url + } + + var contentProvider cwhub.ContentProvider + + if diff { + contentProvider = require.HubDownloader(ctx, cfg) + } + + hub, err := require.Hub(cfg, log.StandardLogger()) + if err != nil { + return err + } + + for _, name := range args { + item := hub.GetItem(cli.name, name) + if item == nil { + return fmt.Errorf("can't find '%s' in %s", name, cli.name) + } + + if diff { + fmt.Println(cli.whyTainted(ctx, hub, contentProvider, item, rev)) + + continue + } + + if err = inspectItem(hub, item, !noMetrics, cfg.Cscli.Output, cfg.Cscli.PrometheusUrl, cfg.Cscli.Color); err != nil { + return err + } + + if cli.inspectDetail != nil { + if err = cli.inspectDetail(item); err != nil { + return err + } + } + } + + return nil +} + +// return the diff between the installed version and the latest version +func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, contentProvider cwhub.ContentProvider, reverse bool) (string, error) { + if !item.State.Installed { + return "", fmt.Errorf("'%s' is not installed", item.FQName()) + } + + dest, err := os.CreateTemp("", "cscli-diff-*") + if err != nil { + return "", fmt.Errorf("while creating temporary file: %w", err) + } + defer os.Remove(dest.Name()) + + _, remoteURL, err := item.FetchContentTo(ctx, contentProvider, dest.Name()) + if err != nil { + return "", err + } + + latestContent, err := os.ReadFile(dest.Name()) + if err != nil { + return "", fmt.Errorf("while reading %s: %w", dest.Name(), err) + } + + localContent, err := os.ReadFile(item.State.LocalPath) + if err != nil { + return "", fmt.Errorf("while reading %s: %w", item.State.LocalPath, err) + } + + file1 := item.State.LocalPath + file2 := remoteURL + content1 := string(localContent) + content2 := string(latestContent) + + if reverse { + file1, file2 = file2, file1 + content1, content2 = content2, content1 + } + + edits := myers.ComputeEdits(span.URIFromPath(file1), content1, content2) + diff := gotextdiff.ToUnified(file1, file2, content1, edits) + + return fmt.Sprintf("%s", diff), nil +} + +func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub.ContentProvider, item *cwhub.Item, reverse bool) string { + if !item.State.Installed { + return fmt.Sprintf("# %s is not installed", item.FQName()) + } + + if !item.State.Tainted { + return fmt.Sprintf("# %s is not tainted", item.FQName()) + } + + if len(item.State.TaintedBy) == 0 { + return fmt.Sprintf("# %s is tainted but we don't know why. please report this as a bug", item.FQName()) + } + + ret := []string{ + fmt.Sprintf("# Let's see why %s is tainted.", item.FQName()), + } + + for _, fqsub := range item.State.TaintedBy { + ret = append(ret, fmt.Sprintf("\n-> %s\n", fqsub)) + + sub, err := hub.GetItemFQ(fqsub) + if err != nil { + ret = append(ret, err.Error()) + } + + diff, err := cli.itemDiff(ctx, sub, contentProvider, reverse) + if err != nil { + ret = append(ret, err.Error()) + } + + if diff != "" { + ret = append(ret, diff) + } else if len(sub.State.TaintedBy) > 0 { + taintList := strings.Join(sub.State.TaintedBy, ", ") + if sub.FQName() == taintList { + // hack: avoid message "item is tainted by itself" + continue + } + + ret = append(ret, fmt.Sprintf("# %s is tainted by %s", sub.FQName(), taintList)) + } + } + + return strings.Join(ret, "\n") +} + +func (cli cliItem) newInspectCmd() *cobra.Command { + var ( + url string + diff bool + rev bool + noMetrics bool + ) + + cmd := &cobra.Command{ + Use: cmp.Or(cli.inspectHelp.use, "inspect [item]..."), + Short: cmp.Or(cli.inspectHelp.short, "Inspect given "+cli.oneOrMore), + Long: cmp.Or(cli.inspectHelp.long, "Inspect the state of one or more "+cli.name), + Example: cli.inspectHelp.example, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, + ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(cli.name, args, toComplete, cli.cfg) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return cli.inspect(cmd.Context(), args, url, diff, rev, noMetrics) + }, + } + + flags := cmd.Flags() + flags.StringVarP(&url, "url", "u", "", "Prometheus url") + flags.BoolVar(&diff, "diff", false, "Show diff with latest version (for tainted items)") + flags.BoolVar(&rev, "rev", false, "Reverse diff output") + flags.BoolVar(&noMetrics, "no-metrics", false, "Don't show metrics (when cscli.output=human)") + + return cmd +} + +func inspectItem(hub *cwhub.Hub, item *cwhub.Item, wantMetrics bool, output string, prometheusURL string, wantColor string) error { + // This is dirty... + // We want to show current dependencies (from content), not latest (from index). + // The item is modifed but after this function the whole hub should be thrown away. + // A cleaner way would be to copy the struct first. + item.Dependencies = item.CurrentDependencies() + + switch output { + case "human", "raw": + enc := yaml.NewEncoder(os.Stdout) + enc.SetIndent(2) + + if err := enc.Encode(item); err != nil { + return fmt.Errorf("unable to encode item: %w", err) + } + case "json": + b, err := json.MarshalIndent(*item, "", " ") + if err != nil { + return fmt.Errorf("unable to serialize item: %w", err) + } + + fmt.Print(string(b)) + } + + if output != "human" { + return nil + } + + if item.State.Tainted { + fmt.Println() + fmt.Printf(`This item is tainted. Use "%s %s inspect --diff %s" to see why.`, filepath.Base(os.Args[0]), item.Type, item.Name) + fmt.Println() + } + + if wantMetrics { + fmt.Printf("\nCurrent metrics: \n") + + if err := showMetrics(prometheusURL, hub, item, wantColor); err != nil { + return err + } + } + + return nil +} diff --git a/cmd/crowdsec-cli/cliitem/cmdinstall.go b/cmd/crowdsec-cli/cliitem/cmdinstall.go new file mode 100644 index 00000000000..daddbe84a4b --- /dev/null +++ b/cmd/crowdsec-cli/cliitem/cmdinstall.go @@ -0,0 +1,150 @@ +package cliitem + +import ( + "cmp" + "context" + "errors" + "fmt" + "slices" + "strings" + + "github.com/agext/levenshtein" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/reload" + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/hubops" +) + +// suggestNearestMessage returns a message with the most similar item name, if one is found +func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string { + const maxDistance = 7 + + score := 100 + nearest := "" + + for _, item := range hub.GetItemsByType(itemType, false) { + d := levenshtein.Distance(itemName, item.Name, nil) + if d < score { + score = d + nearest = item.Name + } + } + + msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType) + + if score < maxDistance { + msg += fmt.Sprintf(", did you mean '%s'?", nearest) + } + + return msg +} + +func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun bool, downloadOnly bool, force bool, ignoreError bool) error { + cfg := cli.cfg() + + hub, err := require.Hub(cfg, log.StandardLogger()) + if err != nil { + return err + } + + plan := hubops.NewActionPlan(hub) + + contentProvider := require.HubDownloader(ctx, cfg) + + for _, name := range args { + item := hub.GetItem(cli.name, name) + if item == nil { + msg := suggestNearestMessage(hub, cli.name, name) + if !ignoreError { + return errors.New(msg) + } + + log.Error(msg) + + continue + } + + if err = plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil { + return err + } + + if !downloadOnly { + if err = plan.AddCommand(hubops.NewEnableCommand(item, force)); err != nil { + return err + } + } + } + + verbose := (cfg.Cscli.Output == "raw") + + if err := plan.Execute(ctx, yes, dryRun, verbose); err != nil { + if !ignoreError { + return err + } + + log.Error(err) + } + + if plan.ReloadNeeded { + fmt.Println("\n" + reload.Message) + } + + return nil +} + +func compAllItems(itemType string, args []string, toComplete string, cfg configGetter) ([]string, cobra.ShellCompDirective) { + hub, err := require.Hub(cfg(), nil) + if err != nil { + return nil, cobra.ShellCompDirectiveDefault + } + + comp := make([]string, 0) + + for _, item := range hub.GetItemsByType(itemType, false) { + if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) { + comp = append(comp, item.Name) + } + } + + cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) + + return comp, cobra.ShellCompDirectiveNoFileComp +} + +func (cli cliItem) newInstallCmd() *cobra.Command { + var ( + yes bool + dryRun bool + downloadOnly bool + force bool + ignoreError bool + ) + + cmd := &cobra.Command{ + Use: cmp.Or(cli.installHelp.use, "install [item]..."), + Short: cmp.Or(cli.installHelp.short, "Install given "+cli.oneOrMore), + Long: cmp.Or(cli.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", cli.name)), + Example: cli.installHelp.example, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, + ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compAllItems(cli.name, args, toComplete, cli.cfg) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return cli.install(cmd.Context(), args, yes, dryRun, downloadOnly, force, ignoreError) + }, + } + + flags := cmd.Flags() + flags.BoolVarP(&yes, "yes", "y", false, "Confirm execution without prompt") + flags.BoolVar(&dryRun, "dry-run", false, "Don't install or remove anything; print the execution plan") + flags.BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") + flags.BoolVar(&force, "force", false, "Force install: overwrite tainted and outdated files") + flags.BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple "+cli.name) + cmd.MarkFlagsMutuallyExclusive("yes", "dry-run") + + return cmd +} diff --git a/cmd/crowdsec-cli/cliitem/cmdremove.go b/cmd/crowdsec-cli/cliitem/cmdremove.go new file mode 100644 index 00000000000..ac9410c047d --- /dev/null +++ b/cmd/crowdsec-cli/cliitem/cmdremove.go @@ -0,0 +1,151 @@ +package cliitem + +import ( + "cmp" + "context" + "errors" + "fmt" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/reload" + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/hubops" +) + +func (cli cliItem) removePlan(hub *cwhub.Hub, args []string, purge bool, force bool, all bool) (*hubops.ActionPlan, error) { + plan := hubops.NewActionPlan(hub) + + if all { + itemGetter := hub.GetInstalledByType + if purge { + itemGetter = hub.GetItemsByType + } + + for _, item := range itemGetter(cli.name, true) { + if err := plan.AddCommand(hubops.NewDisableCommand(item, force)); err != nil { + return nil, err + } + + if purge { + if err := plan.AddCommand(hubops.NewPurgeCommand(item, force)); err != nil { + return nil, err + } + } + } + + return plan, nil + } + + if len(args) == 0 { + return nil, fmt.Errorf("specify at least one %s to remove or '--all'", cli.singular) + } + + for _, itemName := range args { + item := hub.GetItem(cli.name, itemName) + if item == nil { + return nil, fmt.Errorf("can't find '%s' in %s", itemName, cli.name) + } + + parents := installedParentNames(item) + + if !force && len(parents) > 0 { + log.Warningf("%s belongs to collections: %s", item.Name, parents) + log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, cli.singular) + + continue + } + + if err := plan.AddCommand(hubops.NewDisableCommand(item, force)); err != nil { + return nil, err + } + + if purge { + if err := plan.AddCommand(hubops.NewPurgeCommand(item, force)); err != nil { + return nil, err + } + } + } + + return plan, nil +} + +// return the names of the installed parents of an item, used to check if we can remove it +func installedParentNames(item *cwhub.Item) []string { + ret := make([]string, 0) + + for _, parent := range item.Ancestors() { + if parent.State.Installed { + ret = append(ret, parent.Name) + } + } + + return ret +} + +func (cli cliItem) remove(ctx context.Context, args []string, yes bool, dryRun bool, purge bool, force bool, all bool) error { + cfg := cli.cfg() + + hub, err := require.Hub(cli.cfg(), log.StandardLogger()) + if err != nil { + return err + } + + plan, err := cli.removePlan(hub, args, purge, force, all) + if err != nil { + return err + } + + verbose := (cfg.Cscli.Output == "raw") + + if err := plan.Execute(ctx, yes, dryRun, verbose); err != nil { + return err + } + + if plan.ReloadNeeded { + fmt.Println("\n" + reload.Message) + } + + return nil +} + +func (cli cliItem) newRemoveCmd() *cobra.Command { + var ( + yes bool + dryRun bool + purge bool + force bool + all bool + ) + + cmd := &cobra.Command{ + Use: cmp.Or(cli.removeHelp.use, "remove [item]..."), + Short: cmp.Or(cli.removeHelp.short, "Remove given "+cli.oneOrMore), + Long: cmp.Or(cli.removeHelp.long, "Remove one or more "+cli.name), + Example: cli.removeHelp.example, + Aliases: []string{"delete"}, + DisableAutoGenTag: true, + ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(cli.name, args, toComplete, cli.cfg) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) > 0 && all { + return errors.New("can't specify items and '--all' at the same time") + } + + return cli.remove(cmd.Context(), args, yes, dryRun, purge, force, all) + }, + } + + flags := cmd.Flags() + flags.BoolVarP(&yes, "yes", "y", false, "Confirm execution without prompt") + flags.BoolVar(&dryRun, "dry-run", false, "Don't install or remove anything; print the execution plan") + flags.BoolVar(&purge, "purge", false, "Delete source file too") + flags.BoolVar(&force, "force", false, "Force remove: remove tainted and outdated files") + flags.BoolVar(&all, "all", false, "Remove all the "+cli.name) + cmd.MarkFlagsMutuallyExclusive("yes", "dry-run") + + return cmd +} diff --git a/cmd/crowdsec-cli/cliitem/cmdupgrade.go b/cmd/crowdsec-cli/cliitem/cmdupgrade.go new file mode 100644 index 00000000000..1ddd07485d4 --- /dev/null +++ b/cmd/crowdsec-cli/cliitem/cmdupgrade.go @@ -0,0 +1,106 @@ +package cliitem + +import ( + "cmp" + "context" + "fmt" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/reload" + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/hubops" +) + +func (cli cliItem) upgradePlan(hub *cwhub.Hub, contentProvider cwhub.ContentProvider, args []string, force bool, all bool) (*hubops.ActionPlan, error) { + plan := hubops.NewActionPlan(hub) + + if all { + for _, item := range hub.GetInstalledByType(cli.name, true) { + if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil { + return nil, err + } + } + + return plan, nil + } + + if len(args) == 0 { + return nil, fmt.Errorf("specify at least one %s to upgrade or '--all'", cli.singular) + } + + for _, itemName := range args { + item := hub.GetItem(cli.name, itemName) + if item == nil { + return nil, fmt.Errorf("can't find '%s' in %s", itemName, cli.name) + } + + if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil { + return nil, err + } + } + + return plan, nil +} + +func (cli cliItem) upgrade(ctx context.Context, args []string, yes bool, dryRun bool, force bool, all bool) error { + cfg := cli.cfg() + + hub, err := require.Hub(cfg, log.StandardLogger()) + if err != nil { + return err + } + + contentProvider := require.HubDownloader(ctx, cfg) + + plan, err := cli.upgradePlan(hub, contentProvider, args, force, all) + if err != nil { + return err + } + + verbose := (cfg.Cscli.Output == "raw") + + if err := plan.Execute(ctx, yes, dryRun, verbose); err != nil { + return err + } + + if plan.ReloadNeeded { + fmt.Println("\n" + reload.Message) + } + + return nil +} + +func (cli cliItem) newUpgradeCmd() *cobra.Command { + var ( + yes bool + dryRun bool + all bool + force bool + ) + + cmd := &cobra.Command{ + Use: cmp.Or(cli.upgradeHelp.use, "upgrade [item]..."), + Short: cmp.Or(cli.upgradeHelp.short, "Upgrade given "+cli.oneOrMore), + Long: cmp.Or(cli.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", cli.name)), + Example: cli.upgradeHelp.example, + DisableAutoGenTag: true, + ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(cli.name, args, toComplete, cli.cfg) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return cli.upgrade(cmd.Context(), args, yes, dryRun, force, all) + }, + } + + flags := cmd.Flags() + flags.BoolVarP(&yes, "yes", "y", false, "Confirm execution without prompt") + flags.BoolVar(&dryRun, "dry-run", false, "Don't install or remove anything; print the execution plan") + flags.BoolVarP(&all, "all", "a", false, "Upgrade all the "+cli.name) + flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated files") + cmd.MarkFlagsMutuallyExclusive("yes", "dry-run") + + return cmd +} diff --git a/cmd/crowdsec-cli/cliitem/collection.go b/cmd/crowdsec-cli/cliitem/collection.go deleted file mode 100644 index ea91c1e537a..00000000000 --- a/cmd/crowdsec-cli/cliitem/collection.go +++ /dev/null @@ -1,41 +0,0 @@ -package cliitem - -import ( - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewCollection(cfg configGetter) *cliItem { - return &cliItem{ - cfg: cfg, - name: cwhub.COLLECTIONS, - singular: "collection", - oneOrMore: "collection(s)", - help: cliHelp{ - example: `cscli collections list -a -cscli collections install crowdsecurity/http-cve crowdsecurity/iptables -cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables -cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables -cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables -`, - }, - installHelp: cliHelp{ - example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`, - }, - removeHelp: cliHelp{ - example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`, - }, - upgradeHelp: cliHelp{ - example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`, - }, - inspectHelp: cliHelp{ - example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`, - }, - listHelp: cliHelp{ - example: `cscli collections list -cscli collections list -a -cscli collections list crowdsecurity/http-cve crowdsecurity/iptables - -List only enabled collections unless "-a" or names are specified.`, - }, - } -} diff --git a/cmd/crowdsec-cli/cliitem/context.go b/cmd/crowdsec-cli/cliitem/context.go deleted file mode 100644 index 7d110b8203d..00000000000 --- a/cmd/crowdsec-cli/cliitem/context.go +++ /dev/null @@ -1,41 +0,0 @@ -package cliitem - -import ( - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewContext(cfg configGetter) *cliItem { - return &cliItem{ - cfg: cfg, - name: cwhub.CONTEXTS, - singular: "context", - oneOrMore: "context(s)", - help: cliHelp{ - example: `cscli contexts list -a -cscli contexts install crowdsecurity/yyy crowdsecurity/zzz -cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz -cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz -cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz -`, - }, - installHelp: cliHelp{ - example: `cscli contexts install crowdsecurity/yyy crowdsecurity/zzz`, - }, - removeHelp: cliHelp{ - example: `cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz`, - }, - upgradeHelp: cliHelp{ - example: `cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz`, - }, - inspectHelp: cliHelp{ - example: `cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz`, - }, - listHelp: cliHelp{ - example: `cscli contexts list -cscli contexts list -a -cscli contexts list crowdsecurity/yyy crowdsecurity/zzz - -List only enabled contexts unless "-a" or names are specified.`, - }, - } -} diff --git a/cmd/crowdsec-cli/cliitem/hubappsec.go b/cmd/crowdsec-cli/cliitem/hubappsec.go new file mode 100644 index 00000000000..7f9143d35b8 --- /dev/null +++ b/cmd/crowdsec-cli/cliitem/hubappsec.go @@ -0,0 +1,255 @@ +package cliitem + +import ( + "fmt" + "os" + + "golang.org/x/text/cases" + "golang.org/x/text/language" + "gopkg.in/yaml.v3" + + "github.com/crowdsecurity/crowdsec/pkg/appsec" + "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewAppsecConfig(cfg configGetter) *cliItem { + return &cliItem{ + cfg: cfg, + name: cwhub.APPSEC_CONFIGS, + singular: "appsec-config", + oneOrMore: "appsec-config(s)", + help: cliHelp{ + example: `cscli appsec-configs list -a +cscli appsec-configs install crowdsecurity/virtual-patching +cscli appsec-configs inspect crowdsecurity/virtual-patching +cscli appsec-configs upgrade crowdsecurity/virtual-patching +cscli appsec-configs remove crowdsecurity/virtual-patching +`, + }, + installHelp: cliHelp{ + example: `# Install some appsec-configs. +cscli appsec-configs install crowdsecurity/virtual-patching + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli appsec-configs install crowdsecurity/virtual-patching --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli appsec-configs install crowdsecurity/virtual-patching --dry-run -o raw + +# Download only, to be installed later. +cscli appsec-configs install crowdsecurity/virtual-patching --download-only + +# Install over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli appsec-configs install crowdsecurity/virtual-patching --force + +# Proceed without prompting. +cscli appsec-configs install crowdsecurity/virtual-patching --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + removeHelp: cliHelp{ + example: `# Uninstall some appsec-configs. +cscli appsec-configs remove crowdsecurity/virtual-patching + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli appsec-configs remove crowdsecurity/virtual-patching --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli appsec-configs remove crowdsecurity/virtual-patching --dry-run -o raw + +# Uninstall and also remove the downloaded files. +cscli appsec-configs remove crowdsecurity/virtual-patching --purge + +# Remove tainted items. +cscli appsec-configs remove crowdsecurity/virtual-patching --force + +# Proceed without prompting. +cscli appsec-configs remove crowdsecurity/virtual-patching --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + upgradeHelp: cliHelp{ + example: `# Upgrade some appsec-configs. If they are not currently installed, they are downloaded but not installed. +cscli appsec-configs upgrade crowdsecurity/virtual-patching + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli appsec-configs upgrade crowdsecurity/virtual-patching --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli appsec-configs upgrade crowdsecurity/virtual-patching --dry-run -o raw + +# Upgrade over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli appsec-configs upgrade crowdsecurity/virtual-patching --force + +# Proceed without prompting. +cscli appsec-configs upgrade crowdsecurity/virtual-patching --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + inspectHelp: cliHelp{ + example: `# Display metadata, state, metrics and ancestor collections of appsec-configs (installed or not). +cscli appsec-configs inspect crowdsecurity/virtual-patching + +# Don't collect metrics (avoid error if crowdsec is not running). +cscli appsec-configs inspect crowdsecurity/virtual-patching --no-metrics + +# Display difference between a tainted item and the latest one. +cscli appsec-configs inspect crowdsecurity/virtual-patching --diff + +# Reverse the above diff +cscli appsec-configs inspect crowdsecurity/virtual-patching --diff --rev`, + }, + listHelp: cliHelp{ + example: `# List enabled (installed) appsec-configs. +cscli appsec-configs list + +# List all available appsec-configs (installed or not). +cscli appsec-configs list -a + +# List specific appsec-configs (installed or not). +cscli appsec-configs list crowdsecurity/virtual-patching crowdsecurity/generic-rules`, + }, + } +} + +func NewAppsecRule(cfg configGetter) *cliItem { + inspectDetail := func(item *cwhub.Item) error { + // Only show the converted rules in human mode + if cfg().Cscli.Output != "human" { + return nil + } + + appsecRule := appsec.AppsecCollectionConfig{} + + yamlContent, err := os.ReadFile(item.State.LocalPath) + if err != nil { + return fmt.Errorf("unable to read file %s: %w", item.State.LocalPath, err) + } + + if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil { + return fmt.Errorf("unable to parse yaml file %s: %w", item.State.LocalPath, err) + } + + for _, ruleType := range appsec_rule.SupportedTypes() { + fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType)) + + for _, rule := range appsecRule.Rules { + convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name) + if err != nil { + return fmt.Errorf("unable to convert rule %s: %w", rule.Name, err) + } + + fmt.Println(convertedRule) + } + + switch ruleType { //nolint:gocritic + case appsec_rule.ModsecurityRuleType: + for _, rule := range appsecRule.SecLangRules { + fmt.Println(rule) + } + } + } + + return nil + } + + return &cliItem{ + cfg: cfg, + name: "appsec-rules", + singular: "appsec-rule", + oneOrMore: "appsec-rule(s)", + help: cliHelp{ + example: `cscli appsec-rules list -a +cscli appsec-rules install crowdsecurity/crs +cscli appsec-rules inspect crowdsecurity/crs +cscli appsec-rules upgrade crowdsecurity/crs +cscli appsec-rules remove crowdsecurity/crs +`, + }, + installHelp: cliHelp{ + example: `# Install some appsec-rules. +cscli appsec-rules install crowdsecurity/crs + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli appsec-rules install crowdsecurity/crs --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli appsec-rules install crowdsecurity/crs --dry-run -o raw + +# Download only, to be installed later. +cscli appsec-rules install crowdsecurity/crs --download-only + +# Install over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli appsec-rules install crowdsecurity/crs --force + +# Proceed without prompting. +cscli appsec-rules install crowdsecurity/crs --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + removeHelp: cliHelp{ + example: `# Uninstall some appsec-rules. +cscli appsec-rules remove crowdsecurity/crs + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli appsec-rules remove crowdsecurity/crs --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli appsec-rules remove crowdsecurity/crs --dry-run -o raw + +# Uninstall and also remove the downloaded files. +cscli appsec-rules remove crowdsecurity/crs --purge + +# Remove tainted items. +cscli appsec-rules remove crowdsecurity/crs --force + +# Proceed without prompting. +cscli appsec-rules remove crowdsecurity/crs --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + upgradeHelp: cliHelp{ + example: `# Upgrade some appsec-rules. If they are not currently installed, they are downloaded but not installed. +cscli appsec-rules upgrade crowdsecurity/crs + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli appsec-rules upgrade crowdsecurity/crs --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli appsec-rules upgrade crowdsecurity/crs --dry-run -o raw + +# Upgrade over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli appsec-rules upgrade crowdsecurity/crs --force + +# Proceed without prompting. +cscli appsec-rules upgrade crowdsecurity/crs --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + inspectHelp: cliHelp{ + example: `# Display metadata, state, metrics and ancestor collections of appsec-rules (installed or not). +cscli appsec-rules inspect crowdsecurity/crs + +# Don't collect metrics (avoid error if crowdsec is not running). +cscli appsec-configs inspect crowdsecurity/crs --no-metrics + +# Display difference between a tainted item and the latest one. +cscli appsec-rules inspect crowdsecurity/crs --diff + +# Reverse the above diff +cscli appsec-rules inspect crowdsecurity/crs --diff --rev`, + }, + inspectDetail: inspectDetail, + listHelp: cliHelp{ + example: `# List enabled (installed) appsec-rules. +cscli appsec-rules list + +# List all available appsec-rules (installed or not). +cscli appsec-rules list -a + +# List specific appsec-rules (installed or not). +cscli appsec-rules list crowdsecurity/crs crowdsecurity/vpatch-git-config`, + }, + } +} diff --git a/cmd/crowdsec-cli/cliitem/hubcollection.go b/cmd/crowdsec-cli/cliitem/hubcollection.go new file mode 100644 index 00000000000..b45f956e0ac --- /dev/null +++ b/cmd/crowdsec-cli/cliitem/hubcollection.go @@ -0,0 +1,105 @@ +package cliitem + +import ( + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewCollection(cfg configGetter) *cliItem { + return &cliItem{ + cfg: cfg, + name: cwhub.COLLECTIONS, + singular: "collection", + oneOrMore: "collection(s)", + help: cliHelp{ + example: `cscli collections list -a +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables +cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables +`, + }, + installHelp: cliHelp{ + example: `# Install some collections. +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables --dry-run -o raw + +# Download only, to be installed later. +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables --download-only + +# Install over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables --force + +# Proceed without prompting. +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + removeHelp: cliHelp{ + example: `# Uninstall some collections. +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables --dry-run -o raw + +# Uninstall and also remove the downloaded files. +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables --purge + +# Remove tainted items. +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables --force + +# Proceed without prompting. +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + upgradeHelp: cliHelp{ + example: `# Upgrade some collections. If they are not currently installed, they are downloaded but not installed. +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables --dry-run -o raw + +# Upgrade over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables --force + +# Proceed without prompting. +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + inspectHelp: cliHelp{ + example: `# Display metadata, state, metrics and dependencies of collections (installed or not). +cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables + +# Don't collect metrics (avoid error if crowdsec is not running). +cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables --no-metrics + +# Display difference between a tainted item and the latest one, or the reason for the taint if it's a dependency. +cscli collections inspect crowdsecurity/http-cve --diff + +# Reverse the above diff +cscli collections inspect crowdsecurity/http-cve --diff --rev`, + }, + listHelp: cliHelp{ + example: `# List enabled (installed) collections. +cscli collections list + +# List all available collections (installed or not). +cscli collections list -a + +# List specific collections (installed or not). +cscli collections list crowdsecurity/http-cve crowdsecurity/iptables`, + }, + } +} diff --git a/cmd/crowdsec-cli/cliitem/hubcontext.go b/cmd/crowdsec-cli/cliitem/hubcontext.go new file mode 100644 index 00000000000..3a94687843d --- /dev/null +++ b/cmd/crowdsec-cli/cliitem/hubcontext.go @@ -0,0 +1,102 @@ +package cliitem + +import ( + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewContext(cfg configGetter) *cliItem { + return &cliItem{ + cfg: cfg, + name: cwhub.CONTEXTS, + singular: "context", + oneOrMore: "context(s)", + help: cliHelp{ + example: `cscli contexts list -a +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet +cscli contexts inspect crowdsecurity/bf_base crowdsecurity/fortinet +cscli contexts upgrade crowdsecurity/bf_base crowdsecurity/fortinet +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet +`, + }, + installHelp: cliHelp{ + example: `# Install some contexts. +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet --dry-run -o raw + +# Download only, to be installed later. +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet --download-only + +# Install over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet --force + +# Proceed without prompting. +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + removeHelp: cliHelp{ + example: `# Uninstall some contexts. +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet --dry-run -o raw + +# Uninstall and also remove the downloaded files. +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet --purge + +# Remove tainted items. +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet --force + +# Proceed without prompting. +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + upgradeHelp: cliHelp{ + example: `# Upgrade some contexts. If they are not currently installed, they are downloaded but not installed. +cscli contexts upgrade crowdsecurity/bf_base crowdsecurity/fortinet + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli contexts upgrade crowdsecurity/bf_base crowdsecurity/fortinet --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli contexts upgrade crowdsecurity/bf_base crowdsecurity/fortinet --dry-run -o raw + +# Upgrade over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli contexts upgrade crowdsecurity/bf_base crowdsecurity/fortinet --force + +# Proceed without prompting. +cscli contexts upgrade crowdsecurity/bf_base crowdsecurity/fortinet --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + inspectHelp: cliHelp{ + example: `# Display metadata, state and ancestor collections of contexts (installed or not). +cscli contexts inspect crowdsecurity/bf_base crowdsecurity/fortinet + +# Display difference between a tainted item and the latest one. +cscli contexts inspect crowdsecurity/bf_base --diff + +# Reverse the above diff +cscli contexts inspect crowdsecurity/bf_base --diff --rev`, + }, + listHelp: cliHelp{ + example: `# List enabled (installed) contexts. +cscli contexts list + +# List all available contexts (installed or not). +cscli contexts list -a + +# List specific contexts (installed or not). +cscli contexts list crowdsecurity/bf_base crowdsecurity/fortinet`, + }, + } +} diff --git a/cmd/crowdsec-cli/cliitem/hubparser.go b/cmd/crowdsec-cli/cliitem/hubparser.go new file mode 100644 index 00000000000..440cb61204f --- /dev/null +++ b/cmd/crowdsec-cli/cliitem/hubparser.go @@ -0,0 +1,105 @@ +package cliitem + +import ( + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewParser(cfg configGetter) *cliItem { + return &cliItem{ + cfg: cfg, + name: cwhub.PARSERS, + singular: "parser", + oneOrMore: "parser(s)", + help: cliHelp{ + example: `cscli parsers list -a +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs +`, + }, + installHelp: cliHelp{ + example: `# Install some parsers. +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs --dry-run -o raw + +# Download only, to be installed later. +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs --download-only + +# Install over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs --force + +# Proceed without prompting. +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + removeHelp: cliHelp{ + example: `# Uninstall some parsers. +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs --dry-run -o raw + +# Uninstall and also remove the downloaded files. +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs --purge + +# Remove tainted items. +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs --force + +# Proceed without prompting. +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + upgradeHelp: cliHelp{ + example: `# Upgrade some parsers. If they are not currently installed, they are downloaded but not installed. +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs --dry-run -o raw + +# Upgrade over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs --force + +# Proceed without prompting. +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + inspectHelp: cliHelp{ + example: `# Display metadata, state, metrics and ancestor collections of parsers (installed or not). +cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs + +# Don't collect metrics (avoid error if crowdsec is not running). +cscli parsers inspect crowdsecurity/httpd-logs --no-metrics + +# Display difference between a tainted item and the latest one. +cscli parsers inspect crowdsecurity/httpd-logs --diff + +# Reverse the above diff +cscli parsers inspect crowdsecurity/httpd-logs --diff --rev`, + }, + listHelp: cliHelp{ + example: `# List enabled (installed) parsers. +cscli parsers list + +# List all available parsers (installed or not). +cscli parsers list -a + +# List specific parsers (installed or not). +cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + } +} diff --git a/cmd/crowdsec-cli/cliitem/hubpostoverflow.go b/cmd/crowdsec-cli/cliitem/hubpostoverflow.go new file mode 100644 index 00000000000..cfd5f7c95aa --- /dev/null +++ b/cmd/crowdsec-cli/cliitem/hubpostoverflow.go @@ -0,0 +1,102 @@ +package cliitem + +import ( + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewPostOverflow(cfg configGetter) *cliItem { + return &cliItem{ + cfg: cfg, + name: cwhub.POSTOVERFLOWS, + singular: "postoverflow", + oneOrMore: "postoverflow(s)", + help: cliHelp{ + example: `cscli postoverflows list -a +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns +`, + }, + installHelp: cliHelp{ + example: `# Install some postoverflows. +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns --dry-run -o raw + +# Download only, to be installed later. +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns --download-only + +# Install over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns --force + +# Proceed without prompting. +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + removeHelp: cliHelp{ + example: `# Uninstall some postoverflows. +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns --dry-run -o raw + +# Uninstall and also remove the downloaded files. +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns --purge + +# Remove tainted items. +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns --force + +# Proceed without prompting. +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + upgradeHelp: cliHelp{ + example: `# Upgrade some postoverflows. If they are not currently installed, they are downloaded but not installed. +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdnss + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdnss --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdnss --dry-run -o raw + +# Upgrade over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdnss --force + +# Proceed without prompting. +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdnss --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, + }, + inspectHelp: cliHelp{ + example: `# Display metadata, state and ancestor collections of postoverflows (installed or not). +cscli postoverflows inspect crowdsecurity/cdn-whitelist + +# Display difference between a tainted item and the latest one. +cscli postoverflows inspect crowdsecurity/cdn-whitelist --diff + +# Reverse the above diff +cscli postoverflows inspect crowdsecurity/cdn-whitelist --diff --rev`, + }, + listHelp: cliHelp{ + example: `# List enabled (installed) postoverflows. +cscli postoverflows list + +# List all available postoverflows (installed or not). +cscli postoverflows list -a + +# List specific postoverflows (installed or not). +cscli postoverflows list crowdsecurity/cdn-whitelists crowdsecurity/rdns`, + }, + } +} diff --git a/cmd/crowdsec-cli/cliitem/hubscenario.go b/cmd/crowdsec-cli/cliitem/hubscenario.go index a5e854b3c82..5dee3323f6f 100644 --- a/cmd/crowdsec-cli/cliitem/hubscenario.go +++ b/cmd/crowdsec-cli/cliitem/hubscenario.go @@ -19,23 +19,87 @@ cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing `, }, installHelp: cliHelp{ - example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`, + example: `# Install some scenarios. +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing --dry-run -o raw + +# Download only, to be installed later. +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing --download-only + +# Install over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing --force + +# Proceed without prompting. +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, }, removeHelp: cliHelp{ - example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`, + example: `# Uninstall some scenarios. +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing --dry-run -o raw + +# Uninstall and also remove the downloaded files. +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing --purge + +# Remove tainted items. +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing --force + +# Proceed without prompting. +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, }, upgradeHelp: cliHelp{ - example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`, + example: `# Upgrade some scenarios. If they are not currently installed, they are downloaded but not installed. +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing --dry-run -o raw + +# Upgrade over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing --force + +# Proceed without prompting. +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing --yes + +# The "--yes" parameter is implied when the command is not connected to a terminal, like pipes or scripts.`, }, inspectHelp: cliHelp{ - example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`, + example: `# Display metadata, state, metrics and ancestor collections of scenarios (installed or not). +cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing + +# Don't collect metrics (avoid error if crowdsec is not running). +cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + +# Display difference between a tainted item and the latest one. +cscli scenarios inspect crowdsecurity/ssh-bf --diff + +# Reverse the above diff +cscli scenarios inspect crowdsecurity/ssh-bf --diff --rev`, }, listHelp: cliHelp{ - example: `cscli scenarios list + example: `# List enabled (installed) scenarios. +cscli scenarios list + +# List all available scenarios (installed or not). cscli scenarios list -a -cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing -List only enabled scenarios unless "-a" or names are specified.`, +# List specific scenarios (installed or not). +cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing`, }, } } diff --git a/cmd/crowdsec-cli/cliitem/inspect.go b/cmd/crowdsec-cli/cliitem/inspect.go deleted file mode 100644 index 9939de1810e..00000000000 --- a/cmd/crowdsec-cli/cliitem/inspect.go +++ /dev/null @@ -1,57 +0,0 @@ -package cliitem - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - - "gopkg.in/yaml.v3" - - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func inspectItem(hub *cwhub.Hub, item *cwhub.Item, wantMetrics bool, output string, prometheusURL string, wantColor string) error { - // This is dirty... - // We want to show current dependencies (from content), not latest (from index). - // The item is modifed but after this function the whole hub should be thrown away. - // A cleaner way would be to copy the struct first. - item.Dependencies = item.CurrentDependencies() - - switch output { - case "human", "raw": - enc := yaml.NewEncoder(os.Stdout) - enc.SetIndent(2) - - if err := enc.Encode(item); err != nil { - return fmt.Errorf("unable to encode item: %w", err) - } - case "json": - b, err := json.MarshalIndent(*item, "", " ") - if err != nil { - return fmt.Errorf("unable to serialize item: %w", err) - } - - fmt.Print(string(b)) - } - - if output != "human" { - return nil - } - - if item.State.Tainted { - fmt.Println() - fmt.Printf(`This item is tainted. Use "%s %s inspect --diff %s" to see why.`, filepath.Base(os.Args[0]), item.Type, item.Name) - fmt.Println() - } - - if wantMetrics { - fmt.Printf("\nCurrent metrics: \n") - - if err := showMetrics(prometheusURL, hub, item, wantColor); err != nil { - return err - } - } - - return nil -} diff --git a/cmd/crowdsec-cli/cliitem/item.go b/cmd/crowdsec-cli/cliitem/item.go index 637bd3023cf..3dcc0665a89 100644 --- a/cmd/crowdsec-cli/cliitem/item.go +++ b/cmd/crowdsec-cli/cliitem/item.go @@ -2,25 +2,17 @@ package cliitem import ( "cmp" - "context" - "errors" "fmt" - "os" "strings" "github.com/fatih/color" - "github.com/hexops/gotextdiff" - "github.com/hexops/gotextdiff/myers" - "github.com/hexops/gotextdiff/span" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clihub" - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/reload" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" - "github.com/crowdsecurity/crowdsec/pkg/hubops" ) type cliHelp struct { @@ -68,400 +60,6 @@ func (cli cliItem) NewCommand() *cobra.Command { return cmd } -func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun bool, downloadOnly bool, force bool, ignoreError bool) error { - cfg := cli.cfg() - - hub, err := require.Hub(cfg, log.StandardLogger()) - if err != nil { - return err - } - - plan := hubops.NewActionPlan(hub) - - contentProvider := require.HubDownloader(ctx, cfg) - - for _, name := range args { - item := hub.GetItem(cli.name, name) - if item == nil { - msg := suggestNearestMessage(hub, cli.name, name) - if !ignoreError { - return errors.New(msg) - } - - log.Error(msg) - - continue - } - - if err = plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil { - return err - } - - if !downloadOnly { - if err = plan.AddCommand(hubops.NewEnableCommand(item, force)); err != nil { - return err - } - } - } - - verbose := (cfg.Cscli.Output == "raw") - - if err := plan.Execute(ctx, yes, dryRun, verbose); err != nil { - if !ignoreError { - return err - } - - log.Error(err) - } - - if plan.ReloadNeeded { - fmt.Println("\n" + reload.Message) - } - - return nil -} - -func (cli cliItem) newInstallCmd() *cobra.Command { - var ( - yes bool - dryRun bool - downloadOnly bool - force bool - ignoreError bool - ) - - cmd := &cobra.Command{ - Use: cmp.Or(cli.installHelp.use, "install [item]..."), - Short: cmp.Or(cli.installHelp.short, "Install given "+cli.oneOrMore), - Long: cmp.Or(cli.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", cli.name)), - Example: cli.installHelp.example, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cli.name, args, toComplete, cli.cfg) - }, - RunE: func(cmd *cobra.Command, args []string) error { - return cli.install(cmd.Context(), args, yes, dryRun, downloadOnly, force, ignoreError) - }, - } - - flags := cmd.Flags() - flags.BoolVarP(&yes, "yes", "y", false, "Confirm execution without prompt") - flags.BoolVar(&dryRun, "dry-run", false, "Don't install or remove anything; print the execution plan") - flags.BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") - flags.BoolVar(&force, "force", false, "Force install: overwrite tainted and outdated files") - flags.BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple "+cli.name) - cmd.MarkFlagsMutuallyExclusive("yes", "dry-run") - - return cmd -} - -// return the names of the installed parents of an item, used to check if we can remove it -func installedParentNames(item *cwhub.Item) []string { - ret := make([]string, 0) - - for _, parent := range item.Ancestors() { - if parent.State.Installed { - ret = append(ret, parent.Name) - } - } - - return ret -} - -func (cli cliItem) removePlan(hub *cwhub.Hub, args []string, purge bool, force bool, all bool) (*hubops.ActionPlan, error) { - plan := hubops.NewActionPlan(hub) - - if all { - itemGetter := hub.GetInstalledByType - if purge { - itemGetter = hub.GetItemsByType - } - - for _, item := range itemGetter(cli.name, true) { - if err := plan.AddCommand(hubops.NewDisableCommand(item, force)); err != nil { - return nil, err - } - - if purge { - if err := plan.AddCommand(hubops.NewPurgeCommand(item, force)); err != nil { - return nil, err - } - } - } - - return plan, nil - } - - if len(args) == 0 { - return nil, fmt.Errorf("specify at least one %s to remove or '--all'", cli.singular) - } - - for _, itemName := range args { - item := hub.GetItem(cli.name, itemName) - if item == nil { - return nil, fmt.Errorf("can't find '%s' in %s", itemName, cli.name) - } - - parents := installedParentNames(item) - - if !force && len(parents) > 0 { - log.Warningf("%s belongs to collections: %s", item.Name, parents) - log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, cli.singular) - - continue - } - - if err := plan.AddCommand(hubops.NewDisableCommand(item, force)); err != nil { - return nil, err - } - - if purge { - if err := plan.AddCommand(hubops.NewPurgeCommand(item, force)); err != nil { - return nil, err - } - } - } - - return plan, nil -} - -func (cli cliItem) remove(ctx context.Context, args []string, yes bool, dryRun bool, purge bool, force bool, all bool) error { - cfg := cli.cfg() - - hub, err := require.Hub(cli.cfg(), log.StandardLogger()) - if err != nil { - return err - } - - plan, err := cli.removePlan(hub, args, purge, force, all) - if err != nil { - return err - } - - verbose := (cfg.Cscli.Output == "raw") - - if err := plan.Execute(ctx, yes, dryRun, verbose); err != nil { - return err - } - - if plan.ReloadNeeded { - fmt.Println("\n" + reload.Message) - } - - return nil -} - -func (cli cliItem) newRemoveCmd() *cobra.Command { - var ( - yes bool - dryRun bool - purge bool - force bool - all bool - ) - - cmd := &cobra.Command{ - Use: cmp.Or(cli.removeHelp.use, "remove [item]..."), - Short: cmp.Or(cli.removeHelp.short, "Remove given "+cli.oneOrMore), - Long: cmp.Or(cli.removeHelp.long, "Remove one or more "+cli.name), - Example: cli.removeHelp.example, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cli.name, args, toComplete, cli.cfg) - }, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) > 0 && all { - return errors.New("can't specify items and '--all' at the same time") - } - - return cli.remove(cmd.Context(), args, yes, dryRun, purge, force, all) - }, - } - - flags := cmd.Flags() - flags.BoolVarP(&yes, "yes", "y", false, "Confirm execution without prompt") - flags.BoolVar(&dryRun, "dry-run", false, "Don't install or remove anything; print the execution plan") - flags.BoolVar(&purge, "purge", false, "Delete source file too") - flags.BoolVar(&force, "force", false, "Force remove: remove tainted and outdated files") - flags.BoolVar(&all, "all", false, "Remove all the "+cli.name) - cmd.MarkFlagsMutuallyExclusive("yes", "dry-run") - - return cmd -} - -func (cli cliItem) upgradePlan(hub *cwhub.Hub, contentProvider cwhub.ContentProvider, args []string, force bool, all bool) (*hubops.ActionPlan, error) { - plan := hubops.NewActionPlan(hub) - - if all { - for _, item := range hub.GetInstalledByType(cli.name, true) { - if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil { - return nil, err - } - } - - return plan, nil - } - - if len(args) == 0 { - return nil, fmt.Errorf("specify at least one %s to upgrade or '--all'", cli.singular) - } - - for _, itemName := range args { - item := hub.GetItem(cli.name, itemName) - if item == nil { - return nil, fmt.Errorf("can't find '%s' in %s", itemName, cli.name) - } - - if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil { - return nil, err - } - } - - return plan, nil -} - -func (cli cliItem) upgrade(ctx context.Context, args []string, yes bool, dryRun bool, force bool, all bool) error { - cfg := cli.cfg() - - hub, err := require.Hub(cfg, log.StandardLogger()) - if err != nil { - return err - } - - contentProvider := require.HubDownloader(ctx, cfg) - - plan, err := cli.upgradePlan(hub, contentProvider, args, force, all) - if err != nil { - return err - } - - verbose := (cfg.Cscli.Output == "raw") - - if err := plan.Execute(ctx, yes, dryRun, verbose); err != nil { - return err - } - - if plan.ReloadNeeded { - fmt.Println("\n" + reload.Message) - } - - return nil -} - -func (cli cliItem) newUpgradeCmd() *cobra.Command { - var ( - yes bool - dryRun bool - all bool - force bool - ) - - cmd := &cobra.Command{ - Use: cmp.Or(cli.upgradeHelp.use, "upgrade [item]..."), - Short: cmp.Or(cli.upgradeHelp.short, "Upgrade given "+cli.oneOrMore), - Long: cmp.Or(cli.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", cli.name)), - Example: cli.upgradeHelp.example, - DisableAutoGenTag: true, - ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cli.name, args, toComplete, cli.cfg) - }, - RunE: func(cmd *cobra.Command, args []string) error { - return cli.upgrade(cmd.Context(), args, yes, dryRun, force, all) - }, - } - - flags := cmd.Flags() - flags.BoolVarP(&yes, "yes", "y", false, "Confirm execution without prompt") - flags.BoolVar(&dryRun, "dry-run", false, "Don't install or remove anything; print the execution plan") - flags.BoolVarP(&all, "all", "a", false, "Upgrade all the "+cli.name) - flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated files") - cmd.MarkFlagsMutuallyExclusive("yes", "dry-run") - - return cmd -} - -func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff bool, rev bool, noMetrics bool) error { - cfg := cli.cfg() - - if rev && !diff { - return errors.New("--rev can only be used with --diff") - } - - if url != "" { - cfg.Cscli.PrometheusUrl = url - } - - var contentProvider cwhub.ContentProvider - - if diff { - contentProvider = require.HubDownloader(ctx, cfg) - } - - hub, err := require.Hub(cfg, log.StandardLogger()) - if err != nil { - return err - } - - for _, name := range args { - item := hub.GetItem(cli.name, name) - if item == nil { - return fmt.Errorf("can't find '%s' in %s", name, cli.name) - } - - if diff { - fmt.Println(cli.whyTainted(ctx, hub, contentProvider, item, rev)) - - continue - } - - if err = inspectItem(hub, item, !noMetrics, cfg.Cscli.Output, cfg.Cscli.PrometheusUrl, cfg.Cscli.Color); err != nil { - return err - } - - if cli.inspectDetail != nil { - if err = cli.inspectDetail(item); err != nil { - return err - } - } - } - - return nil -} - -func (cli cliItem) newInspectCmd() *cobra.Command { - var ( - url string - diff bool - rev bool - noMetrics bool - ) - - cmd := &cobra.Command{ - Use: cmp.Or(cli.inspectHelp.use, "inspect [item]..."), - Short: cmp.Or(cli.inspectHelp.short, "Inspect given "+cli.oneOrMore), - Long: cmp.Or(cli.inspectHelp.long, "Inspect the state of one or more "+cli.name), - Example: cli.inspectHelp.example, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cli.name, args, toComplete, cli.cfg) - }, - RunE: func(cmd *cobra.Command, args []string) error { - return cli.inspect(cmd.Context(), args, url, diff, rev, noMetrics) - }, - } - - flags := cmd.Flags() - flags.StringVarP(&url, "url", "u", "", "Prometheus url") - flags.BoolVar(&diff, "diff", false, "Show diff with latest version (for tainted items)") - flags.BoolVar(&rev, "rev", false, "Reverse diff output") - flags.BoolVar(&noMetrics, "no-metrics", false, "Don't show metrics (when cscli.output=human)") - - return cmd -} - func (cli cliItem) list(args []string, all bool) error { cfg := cli.cfg() @@ -500,91 +98,23 @@ func (cli cliItem) newListCmd() *cobra.Command { return cmd } -// return the diff between the installed version and the latest version -func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, contentProvider cwhub.ContentProvider, reverse bool) (string, error) { - if !item.State.Installed { - return "", fmt.Errorf("'%s' is not installed", item.FQName()) - } - - dest, err := os.CreateTemp("", "cscli-diff-*") +func compInstalledItems(itemType string, args []string, toComplete string, cfg configGetter) ([]string, cobra.ShellCompDirective) { + hub, err := require.Hub(cfg(), nil) if err != nil { - return "", fmt.Errorf("while creating temporary file: %w", err) + return nil, cobra.ShellCompDirectiveDefault } - defer os.Remove(dest.Name()) - _, remoteURL, err := item.FetchContentTo(ctx, contentProvider, dest.Name()) - if err != nil { - return "", err - } + items := hub.GetInstalledByType(itemType, true) - latestContent, err := os.ReadFile(dest.Name()) - if err != nil { - return "", fmt.Errorf("while reading %s: %w", dest.Name(), err) - } - - localContent, err := os.ReadFile(item.State.LocalPath) - if err != nil { - return "", fmt.Errorf("while reading %s: %w", item.State.LocalPath, err) - } - - file1 := item.State.LocalPath - file2 := remoteURL - content1 := string(localContent) - content2 := string(latestContent) - - if reverse { - file1, file2 = file2, file1 - content1, content2 = content2, content1 - } - - edits := myers.ComputeEdits(span.URIFromPath(file1), content1, content2) - diff := gotextdiff.ToUnified(file1, file2, content1, edits) - - return fmt.Sprintf("%s", diff), nil -} - -func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub.ContentProvider, item *cwhub.Item, reverse bool) string { - if !item.State.Installed { - return fmt.Sprintf("# %s is not installed", item.FQName()) - } + comp := make([]string, 0) - if !item.State.Tainted { - return fmt.Sprintf("# %s is not tainted", item.FQName()) - } - - if len(item.State.TaintedBy) == 0 { - return fmt.Sprintf("# %s is tainted but we don't know why. please report this as a bug", item.FQName()) - } - - ret := []string{ - fmt.Sprintf("# Let's see why %s is tainted.", item.FQName()), - } - - for _, fqsub := range item.State.TaintedBy { - ret = append(ret, fmt.Sprintf("\n-> %s\n", fqsub)) - - sub, err := hub.GetItemFQ(fqsub) - if err != nil { - ret = append(ret, err.Error()) - } - - diff, err := cli.itemDiff(ctx, sub, contentProvider, reverse) - if err != nil { - ret = append(ret, err.Error()) - } - - if diff != "" { - ret = append(ret, diff) - } else if len(sub.State.TaintedBy) > 0 { - taintList := strings.Join(sub.State.TaintedBy, ", ") - if sub.FQName() == taintList { - // hack: avoid message "item is tainted by itself" - continue - } - - ret = append(ret, fmt.Sprintf("# %s is tainted by %s", sub.FQName(), taintList)) + for _, item := range items { + if strings.Contains(item.Name, toComplete) { + comp = append(comp, item.Name) } } - return strings.Join(ret, "\n") + cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) + + return comp, cobra.ShellCompDirectiveNoFileComp } diff --git a/cmd/crowdsec-cli/cliitem/parser.go b/cmd/crowdsec-cli/cliitem/parser.go deleted file mode 100644 index bc1d96bdaf0..00000000000 --- a/cmd/crowdsec-cli/cliitem/parser.go +++ /dev/null @@ -1,41 +0,0 @@ -package cliitem - -import ( - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewParser(cfg configGetter) *cliItem { - return &cliItem{ - cfg: cfg, - name: cwhub.PARSERS, - singular: "parser", - oneOrMore: "parser(s)", - help: cliHelp{ - example: `cscli parsers list -a -cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs -`, - }, - installHelp: cliHelp{ - example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - }, - removeHelp: cliHelp{ - example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - }, - upgradeHelp: cliHelp{ - example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - }, - inspectHelp: cliHelp{ - example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`, - }, - listHelp: cliHelp{ - example: `cscli parsers list -cscli parsers list -a -cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs - -List only enabled parsers unless "-a" or names are specified.`, - }, - } -} diff --git a/cmd/crowdsec-cli/cliitem/postoverflow.go b/cmd/crowdsec-cli/cliitem/postoverflow.go deleted file mode 100644 index ea53aef327d..00000000000 --- a/cmd/crowdsec-cli/cliitem/postoverflow.go +++ /dev/null @@ -1,41 +0,0 @@ -package cliitem - -import ( - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewPostOverflow(cfg configGetter) *cliItem { - return &cliItem{ - cfg: cfg, - name: cwhub.POSTOVERFLOWS, - singular: "postoverflow", - oneOrMore: "postoverflow(s)", - help: cliHelp{ - example: `cscli postoverflows list -a -cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns -`, - }, - installHelp: cliHelp{ - example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - }, - removeHelp: cliHelp{ - example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - }, - upgradeHelp: cliHelp{ - example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - }, - inspectHelp: cliHelp{ - example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - }, - listHelp: cliHelp{ - example: `cscli postoverflows list -cscli postoverflows list -a -cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns - -List only enabled postoverflows unless "-a" or names are specified.`, - }, - } -} diff --git a/cmd/crowdsec-cli/cliitem/suggest.go b/cmd/crowdsec-cli/cliitem/suggest.go deleted file mode 100644 index b0f19b6993c..00000000000 --- a/cmd/crowdsec-cli/cliitem/suggest.go +++ /dev/null @@ -1,77 +0,0 @@ -package cliitem - -import ( - "fmt" - "slices" - "strings" - - "github.com/agext/levenshtein" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -// suggestNearestMessage returns a message with the most similar item name, if one is found -func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string { - const maxDistance = 7 - - score := 100 - nearest := "" - - for _, item := range hub.GetItemsByType(itemType, false) { - d := levenshtein.Distance(itemName, item.Name, nil) - if d < score { - score = d - nearest = item.Name - } - } - - msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType) - - if score < maxDistance { - msg += fmt.Sprintf(", did you mean '%s'?", nearest) - } - - return msg -} - -func compAllItems(itemType string, args []string, toComplete string, cfg configGetter) ([]string, cobra.ShellCompDirective) { - hub, err := require.Hub(cfg(), nil) - if err != nil { - return nil, cobra.ShellCompDirectiveDefault - } - - comp := make([]string, 0) - - for _, item := range hub.GetItemsByType(itemType, false) { - if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) { - comp = append(comp, item.Name) - } - } - - cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) - - return comp, cobra.ShellCompDirectiveNoFileComp -} - -func compInstalledItems(itemType string, args []string, toComplete string, cfg configGetter) ([]string, cobra.ShellCompDirective) { - hub, err := require.Hub(cfg(), nil) - if err != nil { - return nil, cobra.ShellCompDirectiveDefault - } - - items := hub.GetInstalledByType(itemType, true) - - comp := make([]string, 0) - - for _, item := range items { - if strings.Contains(item.Name, toComplete) { - comp = append(comp, item.Name) - } - } - - cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) - - return comp, cobra.ShellCompDirectiveNoFileComp -} From 9dfa74bb52d7905c5b711ff9bebf8d077bc391ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jan 2025 10:24:58 +0100 Subject: [PATCH 06/25] build(deps): bump github.com/golang-jwt/jwt/v4 from 4.5.0 to 4.5.1 (#3380) Bumps [github.com/golang-jwt/jwt/v4](https://github.com/golang-jwt/jwt) from 4.5.0 to 4.5.1. - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Changelog](https://github.com/golang-jwt/jwt/blob/main/VERSION_HISTORY.md) - [Commits](https://github.com/golang-jwt/jwt/compare/v4.5.0...v4.5.1) --- updated-dependencies: - dependency-name: github.com/golang-jwt/jwt/v4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index aa723b38409..20f8de63218 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/goccy/go-yaml v1.11.0 github.com/gofrs/uuid v4.0.0+incompatible github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 + github.com/golang-jwt/jwt/v4 v4.5.1 github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 diff --git a/go.sum b/go.sum index e7f181d7d24..950f901839c 100644 --- a/go.sum +++ b/go.sum @@ -301,8 +301,8 @@ github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= From aebe9724f8053aa8f336f479b31b030bef5d171f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jan 2025 11:09:13 +0100 Subject: [PATCH 07/25] build(deps): bump golang.org/x/crypto from 0.26.0 to 0.31.0 (#3381) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.26.0 to 0.31.0. - [Commits](https://github.com/golang/crypto/compare/v0.26.0...v0.31.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 20f8de63218..d002aa43d9b 100644 --- a/go.mod +++ b/go.mod @@ -105,12 +105,12 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect go.opentelemetry.io/otel/sdk v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect - golang.org/x/crypto v0.26.0 + golang.org/x/crypto v0.31.0 golang.org/x/mod v0.20.0 golang.org/x/net v0.28.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.24.0 - golang.org/x/text v0.17.0 + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 + golang.org/x/text v0.21.0 golang.org/x/time v0.6.0 // indirect google.golang.org/grpc v1.67.1 google.golang.org/protobuf v1.34.2 @@ -218,7 +218,7 @@ require ( go.opentelemetry.io/otel/metric v1.28.0 // indirect go.uber.org/atomic v1.10.0 // indirect golang.org/x/arch v0.7.0 // indirect - golang.org/x/term v0.23.0 // indirect + golang.org/x/term v0.27.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect diff --git a/go.sum b/go.sum index 950f901839c..63b6394ef1c 100644 --- a/go.sum +++ b/go.sum @@ -794,8 +794,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -839,8 +839,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -876,8 +876,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -885,8 +885,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -899,8 +899,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 866b0ad60c97c515c069991d5e31a320a047525e Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 7 Jan 2025 16:39:32 +0100 Subject: [PATCH 08/25] Hubops tests (#3393) --- cmd/crowdsec-cli/require/require.go | 3 +- pkg/cwhub/cwhub_test.go | 18 +- pkg/cwhub/doc.go | 1 - pkg/cwhub/download.go | 6 +- pkg/cwhub/download_test.go | 150 ++++++++++++++-- pkg/cwhub/fetch.go | 8 +- pkg/cwhub/hub.go | 18 +- pkg/cwhub/hub_test.go | 260 +++++++++++++++++++++++----- pkg/cwhub/item.go | 106 +----------- pkg/cwhub/item_test.go | 25 +-- pkg/cwhub/itemupgrade_test.go | 3 - pkg/cwhub/state.go | 61 +++++++ pkg/cwhub/state_test.go | 76 ++++++++ pkg/cwhub/sync.go | 11 +- pkg/hubops/doc.go | 45 +++++ test/bats/20_hub.bats | 4 +- test/bats/20_hub_items.bats | 4 +- test/bats/hub-index.bats | 6 +- 18 files changed, 580 insertions(+), 225 deletions(-) create mode 100644 pkg/cwhub/state.go create mode 100644 pkg/cwhub/state_test.go create mode 100644 pkg/hubops/doc.go diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index a44e76ae47d..dd98cd092cb 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -89,7 +89,6 @@ func HubDownloader(ctx context.Context, c *csconfig.Config) *cwhub.Downloader { remote := &cwhub.Downloader{ Branch: branch, URLTemplate: urlTemplate, - IndexPath: ".index.json", } return remote @@ -115,7 +114,7 @@ func Hub(c *csconfig.Config, logger *logrus.Logger) (*cwhub.Hub, error) { } if err := hub.Load(); err != nil { - return nil, fmt.Errorf("failed to read hub index: %w. Run 'sudo cscli hub update' to download the index again", err) + return nil, err } return hub, nil diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 94a1d6ef6fd..befd279ff65 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -29,10 +29,9 @@ const mockURLTemplate = "https://cdn-hub.crowdsec.net/crowdsecurity/%s/%s" var responseByPath map[string]string -// testHub initializes a temporary hub with an empty json file, optionally updating it. -func testHub(t *testing.T, update bool) *Hub { - tmpDir, err := os.MkdirTemp("", "testhub") - require.NoError(t, err) +// testHubOld initializes a temporary hub with an empty json file, optionally updating it. +func testHubOld(t *testing.T, update bool) *Hub { + tmpDir := t.TempDir() local := &csconfig.LocalHubCfg{ HubDir: filepath.Join(tmpDir, "crowdsec", "hub"), @@ -41,7 +40,7 @@ func testHub(t *testing.T, update bool) *Hub { InstallDataDir: filepath.Join(tmpDir, "installed-data"), } - err = os.MkdirAll(local.HubDir, 0o700) + err := os.MkdirAll(local.HubDir, 0o700) require.NoError(t, err) err = os.MkdirAll(local.InstallDir, 0o700) @@ -53,10 +52,6 @@ func testHub(t *testing.T, update bool) *Hub { err = os.WriteFile(local.HubIndexFile, []byte("{}"), 0o644) require.NoError(t, err) - t.Cleanup(func() { - os.RemoveAll(tmpDir) - }) - hub, err := NewHub(local, log.StandardLogger()) require.NoError(t, err) @@ -64,11 +59,10 @@ func testHub(t *testing.T, update bool) *Hub { indexProvider := &Downloader{ Branch: "master", URLTemplate: mockURLTemplate, - IndexPath: ".index.json", } ctx := context.Background() - err := hub.Update(ctx, indexProvider, false) + err = hub.Update(ctx, indexProvider, false) require.NoError(t, err) } @@ -92,7 +86,7 @@ func envSetup(t *testing.T) *Hub { // Mock the http client HubClient.Transport = newMockTransport() - hub := testHub(t, true) + hub := testHubOld(t, true) return hub } diff --git a/pkg/cwhub/doc.go b/pkg/cwhub/doc.go index b85d7634da4..fb7209b77ae 100644 --- a/pkg/cwhub/doc.go +++ b/pkg/cwhub/doc.go @@ -90,7 +90,6 @@ // indexProvider := cwhub.Downloader{ // URLTemplate: "https://cdn-hub.crowdsec.net/crowdsecurity/%s/%s", // Branch: "master", -// IndexPath: ".index.json", // } // // The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 48cb2382668..fa92e9960de 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -12,11 +12,13 @@ import ( "github.com/crowdsecurity/go-cs-lib/downloader" ) +// no need to import the lib package to use this +type NotFoundError = downloader.NotFoundError + // Downloader is used to retrieve index and items from a remote hub, with cache control. type Downloader struct { Branch string URLTemplate string - IndexPath string } // IndexProvider retrieves and writes .index.json @@ -61,7 +63,7 @@ func addURLParam(rawURL string, param string, value string) (string, error) { // It uses a temporary file to avoid partial downloads, and won't overwrite the original // if it has not changed. func (d *Downloader) FetchIndex(ctx context.Context, destPath string, withContent bool, logger *logrus.Logger) (bool, error) { - url, err := d.urlTo(d.IndexPath) + url, err := d.urlTo(".index.json") if err != nil { return false, fmt.Errorf("failed to build hub index request: %w", err) } diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go index fc0b257a284..7b0b99c28d8 100644 --- a/pkg/cwhub/download_test.go +++ b/pkg/cwhub/download_test.go @@ -10,43 +10,173 @@ import ( "testing" "github.com/sirupsen/logrus" + logtest "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/go-cs-lib/cstest" ) func TestFetchIndex(t *testing.T) { ctx := context.Background() mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } + if r.URL.Query().Get("with_content") == "true" { - w.WriteHeader(http.StatusOK) _, err := w.Write([]byte(`Hi I'm an index with content`)) assert.NoError(t, err) } else { - w.WriteHeader(http.StatusOK) - _, err := w.Write([]byte(`Hi I'm a regular index`)) + _, err := w.Write([]byte(`Hi I'm a minified index`)) assert.NoError(t, err) } })) defer mockServer.Close() + discard := logrus.New() + discard.Out = io.Discard + downloader := &Downloader{ - Branch: "main", URLTemplate: mockServer.URL + "/%s/%s", - IndexPath: "index.txt", } - logger := logrus.New() - logger.Out = io.Discard - - destPath := filepath.Join(t.TempDir(), "index.txt") + destPath := filepath.Join(t.TempDir(), "index-here") withContent := true - downloaded, err := downloader.FetchIndex(ctx, destPath, withContent, logger) + var notFoundError NotFoundError + + // bad branch + + downloader.Branch = "dev" + + downloaded, err := downloader.FetchIndex(ctx, destPath, withContent, discard) + require.ErrorAs(t, err, ¬FoundError) + assert.False(t, downloaded) + + // ok + + downloader.Branch = "main" + + downloaded, err = downloader.FetchIndex(ctx, destPath, withContent, discard) require.NoError(t, err) assert.True(t, downloaded) content, err := os.ReadFile(destPath) require.NoError(t, err) assert.Equal(t, "Hi I'm an index with content", string(content)) + + // not "downloading" a second time + // since we don't have cache control in the mockServer, + // the file is downloaded to a temporary location but not replaced + + downloaded, err = downloader.FetchIndex(ctx, destPath, withContent, discard) + require.NoError(t, err) + assert.False(t, downloaded) + + // download without item content + + downloaded, err = downloader.FetchIndex(ctx, destPath, !withContent, discard) + require.NoError(t, err) + assert.True(t, downloaded) + + content, err = os.ReadFile(destPath) + require.NoError(t, err) + assert.Equal(t, "Hi I'm a minified index", string(content)) + + // bad domain name + + downloader.URLTemplate = "x/%s/%s" + downloaded, err = downloader.FetchIndex(ctx, destPath, !withContent, discard) + cstest.AssertErrorContains(t, err, `Get "x/main/.index.json": unsupported protocol scheme ""`) + assert.False(t, downloaded) + + downloader.URLTemplate = "http://x/%s/%s" + downloaded, err = downloader.FetchIndex(ctx, destPath, !withContent, discard) + // can be no such host, server misbehaving, etc + cstest.AssertErrorContains(t, err, `Get "http://x/main/.index.json": dial tcp: lookup x`) + assert.False(t, downloaded) +} + +func TestFetchContent(t *testing.T) { + ctx := context.Background() + + wantContent := "{'description':'linux'}" + wantHash := "e557cb9e1cb051bc3b6a695e4396c5f8e0eff4b7b0d2cc09f7684e1d52ea2224" + remotePath := "collections/crowdsecurity/linux.yaml" + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/"+remotePath { + w.WriteHeader(http.StatusNotFound) + } + + _, err := w.Write([]byte(wantContent)) + assert.NoError(t, err) + })) + defer mockServer.Close() + + wantURL := mockServer.URL + "/main/collections/crowdsecurity/linux.yaml" + + // bad branch + + hubDownloader := &Downloader{ + URLTemplate: mockServer.URL + "/%s/%s", + } + + discard := logrus.New() + discard.Out = io.Discard + + destPath := filepath.Join(t.TempDir(), "content-here") + + var notFoundError NotFoundError + + // bad branch + + hubDownloader.Branch = "dev" + + downloaded, url, err := hubDownloader.FetchContent(ctx, remotePath, destPath, wantHash, discard) + assert.Empty(t, url) + require.ErrorAs(t, err, ¬FoundError) + assert.False(t, downloaded) + + // bad path + + hubDownloader.Branch = "main" + + downloaded, url, err = hubDownloader.FetchContent(ctx, "collections/linux.yaml", destPath, wantHash, discard) + assert.Empty(t, url) + require.ErrorAs(t, err, ¬FoundError) + assert.False(t, downloaded) + + // hash mismatch: the file is not reported as downloaded because it's not replaced + + capture, hook := logtest.NewNullLogger() + capture.SetLevel(logrus.WarnLevel) + + downloaded, url, err = hubDownloader.FetchContent(ctx, remotePath, destPath, "1234", capture) + assert.Equal(t, wantURL, url) + require.NoError(t, err) + assert.False(t, downloaded) + cstest.RequireLogContains(t, hook, "hash mismatch: expected 1234, got "+wantHash) + + // ok + + downloaded, url, err = hubDownloader.FetchContent(ctx, remotePath, destPath, wantHash, discard) + assert.Equal(t, wantURL, url) + require.NoError(t, err) + assert.True(t, downloaded) + + content, err := os.ReadFile(destPath) + require.NoError(t, err) + assert.Equal(t, wantContent, string(content)) + + // not "downloading" a second time + // since we don't have cache control in the mockServer, + // the file is downloaded to a temporary location but not replaced + + downloaded, url, err = hubDownloader.FetchContent(ctx, remotePath, destPath, wantHash, discard) + assert.Equal(t, wantURL, url) + require.NoError(t, err) + assert.False(t, downloaded) } diff --git a/pkg/cwhub/fetch.go b/pkg/cwhub/fetch.go index dd1a520d7e2..e8dacad4a6d 100644 --- a/pkg/cwhub/fetch.go +++ b/pkg/cwhub/fetch.go @@ -11,8 +11,8 @@ import ( ) // writeEmbeddedContentTo writes the embedded content to the specified path and checks the hash. -// If the content is base64 encoded, it will be decoded before writing. Check for item.Content -// before calling this method. +// If the content is base64 encoded, it will be decoded before writing. Call this method only +// if item.Content if not empty. func (i *Item) writeEmbeddedContentTo(destPath, wantHash string) error { if i.Content == "" { return fmt.Errorf("no embedded content for %s", i.Name) @@ -48,7 +48,9 @@ func (i *Item) writeEmbeddedContentTo(destPath, wantHash string) error { } // FetchContentTo writes the last version of the item's YAML file to the specified path. -// Returns whether the file was downloaded, and the remote url for feedback purposes. +// If the file is embedded in the index file, it will be written directly without downloads. +// Returns whether the file was downloaded (to inform if the security engine needs reloading) +// and the remote url for feedback purposes. func (i *Item) FetchContentTo(ctx context.Context, contentProvider ContentProvider, destPath string) (bool, string, error) { wantHash := i.latestHash() if wantHash == "" { diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 3722ceaafcd..998a4032359 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -36,7 +36,7 @@ func (h *Hub) GetDataDir() string { // and check for unmanaged items. func NewHub(local *csconfig.LocalHubCfg, logger *logrus.Logger) (*Hub, error) { if local == nil { - return nil, errors.New("no hub configuration found") + return nil, errors.New("no hub configuration provided") } if logger == nil { @@ -58,14 +58,10 @@ func (h *Hub) Load() error { h.logger.Debugf("loading hub idx %s", h.local.HubIndexFile) if err := h.parseIndex(); err != nil { - return err - } - - if err := h.localSync(); err != nil { - return fmt.Errorf("failed to sync hub items: %w", err) + return fmt.Errorf("invalid hub index: %w. Run 'sudo cscli hub update' to download the index again", err) } - return nil + return h.localSync() } // parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections. @@ -153,12 +149,14 @@ func (h *Hub) ItemStats() []string { return ret } +var ErrUpdateAfterSync = errors.New("cannot update hub index after load/sync") + // Update downloads the latest version of the index and writes it to disk if it changed. -// It cannot be called after Load() unless the hub is completely empty. +// It cannot be called after Load() unless the index was completely empty. func (h *Hub) Update(ctx context.Context, indexProvider IndexProvider, withContent bool) error { - if len(h.pathIndex) > 0 { + if len(h.items) > 0 { // if this happens, it's a bug. - return errors.New("cannot update hub after items have been loaded") + return ErrUpdateAfterSync } downloaded, err := indexProvider.FetchIndex(ctx, h.local.HubIndexFile, withContent, h.logger) diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index c2b949b7cdf..461b59de78b 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -2,91 +2,261 @@ package cwhub import ( "context" - "fmt" + "net/http" + "net/http/httptest" "os" + "path/filepath" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -func TestInitHubUpdate(t *testing.T) { - hub := envSetup(t) +// testHubCfg creates an empty hub structure in a temporary directory +// and returns its configuration object. +// +// This allow the reuse of the hub content for multiple instances +// of the Hub object. +func testHubCfg(t *testing.T) *csconfig.LocalHubCfg { + tempDir := t.TempDir() + + local := csconfig.LocalHubCfg{ + HubDir: filepath.Join(tempDir, "crowdsec", "hub"), + HubIndexFile: filepath.Join(tempDir, "crowdsec", "hub", ".index.json"), + InstallDir: filepath.Join(tempDir, "crowdsec"), + InstallDataDir: filepath.Join(tempDir, "installed-data"), + } + + err := os.MkdirAll(local.HubDir, 0o755) + require.NoError(t, err) - _, err := NewHub(hub.local, nil) + err = os.MkdirAll(local.InstallDir, 0o755) require.NoError(t, err) - ctx := context.Background() + err = os.MkdirAll(local.InstallDataDir, 0o755) + require.NoError(t, err) + + return &local +} - indexProvider := &Downloader{ - URLTemplate: mockURLTemplate, - Branch: "master", - IndexPath: ".index.json", +func testHub(t *testing.T, localCfg *csconfig.LocalHubCfg, indexJson string) (*Hub, error) { + if localCfg == nil { + localCfg = testHubCfg(t) } - err = hub.Update(ctx, indexProvider, false) + err := os.WriteFile(localCfg.HubIndexFile, []byte(indexJson), 0o644) require.NoError(t, err) + hub, err := NewHub(localCfg, nil) + require.NoError(t, err) err = hub.Load() + + return hub, err +} + +func TestIndexEmpty(t *testing.T) { + // an empty hub is valid, and should not have warnings + hub, err := testHub(t, nil, "{}") require.NoError(t, err) + assert.Empty(t, hub.Warnings) } -func TestUpdateIndex(t *testing.T) { - // bad url template - fmt.Println("Test 'bad URL'") +func TestIndexJSON(t *testing.T) { + // but it can't be an empty string + hub, err := testHub(t, nil, "") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: unexpected end of JSON input") + assert.Empty(t, hub.Warnings) + + // it must be valid json + hub, err = testHub(t, nil, "def not json") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: invalid character 'd' looking for beginning of value. Run 'sudo cscli hub update' to download the index again") + assert.Empty(t, hub.Warnings) + + hub, err = testHub(t, nil, "{") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: unexpected end of JSON input") + assert.Empty(t, hub.Warnings) - tmpIndex, err := os.CreateTemp("", "index.json") + // and by json we mean an object + hub, err = testHub(t, nil, "[]") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: json: cannot unmarshal array into Go value of type cwhub.HubItems") + assert.Empty(t, hub.Warnings) +} + +func TestIndexUnknownItemType(t *testing.T) { + // Allow unknown fields in the top level object, likely new item types + hub, err := testHub(t, nil, `{"goodies": {}}`) require.NoError(t, err) + assert.Empty(t, hub.Warnings) +} - // close the file to avoid preventing the rename on windows - err = tmpIndex.Close() +func TestHubUpdate(t *testing.T) { + // update an empty hub with a index containing a parser. + hub, err := testHub(t, nil, "{}") require.NoError(t, err) - t.Cleanup(func() { - os.Remove(tmpIndex.Name()) - }) + index1 := ` +{ + "parsers": { + "author/pars1": { + "path": "parsers/s01-parse/pars1.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` - hub := envSetup(t) + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } - hub.local.HubIndexFile = tmpIndex.Name() + _, err = w.Write([]byte(index1)) + assert.NoError(t, err) + })) + defer mockServer.Close() ctx := context.Background() - indexProvider := &Downloader{ - URLTemplate: "x", - Branch: "", - IndexPath: "", + downloader := &Downloader{ + Branch: "main", + URLTemplate: mockServer.URL + "/%s/%s", } - err = hub.Update(ctx, indexProvider, false) - cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'") + err = hub.Update(ctx, downloader, true) + require.NoError(t, err) + + err = hub.Load() + require.NoError(t, err) + + item := hub.GetItem("parsers", "author/pars1") + assert.NotEmpty(t, item) + assert.Equal(t, "author/pars1", item.Name) +} + +func TestHubUpdateInvalidTemplate(t *testing.T) { + hub, err := testHub(t, nil, "{}") + require.NoError(t, err) - // bad domain - fmt.Println("Test 'bad domain'") + ctx := context.Background() - indexProvider = &Downloader{ - URLTemplate: "https://baddomain/crowdsecurity/%s/%s", - Branch: "master", - IndexPath: ".index.json", + downloader := &Downloader{ + Branch: "main", + URLTemplate: "x", } - err = hub.Update(ctx, indexProvider, false) + err = hub.Update(ctx, downloader, true) + cstest.RequireErrorMessage(t, err, "failed to build hub index request: invalid URL template 'x'") +} + +func TestHubUpdateCannotWrite(t *testing.T) { + hub, err := testHub(t, nil, "{}") require.NoError(t, err) - // XXX: this is not failing - // cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") - // bad target path - fmt.Println("Test 'bad target path'") + index1 := ` +{ + "parsers": { + "author/pars1": { + "path": "parsers/s01-parse/pars1.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } + + _, err = w.Write([]byte(index1)) + assert.NoError(t, err) + })) + defer mockServer.Close() + + ctx := context.Background() - indexProvider = &Downloader{ - URLTemplate: mockURLTemplate, - Branch: "master", - IndexPath: ".index.json", + downloader := &Downloader{ + Branch: "main", + URLTemplate: mockServer.URL + "/%s/%s", } - hub.local.HubIndexFile = "/does/not/exist/index.json" + hub.local.HubIndexFile = "/proc/foo/bar/baz/.index.json" + + err = hub.Update(ctx, downloader, true) + cstest.RequireErrorContains(t, err, "failed to create temporary download file for /proc/foo/bar/baz/.index.json") +} + +func TestHubUpdateAfterLoad(t *testing.T) { + // Update() can't be called after Load() if the hub is not completely empty. + index1 := ` +{ + "parsers": { + "author/pars1": { + "path": "parsers/s01-parse/pars1.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` + hub, err := testHub(t, nil, index1) + require.NoError(t, err) + + index2 := ` +{ + "parsers": { + "author/pars2": { + "path": "parsers/s01-parse/pars2.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } + + _, err = w.Write([]byte(index2)) + assert.NoError(t, err) + })) + defer mockServer.Close() + + ctx := context.Background() + + downloader := &Downloader{ + Branch: "main", + URLTemplate: mockServer.URL + "/%s/%s", + } - err = hub.Update(ctx, indexProvider, false) - cstest.RequireErrorContains(t, err, "failed to create temporary download file for /does/not/exist/index.json:") + err = hub.Update(ctx, downloader, true) + require.ErrorIs(t, err, ErrUpdateAfterSync) } diff --git a/pkg/cwhub/item.go b/pkg/cwhub/item.go index 74b1cfa3ebe..38385d9399d 100644 --- a/pkg/cwhub/item.go +++ b/pkg/cwhub/item.go @@ -11,8 +11,6 @@ import ( "github.com/Masterminds/semver/v3" yaml "gopkg.in/yaml.v3" - - "github.com/crowdsecurity/crowdsec/pkg/emoji" ) const ( @@ -46,62 +44,6 @@ type ItemVersion struct { Deprecated bool `json:"deprecated,omitempty" yaml:"deprecated,omitempty"` } -// ItemState is used to keep the local state (i.e. at runtime) of an item. -// This data is not stored in the index, but is displayed with "cscli ... inspect". -type ItemState struct { - LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` - LocalVersion string `json:"local_version,omitempty" yaml:"local_version,omitempty"` - LocalHash string `json:"local_hash,omitempty" yaml:"local_hash,omitempty"` - Installed bool `json:"installed"` - Downloaded bool `json:"downloaded"` - UpToDate bool `json:"up_to_date"` - Tainted bool `json:"tainted"` - TaintedBy []string `json:"tainted_by,omitempty" yaml:"tainted_by,omitempty"` - BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` -} - -// IsLocal returns true if the item has been create by a user (not downloaded from the hub). -func (s *ItemState) IsLocal() bool { - return s.Installed && !s.Downloaded -} - -// Text returns the status of the item as a string (eg. "enabled,update-available"). -func (s *ItemState) Text() string { - ret := "disabled" - - if s.Installed { - ret = "enabled" - } - - if s.IsLocal() { - ret += ",local" - } - - if s.Tainted { - ret += ",tainted" - } else if !s.UpToDate && !s.IsLocal() { - ret += ",update-available" - } - - return ret -} - -// Emoji returns the status of the item as an emoji (eg. emoji.Warning). -func (s *ItemState) Emoji() string { - switch { - case s.IsLocal(): - return emoji.House - case !s.Installed: - return emoji.Prohibited - case s.Tainted || (!s.UpToDate && !s.IsLocal()): - return emoji.Warning - case s.Installed: - return emoji.CheckMark - default: - return emoji.QuestionMark - } -} - type Dependencies struct { Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` @@ -292,49 +234,11 @@ func (i *Item) CurrentDependencies() Dependencies { } func (i *Item) logMissingSubItems() { - if !i.HasSubItems() { - return - } - - for _, subName := range i.Parsers { - if i.hub.GetItem(PARSERS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, PARSERS, i.Name) - } - } - - for _, subName := range i.Scenarios { - if i.hub.GetItem(SCENARIOS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, SCENARIOS, i.Name) - } - } - - for _, subName := range i.PostOverflows { - if i.hub.GetItem(POSTOVERFLOWS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, POSTOVERFLOWS, i.Name) - } - } - - for _, subName := range i.Contexts { - if i.hub.GetItem(CONTEXTS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, CONTEXTS, i.Name) - } - } - - for _, subName := range i.AppsecConfigs { - if i.hub.GetItem(APPSEC_CONFIGS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, APPSEC_CONFIGS, i.Name) - } - } - - for _, subName := range i.AppsecRules { - if i.hub.GetItem(APPSEC_RULES, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, APPSEC_RULES, i.Name) - } - } - - for _, subName := range i.Collections { - if i.hub.GetItem(COLLECTIONS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name) + for _, sub := range i.CurrentDependencies().byType() { + for _, subName := range sub.itemNames { + if i.hub.GetItem(sub.typeName, subName) == nil { + i.hub.logger.Errorf("can't find %s:%s, required by %s", sub.typeName, subName, i.Name) + } } } } diff --git a/pkg/cwhub/item_test.go b/pkg/cwhub/item_test.go index 703bbb5cb90..350861ff85e 100644 --- a/pkg/cwhub/item_test.go +++ b/pkg/cwhub/item_test.go @@ -6,39 +6,16 @@ import ( "github.com/stretchr/testify/require" ) -func TestItemStatus(t *testing.T) { +func TestItemStats(t *testing.T) { hub := envSetup(t) // get existing map x := hub.GetItemMap(COLLECTIONS) require.NotEmpty(t, x) - // Get item: good and bad - for k := range x { - item := hub.GetItem(COLLECTIONS, k) - require.NotNil(t, item) - - item.State.Installed = true - item.State.UpToDate = false - item.State.Tainted = false - item.State.Downloaded = true - - txt := item.State.Text() - require.Equal(t, "enabled,update-available", txt) - - item.State.Installed = true - item.State.UpToDate = false - item.State.Tainted = false - item.State.Downloaded = false - - txt = item.State.Text() - require.Equal(t, "enabled,local", txt) - } - stats := hub.ItemStats() require.Equal(t, []string{ "Loaded: 2 parsers, 1 scenarios, 3 collections", - "Unmanaged items: 3 local, 0 tainted", }, stats) } diff --git a/pkg/cwhub/itemupgrade_test.go b/pkg/cwhub/itemupgrade_test.go index da02837e972..3225d2f013b 100644 --- a/pkg/cwhub/itemupgrade_test.go +++ b/pkg/cwhub/itemupgrade_test.go @@ -41,7 +41,6 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", - IndexPath: ".index.json", } hub, err := NewHub(hub.local, remote, nil) @@ -101,7 +100,6 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", - IndexPath: ".index.json", } hub = getHubOrFail(t, hub.local, remote) @@ -173,7 +171,6 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", - IndexPath: ".index.json", } hub = getHubOrFail(t, hub.local, remote) diff --git a/pkg/cwhub/state.go b/pkg/cwhub/state.go new file mode 100644 index 00000000000..518185aff1c --- /dev/null +++ b/pkg/cwhub/state.go @@ -0,0 +1,61 @@ +package cwhub + +import ( + "github.com/crowdsecurity/crowdsec/pkg/emoji" +) + +// ItemState is used to keep the local state (i.e. at runtime) of an item. +// This data is not stored in the index, but is displayed with "cscli ... inspect". +type ItemState struct { + LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` + LocalVersion string `json:"local_version,omitempty" yaml:"local_version,omitempty"` + LocalHash string `json:"local_hash,omitempty" yaml:"local_hash,omitempty"` + Installed bool `json:"installed"` + Downloaded bool `json:"downloaded"` + UpToDate bool `json:"up_to_date"` + Tainted bool `json:"tainted"` + TaintedBy []string `json:"tainted_by,omitempty" yaml:"tainted_by,omitempty"` + BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` +} + +// IsLocal returns true if the item has been create by a user (not downloaded from the hub). +func (s *ItemState) IsLocal() bool { + return s.Installed && !s.Downloaded +} + +// Text returns the status of the item as a string (eg. "enabled,update-available"). +func (s *ItemState) Text() string { + ret := "disabled" + + if s.Installed { + ret = "enabled" + } + + if s.IsLocal() { + ret += ",local" + } + + if s.Tainted { + ret += ",tainted" + } else if !s.UpToDate && !s.IsLocal() { + ret += ",update-available" + } + + return ret +} + +// Emoji returns the status of the item as an emoji (eg. emoji.Warning). +func (s *ItemState) Emoji() string { + switch { + case s.IsLocal(): + return emoji.House + case !s.Installed: + return emoji.Prohibited + case s.Tainted || (!s.UpToDate && !s.IsLocal()): + return emoji.Warning + case s.Installed: + return emoji.CheckMark + default: + return emoji.QuestionMark + } +} diff --git a/pkg/cwhub/state_test.go b/pkg/cwhub/state_test.go new file mode 100644 index 00000000000..3ed3de16fcc --- /dev/null +++ b/pkg/cwhub/state_test.go @@ -0,0 +1,76 @@ +package cwhub + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/crowdsecurity/crowdsec/pkg/emoji" +) + +func TestItemStateText(t *testing.T) { + // Test the text representation of an item state. + type test struct { + state ItemState + want string + wantIcon string + } + + tests := []test{ + { + ItemState{ + Installed: true, + UpToDate: false, + Tainted: false, + Downloaded: true, + }, + "enabled,update-available", + emoji.Warning, + }, { + ItemState{ + Installed: true, + UpToDate: true, + Tainted: false, + Downloaded: true, + }, + "enabled", + emoji.CheckMark, + }, { + ItemState{ + Installed: true, + UpToDate: false, + Tainted: false, + Downloaded: false, + }, + "enabled,local", + emoji.House, + }, { + ItemState{ + Installed: false, + UpToDate: false, + Tainted: false, + Downloaded: true, + }, + "disabled,update-available", + emoji.Prohibited, + }, { + ItemState{ + Installed: true, + UpToDate: false, + Tainted: true, + Downloaded: true, + }, + "enabled,tainted", + emoji.Warning, + }, + } + + for idx, tc := range tests { + t.Run("Test "+strconv.Itoa(idx), func(t *testing.T) { + got := tc.state.Text() + assert.Equal(t, tc.want, got) + assert.Equal(t, tc.wantIcon, tc.state.Emoji()) + }) + } +} diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index d2b59df35d6..ee8e49f2bf0 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -108,6 +108,7 @@ func (h *Hub) getItemFileInfo(path string, logger *logrus.Logger) (*itemFileInfo if len(subsHub) < 4 { return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subsHub)) } + stage = subsHub[1] fauthor = subsHub[2] fname = subsHub[3] @@ -167,6 +168,7 @@ func sortedVersions(raw []string) ([]string, error) { for idx, r := range raw { v, err := semver.NewVersion(r) if err != nil { + // TODO: should catch this during index parsing return nil, fmt.Errorf("%s: %w", r, err) } @@ -461,13 +463,12 @@ func removeDuplicates(sl []string) []string { // localSync updates the hub state with downloaded, installed and local items. func (h *Hub) localSync() error { - err := h.syncDir(h.local.InstallDir) - if err != nil { - return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err) + if err := h.syncDir(h.local.InstallDir); err != nil { + return fmt.Errorf("failed to sync %s: %w", h.local.InstallDir, err) } - if err = h.syncDir(h.local.HubDir); err != nil { - return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err) + if err := h.syncDir(h.local.HubDir); err != nil { + return fmt.Errorf("failed to sync %s: %w", h.local.HubDir, err) } warnings := make([]string, 0) diff --git a/pkg/hubops/doc.go b/pkg/hubops/doc.go new file mode 100644 index 00000000000..b87a42653bc --- /dev/null +++ b/pkg/hubops/doc.go @@ -0,0 +1,45 @@ +/* +Package hubops is responsible for managing the local hub (items and data files) for CrowdSec. + +The index file itself (.index.json) is still managed by pkg/cwhub, which also provides the Hub +and Item structs. + +The hubops package is mostly used by cscli for the "cscli install/remove/upgrade ..." commands. + +It adopts a command-based pattern: a Plan contains a sequence of Commands. Both Plan and Command +have separate preparation and execution methods. + + - Command Interface: + The Command interface defines the contract for all operations that can be + performed on hub items. Each operation implements the Prepare and Run + methods, allowing for pre-execution setup and actual execution logic. + + - ActionPlan: + ActionPlan serves as a container for a sequence of Commands. It manages the + addition of commands, handles dependencies between them, and orchestrates their + execution. ActionPlan also provides a mechanism for interactive confirmation and dry-run. + +To perform operations on hub items, create an ActionPlan and add the desired +Commands to it. Once all commands are added, execute the ActionPlan to perform +the operations in the correct order, handling dependencies and user confirmations. + +Example: + + hub := cwhub.NewHub(...) + plan := hubops.NewActionPlan(hub) + + downloadCmd := hubops.NewDownloadCommand(item, force) + if err := plan.AddCommand(downloadCmd); err != nil { + logrus.Fatalf("Failed to add download command: %v", err) + } + + enableCmd := hubops.NewEnableCommand(item, force) + if err := plan.AddCommand(enableCmd); err != nil { + logrus.Fatalf("Failed to add enable command: %v", err) + } + + if err := plan.Execute(ctx, confirm, dryRun, verbose); err != nil { + logrus.Fatalf("Failed to execute action plan: %v", err) + } +*/ +package hubops diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index 03723ecc82b..b03b58732fa 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -82,8 +82,8 @@ teardown() { new_hub=$(jq <"$INDEX_PATH" 'del(.parsers."crowdsecurity/smb-logs") | del (.scenarios."crowdsecurity/mysql-bf")') echo "$new_hub" >"$INDEX_PATH" rune -0 cscli hub list --error - assert_stderr --partial "can't find crowdsecurity/smb-logs in parsers, required by crowdsecurity/smb" - assert_stderr --partial "can't find crowdsecurity/mysql-bf in scenarios, required by crowdsecurity/mysql" + assert_stderr --partial "can't find parsers:crowdsecurity/smb-logs, required by crowdsecurity/smb" + assert_stderr --partial "can't find scenarios:crowdsecurity/mysql-bf, required by crowdsecurity/mysql" } @test "loading hub reports tainted items (subitem is tainted)" { diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats index 2f1c952848b..8ebe505c6e1 100644 --- a/test/bats/20_hub_items.bats +++ b/test/bats/20_hub_items.bats @@ -80,8 +80,8 @@ teardown() { rune -0 cscli collections install crowdsecurity/sshd rune -1 cscli collections inspect crowdsecurity/sshd --no-metrics - # XXX: we are on the verbose side here... - assert_stderr "Error: failed to read hub index: failed to sync hub items: failed to scan $CONFIG_DIR: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version. Run 'sudo cscli hub update' to download the index again" + # XXX: this must be triggered during parse, not sync + assert_stderr "Error: failed to sync $CONFIG_DIR: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version" } @test "removing or purging an item already removed by hand" { diff --git a/test/bats/hub-index.bats b/test/bats/hub-index.bats index 76759991e4a..a609974d67a 100644 --- a/test/bats/hub-index.bats +++ b/test/bats/hub-index.bats @@ -32,7 +32,7 @@ teardown() { EOF rune -1 cscli hub list - assert_stderr --partial "failed to read hub index: parsers:author/pars1 has no index metadata." + assert_stderr --partial "invalid hub index: parsers:author/pars1 has no index metadata." } @test "malformed index - no download path" { @@ -46,7 +46,7 @@ teardown() { EOF rune -1 cscli hub list - assert_stderr --partial "failed to read hub index: parsers:author/pars1 has no download path." + assert_stderr --partial "invalid hub index: parsers:author/pars1 has no download path." } @test "malformed parser - no stage" { @@ -63,7 +63,7 @@ teardown() { EOF rune -1 cscli hub list -o raw - assert_stderr --partial "failed to read hub index: parsers:author/pars1 has no stage." + assert_stderr --partial "invalid hub index: parsers:author/pars1 has no stage." } @test "malformed parser - short path" { From 34c0e6a4bd2094d77f023d6e5b8a8c55d93a2ce8 Mon Sep 17 00:00:00 2001 From: blotus Date: Fri, 10 Jan 2025 14:57:05 +0100 Subject: [PATCH 09/25] expand env var when loading acquis (#3375) --- pkg/acquisition/acquisition.go | 12 +++++++++++- pkg/acquisition/acquisition_test.go | 15 +++++++++++++++ pkg/acquisition/test_files/env.yaml | 6 ++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 pkg/acquisition/test_files/env.yaml diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 291bc369c3e..06a4918592b 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -16,6 +16,7 @@ import ( tomb "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/csstring" "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" @@ -232,7 +233,16 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg, prom *csconfig return nil, err } - dec := yaml.NewDecoder(yamlFile) + defer yamlFile.Close() + + acquisContent, err := io.ReadAll(yamlFile) + if err != nil { + return nil, fmt.Errorf("failed to read %s: %w", acquisFile, err) + } + + expandedAcquis := csstring.StrictExpand(string(acquisContent), os.LookupEnv) + + dec := yaml.NewDecoder(strings.NewReader(expandedAcquis)) dec.SetStrict(true) idx := -1 diff --git a/pkg/acquisition/acquisition_test.go b/pkg/acquisition/acquisition_test.go index cfe1e74c612..1ea8f11c22a 100644 --- a/pkg/acquisition/acquisition_test.go +++ b/pkg/acquisition/acquisition_test.go @@ -217,6 +217,7 @@ wowo: ajsajasjas func TestLoadAcquisitionFromFile(t *testing.T) { appendMockSource() + t.Setenv("TEST_ENV", "test_value2") tests := []struct { TestName string @@ -282,6 +283,13 @@ func TestLoadAcquisitionFromFile(t *testing.T) { }, ExpectedError: "while configuring datasource of type file from test_files/bad_filetype.yaml", }, + { + TestName: "from_env", + Config: csconfig.CrowdsecServiceCfg{ + AcquisitionFiles: []string{"test_files/env.yaml"}, + }, + ExpectedLen: 1, + }, } for _, tc := range tests { t.Run(tc.TestName, func(t *testing.T) { @@ -293,6 +301,13 @@ func TestLoadAcquisitionFromFile(t *testing.T) { } assert.Len(t, dss, tc.ExpectedLen) + if tc.TestName == "from_env" { + mock := dss[0].Dump().(*MockSource) + assert.Equal(t, "test_value2", mock.Toto) + assert.Equal(t, "foobar", mock.Labels["test"]) + assert.Equal(t, "${NON_EXISTING}", mock.Labels["non_existing"]) + assert.Equal(t, log.InfoLevel, mock.logger.Logger.Level) + } }) } } diff --git a/pkg/acquisition/test_files/env.yaml b/pkg/acquisition/test_files/env.yaml new file mode 100644 index 00000000000..8abd4b16ca5 --- /dev/null +++ b/pkg/acquisition/test_files/env.yaml @@ -0,0 +1,6 @@ +labels: + test: foobar + non_existing: ${NON_EXISTING} +log_level: info +source: mock +toto: ${TEST_ENV} \ No newline at end of file From 303ce8e42ccb1c69b15d01ad871701c447b631b6 Mon Sep 17 00:00:00 2001 From: blotus Date: Fri, 10 Jan 2025 16:00:24 +0100 Subject: [PATCH 10/25] gin: do not use gin context after returning response (#3398) --- pkg/apiserver/controllers/v1/decisions.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/apiserver/controllers/v1/decisions.go b/pkg/apiserver/controllers/v1/decisions.go index ffefffc226b..6a316d8a2e4 100644 --- a/pkg/apiserver/controllers/v1/decisions.go +++ b/pkg/apiserver/controllers/v1/decisions.go @@ -394,8 +394,6 @@ func (c *Controller) StreamDecisionNonChunked(gctx *gin.Context, bouncerInfo *en func (c *Controller) StreamDecision(gctx *gin.Context) { var err error - ctx := gctx.Request.Context() - streamStartTime := time.Now().UTC() bouncerInfo, err := getBouncerFromContext(gctx) @@ -426,7 +424,8 @@ func (c *Controller) StreamDecision(gctx *gin.Context) { if err == nil { // Only update the last pull time if no error occurred when sending the decisions to avoid missing decisions - if err := c.DBClient.UpdateBouncerLastPull(ctx, streamStartTime, bouncerInfo.ID); err != nil { + // Do not reuse the context provided by gin because we already have sent the response to the client, so there's a chance for it to already be canceled + if err := c.DBClient.UpdateBouncerLastPull(context.Background(), streamStartTime, bouncerInfo.ID); err != nil { log.Errorf("unable to update bouncer '%s' pull: %v", bouncerInfo.Name, err) } } From e9c1ed05d97bdbabb1fafbcb09a7a2388e63d9d9 Mon Sep 17 00:00:00 2001 From: Nathanael Demacon <7271496+quantumsheep@users.noreply.github.com> Date: Mon, 13 Jan 2025 09:17:18 +0100 Subject: [PATCH 11/25] feat(cscli): add env variable flag for dashboard setup (#3110) Signed-off-by: Nathanael DEMACON Co-authored-by: Nathanael DEMACON Co-authored-by: Laurence Jones --- cmd/crowdsec-cli/dashboard.go | 12 ++++++---- pkg/metabase/container.go | 44 ++++++++++++++++++----------------- pkg/metabase/metabase.go | 43 +++++++++++++++++----------------- 3 files changed, 52 insertions(+), 47 deletions(-) diff --git a/cmd/crowdsec-cli/dashboard.go b/cmd/crowdsec-cli/dashboard.go index a653fcb3a47..e138b8285c9 100644 --- a/cmd/crowdsec-cli/dashboard.go +++ b/cmd/crowdsec-cli/dashboard.go @@ -36,10 +36,11 @@ var ( metabaseConfigFile = "metabase.yaml" metabaseImage = "metabase/metabase:v0.46.6.1" /**/ - metabaseListenAddress = "127.0.0.1" - metabaseListenPort = "3000" - metabaseContainerID = "crowdsec-metabase" - crowdsecGroup = "crowdsec" + metabaseListenAddress = "127.0.0.1" + metabaseListenPort = "3000" + metabaseContainerID = "crowdsec-metabase" + metabaseContainerEnvironmentVariables []string + crowdsecGroup = "crowdsec" forceYes bool @@ -166,7 +167,7 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password if err = cli.chownDatabase(dockerGroup.Gid); err != nil { return err } - mb, err := metabase.SetupMetabase(cli.cfg().API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDBPath, dockerGroup.Gid, metabaseContainerID, metabaseImage) + mb, err := metabase.SetupMetabase(cli.cfg().API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDBPath, dockerGroup.Gid, metabaseContainerID, metabaseImage, metabaseContainerEnvironmentVariables) if err != nil { return err } @@ -193,6 +194,7 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password flags.BoolVarP(&forceYes, "yes", "y", false, "force yes") // flags.StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user") flags.StringVar(&metabasePassword, "password", "", "metabase password") + flags.StringSliceVarP(&metabaseContainerEnvironmentVariables, "env", "e", nil, "Additional environment variables to pass to the metabase container") return cmd } diff --git a/pkg/metabase/container.go b/pkg/metabase/container.go index 73e4596fcde..9787e535e86 100644 --- a/pkg/metabase/container.go +++ b/pkg/metabase/container.go @@ -16,31 +16,33 @@ import ( ) type Container struct { - ListenAddr string - ListenPort string - SharedFolder string - Image string - Name string - ID string - CLI *client.Client - MBDBUri string - DockerGroupID string + ListenAddr string + ListenPort string + SharedFolder string + Image string + Name string + ID string + CLI *client.Client + MBDBUri string + DockerGroupID string + EnvironmentVariables []string } -func NewContainer(listenAddr string, listenPort string, sharedFolder string, containerName string, image string, mbDBURI string, dockerGroupID string) (*Container, error) { +func NewContainer(listenAddr string, listenPort string, sharedFolder string, containerName string, image string, mbDBURI string, dockerGroupID string, environmentVariables []string) (*Container, error) { cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { return nil, fmt.Errorf("failed to create docker client : %s", err) } return &Container{ - ListenAddr: listenAddr, - ListenPort: listenPort, - SharedFolder: sharedFolder, - Image: image, - Name: containerName, - CLI: cli, - MBDBUri: mbDBURI, - DockerGroupID: dockerGroupID, + ListenAddr: listenAddr, + ListenPort: listenPort, + SharedFolder: sharedFolder, + Image: image, + Name: containerName, + CLI: cli, + MBDBUri: mbDBURI, + DockerGroupID: dockerGroupID, + EnvironmentVariables: environmentVariables, }, nil } @@ -79,9 +81,9 @@ func (c *Container) Create() error { }, } - env := []string{ - fmt.Sprintf("MB_DB_FILE=%s/metabase.db", containerSharedFolder), - } + env := c.EnvironmentVariables + + env = append(env, fmt.Sprintf("MB_DB_FILE=%s/metabase.db", containerSharedFolder)) if c.MBDBUri != "" { env = append(env, c.MBDBUri) } diff --git a/pkg/metabase/metabase.go b/pkg/metabase/metabase.go index 324a05666a1..0ebb219d211 100644 --- a/pkg/metabase/metabase.go +++ b/pkg/metabase/metabase.go @@ -30,15 +30,16 @@ type Metabase struct { } type Config struct { - Database *csconfig.DatabaseCfg `yaml:"database"` - ListenAddr string `yaml:"listen_addr"` - ListenPort string `yaml:"listen_port"` - ListenURL string `yaml:"listen_url"` - Username string `yaml:"username"` - Password string `yaml:"password"` - DBPath string `yaml:"metabase_db_path"` - DockerGroupID string `yaml:"-"` - Image string `yaml:"image"` + Database *csconfig.DatabaseCfg `yaml:"database"` + ListenAddr string `yaml:"listen_addr"` + ListenPort string `yaml:"listen_port"` + ListenURL string `yaml:"listen_url"` + Username string `yaml:"username"` + Password string `yaml:"password"` + DBPath string `yaml:"metabase_db_path"` + DockerGroupID string `yaml:"-"` + Image string `yaml:"image"` + EnvironmentVariables []string `yaml:"environment_variables"` } var ( @@ -88,7 +89,7 @@ func (m *Metabase) Init(containerName string, image string) error { if err != nil { return err } - m.Container, err = NewContainer(m.Config.ListenAddr, m.Config.ListenPort, m.Config.DBPath, containerName, image, DBConnectionURI, m.Config.DockerGroupID) + m.Container, err = NewContainer(m.Config.ListenAddr, m.Config.ListenPort, m.Config.DBPath, containerName, image, DBConnectionURI, m.Config.DockerGroupID, m.Config.EnvironmentVariables) if err != nil { return fmt.Errorf("container init: %w", err) } @@ -137,21 +138,21 @@ func (m *Metabase) LoadConfig(configPath string) error { m.Config = config return nil - } -func SetupMetabase(dbConfig *csconfig.DatabaseCfg, listenAddr string, listenPort string, username string, password string, mbDBPath string, dockerGroupID string, containerName string, image string) (*Metabase, error) { +func SetupMetabase(dbConfig *csconfig.DatabaseCfg, listenAddr string, listenPort string, username string, password string, mbDBPath string, dockerGroupID string, containerName string, image string, environmentVariables []string) (*Metabase, error) { metabase := &Metabase{ Config: &Config{ - Database: dbConfig, - ListenAddr: listenAddr, - ListenPort: listenPort, - Username: username, - Password: password, - ListenURL: fmt.Sprintf("http://%s:%s", listenAddr, listenPort), - DBPath: mbDBPath, - DockerGroupID: dockerGroupID, - Image: image, + Database: dbConfig, + ListenAddr: listenAddr, + ListenPort: listenPort, + Username: username, + Password: password, + ListenURL: fmt.Sprintf("http://%s:%s", listenAddr, listenPort), + DBPath: mbDBPath, + DockerGroupID: dockerGroupID, + Image: image, + EnvironmentVariables: environmentVariables, }, } if err := metabase.Init(containerName, image); err != nil { From 5df56844d93902e5b4d1ad7ce259e0fede001caa Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 13 Jan 2025 13:28:48 +0100 Subject: [PATCH 12/25] log warning if local items have conflicting names (#3399) --- cmd/crowdsec-cli/clihub/items.go | 2 +- cmd/crowdsec-cli/dashboard.go | 4 +- pkg/cwhub/hub.go | 1 + pkg/cwhub/state.go | 11 +- pkg/cwhub/state_test.go | 1 + pkg/cwhub/sync.go | 390 +++++++++++++++++---------- test/bats/20_hub_items.bats | 21 +- test/bats/cscli-hubtype-install.bats | 32 +++ test/bats/cscli-hubtype-list.bats | 2 +- 9 files changed, 306 insertions(+), 158 deletions(-) diff --git a/cmd/crowdsec-cli/clihub/items.go b/cmd/crowdsec-cli/clihub/items.go index 730d2208be0..87cb10b1f93 100644 --- a/cmd/crowdsec-cli/clihub/items.go +++ b/cmd/crowdsec-cli/clihub/items.go @@ -63,7 +63,7 @@ func ListItems(out io.Writer, wantColor string, itemTypes []string, items map[st continue } - listHubItemTable(out, wantColor, "\n"+strings.ToUpper(itemType), items[itemType]) + listHubItemTable(out, wantColor, strings.ToUpper(itemType), items[itemType]) nothingToDisplay = false } diff --git a/cmd/crowdsec-cli/dashboard.go b/cmd/crowdsec-cli/dashboard.go index e138b8285c9..c3c974eb9b8 100644 --- a/cmd/crowdsec-cli/dashboard.go +++ b/cmd/crowdsec-cli/dashboard.go @@ -167,7 +167,9 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password if err = cli.chownDatabase(dockerGroup.Gid); err != nil { return err } - mb, err := metabase.SetupMetabase(cli.cfg().API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDBPath, dockerGroup.Gid, metabaseContainerID, metabaseImage, metabaseContainerEnvironmentVariables) + mb, err := metabase.SetupMetabase(cli.cfg().API.Server.DbConfig, metabaseListenAddress, + metabaseListenPort, metabaseUser, metabasePassword, metabaseDBPath, dockerGroup.Gid, + metabaseContainerID, metabaseImage, metabaseContainerEnvironmentVariables) if err != nil { return err } diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 998a4032359..aeccb3268f7 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -171,6 +171,7 @@ func (h *Hub) Update(ctx context.Context, indexProvider IndexProvider, withConte return nil } +// addItem adds an item to the hub. It silently replaces an existing item with the same type and name. func (h *Hub) addItem(item *Item) { if h.items[item.Type] == nil { h.items[item.Type] = make(map[string]*Item) diff --git a/pkg/cwhub/state.go b/pkg/cwhub/state.go index 518185aff1c..63a433151cd 100644 --- a/pkg/cwhub/state.go +++ b/pkg/cwhub/state.go @@ -7,10 +7,11 @@ import ( // ItemState is used to keep the local state (i.e. at runtime) of an item. // This data is not stored in the index, but is displayed with "cscli ... inspect". type ItemState struct { - LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` - LocalVersion string `json:"local_version,omitempty" yaml:"local_version,omitempty"` - LocalHash string `json:"local_hash,omitempty" yaml:"local_hash,omitempty"` - Installed bool `json:"installed"` + LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` + LocalVersion string `json:"local_version,omitempty" yaml:"local_version,omitempty"` + LocalHash string `json:"local_hash,omitempty" yaml:"local_hash,omitempty"` + Installed bool `json:"installed"` + local bool Downloaded bool `json:"downloaded"` UpToDate bool `json:"up_to_date"` Tainted bool `json:"tainted"` @@ -20,7 +21,7 @@ type ItemState struct { // IsLocal returns true if the item has been create by a user (not downloaded from the hub). func (s *ItemState) IsLocal() bool { - return s.Installed && !s.Downloaded + return s.local } // Text returns the status of the item as a string (eg. "enabled,update-available"). diff --git a/pkg/cwhub/state_test.go b/pkg/cwhub/state_test.go index 3ed3de16fcc..20741809ae2 100644 --- a/pkg/cwhub/state_test.go +++ b/pkg/cwhub/state_test.go @@ -40,6 +40,7 @@ func TestItemStateText(t *testing.T) { ItemState{ Installed: true, UpToDate: false, + local: true, Tainted: false, Downloaded: false, }, diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index ee8e49f2bf0..59c1383d7c2 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -50,9 +50,8 @@ func resolveSymlink(path string) (string, error) { } // isPathInside checks if a path is inside the given directory -// it can return false negatives if the filesystem is case insensitive func isPathInside(path, dir string) (bool, error) { - absFilePath, err := filepath.Abs(path) + absFile, err := filepath.Abs(path) if err != nil { return false, err } @@ -62,103 +61,145 @@ func isPathInside(path, dir string) (bool, error) { return false, err } - return strings.HasPrefix(absFilePath, absDir), nil -} + rel, err := filepath.Rel(absDir, absFile) + if err != nil { + return false, err + } -// information used to create a new Item, from a file path. -type itemFileInfo struct { - fname string - stage string - ftype string - fauthor string - inhub bool + return !strings.HasPrefix(rel, ".."), nil } -func (h *Hub) getItemFileInfo(path string, logger *logrus.Logger) (*itemFileInfo, error) { - var ret *itemFileInfo +// itemSpec contains some information needed to complete the items +// after they have been parsed from the index. itemSpecs are created by +// scanning the hub (/etc/crowdsec/hub/*) and install (/etc/crowdsec/*) directories. +// Only directories for the known types are scanned. +type itemSpec struct { + path string // full path to the file (or link) + fname string // name of the item: + // for local item, taken from the file content or defaults to the filename (including extension) + // for non-local items, always {author}/{name} + stage string // stage for parsers and overflows + ftype string // type, plural (collections, contexts etc.) + fauthor string // author - empty for local items + inhub bool // true if the spec comes from the hub dir + target string // the target of path if it's a link, otherwise == path + local bool // is this a spec for a local item? +} - hubDir := h.local.HubDir - installDir := h.local.InstallDir +func newHubItemSpec(path string, subs []string, logger *logrus.Logger) (*itemSpec, error) { + // .../hub/parsers/s00-raw/crowdsecurity/skip-pretag.yaml + // .../hub/scenarios/crowdsecurity/ssh_bf.yaml + // .../hub/profiles/crowdsecurity/linux.yaml + if len(subs) < 3 { + return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) + } - subsHub := relativePathComponents(path, hubDir) - subsInstall := relativePathComponents(path, installDir) + ftype := subs[0] + if !slices.Contains(ItemTypes, ftype) { + // this doesn't really happen anymore, because we only scan the {hubtype} directories + return nil, fmt.Errorf("unknown configuration type '%s'", ftype) + } - switch { - case len(subsHub) > 0: - logger.Tracef("in hub dir") + stage := "" + fauthor := subs[1] + fname := subs[2] - // .../hub/parsers/s00-raw/crowdsecurity/skip-pretag.yaml - // .../hub/scenarios/crowdsecurity/ssh_bf.yaml - // .../hub/profiles/crowdsecurity/linux.yaml - if len(subsHub) < 3 { - return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subsHub)) + if ftype == PARSERS || ftype == POSTOVERFLOWS { + if len(subs) < 4 { + return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) } - ftype := subsHub[0] - if !slices.Contains(ItemTypes, ftype) { - // this doesn't really happen anymore, because we only scan the {hubtype} directories - return nil, fmt.Errorf("unknown configuration type '%s'", ftype) - } + stage = subs[1] + fauthor = subs[2] + fname = subs[3] + } - stage := "" - fauthor := subsHub[1] - fname := subsHub[2] + spec := itemSpec{ + path: path, + inhub: true, + ftype: ftype, + stage: stage, + fauthor: fauthor, + fname: fname, + } - if ftype == PARSERS || ftype == POSTOVERFLOWS { - if len(subsHub) < 4 { - return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subsHub)) - } + return &spec, nil +} - stage = subsHub[1] - fauthor = subsHub[2] - fname = subsHub[3] - } +func newInstallItemSpec(path string, subs []string, logger *logrus.Logger) (*itemSpec, error) { + logger.Tracef("%s in install dir", path) - ret = &itemFileInfo{ - inhub: true, - ftype: ftype, - stage: stage, - fauthor: fauthor, - fname: fname, - } + // .../config/parser/stage/file.yaml + // .../config/postoverflow/stage/file.yaml + // .../config/scenarios/scenar.yaml + // .../config/collections/linux.yaml //file is empty + + if len(subs) < 2 { + return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) + } - case len(subsInstall) > 0: - logger.Tracef("in install dir") + // this can be in any number of subdirs, we join them to compose the item name - // .../config/parser/stage/file.yaml - // .../config/postoverflow/stage/file.yaml - // .../config/scenarios/scenar.yaml - // .../config/collections/linux.yaml //file is empty + ftype := subs[0] + stage := "" + fname := strings.Join(subs[1:], "/") - if len(subsInstall) < 2 { - return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subsInstall)) - } + if ftype == PARSERS || ftype == POSTOVERFLOWS { + stage = subs[1] + fname = strings.Join(subs[2:], "/") + } - // this can be in any number of subdirs, we join them to compose the item name + spec := itemSpec{ + path: path, + inhub: false, + ftype: ftype, + stage: stage, + fauthor: "", + fname: fname, + } - ftype := subsInstall[0] - stage := "" - fname := strings.Join(subsInstall[1:], "/") + return &spec, nil +} - if ftype == PARSERS || ftype == POSTOVERFLOWS { - stage = subsInstall[1] - fname = strings.Join(subsInstall[2:], "/") - } +func newItemSpec(path, hubDir, installDir string, logger *logrus.Logger) (*itemSpec, error) { + var ( + spec *itemSpec + err error + ) - ret = &itemFileInfo{ - inhub: false, - ftype: ftype, - stage: stage, - fauthor: "", - fname: fname, + if subs := relativePathComponents(path, hubDir); len(subs) > 0 { + spec, err = newHubItemSpec(path, subs, logger) + if err != nil { + return nil, err } - default: + } else if subs := relativePathComponents(path, installDir); len(subs) > 0 { + spec, err = newInstallItemSpec(path, subs, logger) + if err != nil { + return nil, err + } + } + + if spec == nil { return nil, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir) } - logger.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype) + // follow the link to see if it falls in the hub directory + // if it's not a link, target == path + spec.target, err = resolveSymlink(spec.path) + if err != nil { + // target does not exist, the user might have removed the file + // or switched to a hub branch without it; or symlink loop + return nil, err + } - return ret, nil + targetInHub, err := isPathInside(spec.target, hubDir) + if err != nil { + return nil, ErrSkipPath + } + + spec.local = !targetInHub + + return spec, nil } // sortedVersions returns the input data, sorted in reverse order (new, old) by semver. @@ -185,7 +226,7 @@ func sortedVersions(raw []string) ([]string, error) { return ret, nil } -func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) { +func newLocalItem(h *Hub, path string, spec *itemSpec) (*Item, error) { type localItemName struct { Name string `yaml:"name"` } @@ -194,12 +235,13 @@ func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) { item := &Item{ hub: h, - Name: info.fname, - Stage: info.stage, - Type: info.ftype, + Name: spec.fname, + Stage: spec.stage, + Type: spec.ftype, FileName: fileName, State: ItemState{ LocalPath: path, + local: true, Installed: true, UpToDate: true, }, @@ -225,22 +267,25 @@ func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) { return item, nil } -func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { +// A sentinel to skip regular files because "nil, nil" is ambiguous. Returning SkipDir with files would skip the rest of the directory. +var ErrSkipPath = errors.New("sentinel") + +func (h *Hub) itemVisit(path string, f os.DirEntry, err error) (*itemSpec, error) { if err != nil { h.logger.Debugf("while syncing hub dir: %s", err) // there is a path error, we ignore the file - return nil + return nil, ErrSkipPath + } + + // permission errors, files removed while reading, etc. + if f == nil { + return nil, ErrSkipPath } // only happens if the current working directory was removed (!) path, err = filepath.Abs(path) if err != nil { - return err - } - - // permission errors, files removed while reading, etc. - if f == nil { - return nil + return nil, err } if f.IsDir() { @@ -249,101 +294,125 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { // - double dot prefix is used by k8s to mount config maps if strings.HasPrefix(f.Name(), ".") { h.logger.Tracef("skipping hidden directory %s", path) - return filepath.SkipDir + return nil, filepath.SkipDir } // keep traversing - return nil + return nil, nil } // we only care about YAML files if !isYAMLFileName(f.Name()) { - return nil + return nil, ErrSkipPath } - info, err := h.getItemFileInfo(path, h.logger) + spec, err := newItemSpec(path, h.local.HubDir, h.local.InstallDir, h.logger) if err != nil { h.logger.Warningf("Ignoring file %s: %s", path, err) - return nil + return nil, ErrSkipPath } - // follow the link to see if it falls in the hub directory - // if it's not a link, target == path - target, err := resolveSymlink(path) - if err != nil { - // target does not exist, the user might have removed the file - // or switched to a hub branch without it; or symlink loop - h.logger.Warningf("Ignoring file %s: %s", path, err) - return nil - } - - targetInHub, err := isPathInside(target, h.local.HubDir) - if err != nil { - h.logger.Warningf("Ignoring file %s: %s", path, err) - return nil - } - - // local (custom) item if the file or link target is not inside the hub dir - if !targetInHub { - h.logger.Tracef("%s is a local file, skip", path) - - item, err := newLocalItem(h, path, info) - if err != nil { - return err - } - - h.addItem(item) + return spec, nil +} - return nil +func updateNonLocalItem(h *Hub, path string, spec *itemSpec, symlinkTarget string) (*Item, error) { + // look for the matching index entry + tot := 0 + for range h.GetItemMap(spec.ftype) { + tot++ } - hubpath := target - - // try to find which configuration item it is - h.logger.Tracef("check [%s] of %s", info.fname, info.ftype) - - for _, item := range h.GetItemMap(info.ftype) { - if info.fname != item.FileName { + for _, item := range h.GetItemMap(spec.ftype) { + if spec.fname != item.FileName { continue } - if item.Stage != info.stage { + if item.Stage != spec.stage { continue } - // if we are walking hub dir, just mark present files as downloaded - if info.inhub { + // Downloaded item, in the hub dir. + if spec.inhub { // not the item we're looking for - if !item.validPath(info.fauthor, info.fname) { + if !item.validPath(spec.fauthor, spec.fname) { continue } src, err := item.DownloadPath() if err != nil { - return err + return nil, err } - if path == src { + if spec.path == src { h.logger.Tracef("marking %s as downloaded", item.Name) item.State.Downloaded = true } - } else if !hasPathSuffix(hubpath, item.RemotePath) { + } else if !hasPathSuffix(symlinkTarget, item.RemotePath) { // wrong file // ///.yaml continue } - err := item.setVersionState(path, info.inhub) + err := item.setVersionState(spec.path, spec.inhub) + if err != nil { + return nil, err + } + + return item, nil + } + + return nil, nil +} + +// addItemFromSpec adds an item to the hub based on the spec, or updates it if already present. +// +// When the item is: +// +// Local - an itemSpec instance is created while scanning the install directory +// and an Item instance will be added to the hub.items map. +// +// Not downloaded, not installed - an Item instance is already on hub.items (decoded from index) and left untouched. +// +// Downloaded, not installed - an Item instance is on hub.items (decoded from index) and an itemSpec instance is created +// to complete it (i.e. set version and state flags). +// +// Downloaded, installed - an Item instance is on hub.items and is complemented with two itemSpecs: one from the file +// on the hub directory, one from the link in the install directory. +func (h *Hub) addItemFromSpec(spec *itemSpec) error { + var ( + item *Item + err error + ) + + // Local item: links outside the hub directory. + // We add it, or overwrite the existing one if it happened to have the same name. + if spec.local { + item, err = newLocalItem(h, spec.path, spec) if err != nil { return err } - h.pathIndex[path] = item + // we now have the name declared in the file (for local), + // see if there's another installed item of the same name + theOtherItem := h.GetItem(spec.ftype, item.Name) + if theOtherItem != nil { + if theOtherItem.State.Installed { + h.logger.Warnf("multiple %s named %s: ignoring %s", spec.ftype, item.Name, theOtherItem.State.LocalPath) + } + } + } else { + item, err = updateNonLocalItem(h, spec.path, spec, spec.target) + if err != nil { + return err + } + } + if item == nil { + h.logger.Infof("Ignoring file %s of type %s", spec.path, spec.ftype) return nil } - h.logger.Infof("Ignoring file %s of type %s", path, info.ftype) + h.addItem(item) return nil } @@ -411,6 +480,8 @@ func (i *Item) checkSubItemVersions() []string { // syncDir scans a directory for items, and updates the Hub state accordingly. func (h *Hub) syncDir(dir string) error { + specs := []*itemSpec{} + // For each, scan PARSERS, POSTOVERFLOWS... and COLLECTIONS last for _, scan := range ItemTypes { // cpath: top-level item directory, either downloaded or installed items. @@ -423,11 +494,46 @@ func (h *Hub) syncDir(dir string) error { // explicit check for non existing directory, avoid spamming log.Debug if _, err = os.Stat(cpath); os.IsNotExist(err) { - h.logger.Tracef("directory %s doesn't exist, skipping", cpath) continue } - if err = filepath.WalkDir(cpath, h.itemVisit); err != nil { + // wrap itemVisit to collect spec results + specCollector := func(path string, f os.DirEntry, err error) error { + spec, err := h.itemVisit(path, f, err) + if err == nil && spec != nil { + specs = append(specs, spec) + } + + if errors.Is(err, ErrSkipPath) { + return nil + } + + return err + } + + if err = filepath.WalkDir(cpath, specCollector); err != nil { + return err + } + } + + // add non-local items first, so they can find the place in the index + // before it's overridden by local items in case of name collision + for _, spec := range specs { + if spec.local { + continue + } + + if err := h.addItemFromSpec(spec); err != nil { + return err + } + } + + for _, spec := range specs { + if !spec.local { + continue + } + + if err := h.addItemFromSpec(spec); err != nil { return err } } @@ -463,14 +569,16 @@ func removeDuplicates(sl []string) []string { // localSync updates the hub state with downloaded, installed and local items. func (h *Hub) localSync() error { - if err := h.syncDir(h.local.InstallDir); err != nil { - return fmt.Errorf("failed to sync %s: %w", h.local.InstallDir, err) - } - + // add downloaded files first, so they can find the place in the index + // before it's overridden by local items in case of name collision if err := h.syncDir(h.local.HubDir); err != nil { return fmt.Errorf("failed to sync %s: %w", h.local.HubDir, err) } + if err := h.syncDir(h.local.InstallDir); err != nil { + return fmt.Errorf("failed to sync %s: %w", h.local.InstallDir, err) + } + warnings := make([]string, 0) for _, item := range h.GetItemMap(COLLECTIONS) { diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats index 8ebe505c6e1..4ddaf387488 100644 --- a/test/bats/20_hub_items.bats +++ b/test/bats/20_hub_items.bats @@ -80,8 +80,8 @@ teardown() { rune -0 cscli collections install crowdsecurity/sshd rune -1 cscli collections inspect crowdsecurity/sshd --no-metrics - # XXX: this must be triggered during parse, not sync - assert_stderr "Error: failed to sync $CONFIG_DIR: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version" + # XXX: it would be better to trigger this during parse, not sync + assert_stderr "Error: failed to sync $HUB_DIR: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version" } @test "removing or purging an item already removed by hand" { @@ -99,13 +99,6 @@ teardown() { rune -0 cscli parsers remove crowdsecurity/syslog-logs --purge assert_output "Nothing to do." - - rune -0 cscli parsers remove --all --error --purge --force - assert_output "Nothing to do." - refute_stderr - rune -0 cscli collections remove --all --error --purge --force - assert_output "Nothing to do." - refute_stderr } @test "a local item is not tainted" { @@ -199,6 +192,16 @@ teardown() { assert_json '[]' } +@test "replacing a symlink with a regular file makes a local item" { + rune -0 cscli parsers install crowdsecurity/caddy-logs + rune -0 rm "$CONFIG_DIR/parsers/s01-parse/caddy-logs.yaml" + rune -0 cp "$HUB_DIR/parsers/s01-parse/crowdsecurity/caddy-logs.yaml" "$CONFIG_DIR/parsers/s01-parse/caddy-logs.yaml" + rune -0 cscli hub list + rune -0 cscli parsers inspect crowdsecurity/caddy-logs -o json + rune -0 jq -e '[.tainted,.local,.local_version==false,true,"?"]' <(output) + refute_stderr +} + @test "tainted hub file, not enabled, install --force should repair" { rune -0 cscli scenarios install crowdsecurity/ssh-bf rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json diff --git a/test/bats/cscli-hubtype-install.bats b/test/bats/cscli-hubtype-install.bats index 2304e5a72cc..58c16dd968d 100644 --- a/test/bats/cscli-hubtype-install.bats +++ b/test/bats/cscli-hubtype-install.bats @@ -267,3 +267,35 @@ teardown() { assert_line 'enabling contexts:crowdsecurity/bf_base' assert_line 'enabling collections:crowdsecurity/sshd' } + +@test "a local item can override an official one, if it's not installed" { + mkdir -p "$CONFIG_DIR/parsers/s02-enrich" + rune -0 cscli parsers install crowdsecurity/whitelists --download-only + echo "name: crowdsecurity/whitelists" > "$CONFIG_DIR/parsers/s02-enrich/hi.yaml" + # no warning + rune -0 cscli parsers list + refute_stderr + rune -0 cscli parsers list -o json + rune -0 jq -e '.installed,.local==true,true' <(output) +} + +@test "conflicting item names: local and non local - the local one has priority" { + mkdir -p "$CONFIG_DIR/parsers/s02-enrich" + rune -0 cscli parsers install crowdsecurity/whitelists + echo "name: crowdsecurity/whitelists" > "$CONFIG_DIR/parsers/s02-enrich/hi.yaml" + rune -0 cscli parsers list -o json + rune -0 jq -e '.installed,.local==true,true' <(output) + rune -0 cscli parsers list + assert_stderr --partial "multiple parsers named crowdsecurity/whitelists: ignoring $CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" +} + +@test "conflicting item names: both local, the last one wins" { + mkdir -p "$CONFIG_DIR/parsers/s02-enrich" + echo "name: crowdsecurity/whitelists" > "$CONFIG_DIR/parsers/s02-enrich/one.yaml" + echo "name: crowdsecurity/whitelists" > "$CONFIG_DIR/parsers/s02-enrich/two.yaml" + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -r '.local_path' <(output) + assert_output --partial "/parsers/s02-enrich/two.yaml" + rune -0 cscli parsers list + assert_stderr --partial "multiple parsers named crowdsecurity/whitelists: ignoring $CONFIG_DIR/parsers/s02-enrich/one.yaml" +} diff --git a/test/bats/cscli-hubtype-list.bats b/test/bats/cscli-hubtype-list.bats index b3db4743eb9..14113650c74 100644 --- a/test/bats/cscli-hubtype-list.bats +++ b/test/bats/cscli-hubtype-list.bats @@ -80,7 +80,7 @@ teardown() { # the list should be the same in all formats, and sorted (not case sensitive) list_raw=$(cscli parsers list -o raw -a | tail -n +2 | cut -d, -f1) - list_human=$(cscli parsers list -o human -a | tail -n +7 | head -n -1 | cut -d' ' -f2) + list_human=$(cscli parsers list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) list_json=$(cscli parsers list -o json -a | jq -r '.parsers[].name') # use python to sort because it handles "_" like go From cc1196c3ad0741e3a8626987ee00081d098b3c57 Mon Sep 17 00:00:00 2001 From: Laurence Jones Date: Tue, 14 Jan 2025 10:48:05 +0000 Subject: [PATCH 13/25] fix: #2790 (#3378) --- cmd/crowdsec/pour.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/crowdsec/pour.go b/cmd/crowdsec/pour.go index 2fc7d7e42c9..4c83b65bd48 100644 --- a/cmd/crowdsec/pour.go +++ b/cmd/crowdsec/pour.go @@ -46,7 +46,7 @@ func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *lea // here we can bucketify with parsed poured, err := leaky.PourItemToHolders(parsed, holders, buckets) if err != nil { - log.Errorf("bucketify failed for: %v", parsed) + log.Errorf("bucketify failed for: %v with %s", parsed, err) continue } From 9ef5f58f885eb35d0759f73cf0d5caba73ea3dd2 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Wed, 15 Jan 2025 12:13:54 +0100 Subject: [PATCH 14/25] test pkg/exprhelpers: explicit message if the tag "expr_debug" is missing (#3400) * test pkg/exprhelpers: explicit message if the tag "expr_debug" is missing * typo * lint: use build tag expr_debug while linting * lint --- .github/workflows/go-tests.yml | 4 +- .golangci.yml | 4 + pkg/exprhelpers/debugger.go | 112 +++++++++++++++++++-------- pkg/exprhelpers/debugger_test.go | 1 + pkg/exprhelpers/debuggerstub_test.go | 10 +++ 5 files changed, 95 insertions(+), 36 deletions(-) create mode 100644 pkg/exprhelpers/debuggerstub_test.go diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 3a194e1084a..d882f88580e 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -143,11 +143,11 @@ jobs: go generate ./... protoc --version if [[ $(git status --porcelain) ]]; then - echo "Error: Uncommitted changes found after running 'make generate'. Please commit all generated code." + echo "Error: Uncommitted changes found after running 'go generate'. Please commit all generated code." git diff exit 1 else - echo "No changes detected after running 'make generate'." + echo "No changes detected after running 'go generate'." fi - name: Create localstack streams diff --git a/.golangci.yml b/.golangci.yml index deb073c2eea..5995f14c512 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,5 +1,9 @@ # https://github.com/golangci/golangci-lint/blob/master/.golangci.reference.yml +run: + build-tags: + - expr_debug + linters-settings: gci: sections: diff --git a/pkg/exprhelpers/debugger.go b/pkg/exprhelpers/debugger.go index 2e47af6d1de..d44b8fc97e1 100644 --- a/pkg/exprhelpers/debugger.go +++ b/pkg/exprhelpers/debugger.go @@ -21,35 +21,35 @@ var IndentStep = 4 // we use this struct to store the output of the expr runtime type OpOutput struct { - Code string //relevant code part + Code string // relevant code part - CodeDepth int //level of nesting + CodeDepth int // level of nesting BlockStart bool BlockEnd bool - Func bool //true if it's a function call + Func bool // true if it's a function call FuncName string Args []string FuncResults []string // - Comparison bool //true if it's a comparison + Comparison bool // true if it's a comparison Negated bool Left string Right string // - JumpIf bool //true if it's conditional jump + JumpIf bool // true if it's conditional jump IfTrue bool IfFalse bool // - Condition bool //true if it's a condition + Condition bool // true if it's a condition ConditionIn bool ConditionContains bool - //used for comparisons, conditional jumps and conditions + // used for comparisons, conditional jumps and conditions StrConditionResult string - ConditionResult *bool //should always be present for conditions + ConditionResult *bool // should always be present for conditions // - Finalized bool //used when a node is finalized, we already fetched result from next OP + Finalized bool // used when a node is finalized, we already fetched result from next OP } func (o *OpOutput) String() string { @@ -57,6 +57,7 @@ func (o *OpOutput) String() string { if o.Code != "" { ret += fmt.Sprintf("[%s]", o.Code) } + ret += " " switch { @@ -68,19 +69,24 @@ func (o *OpOutput) String() string { if indent < 0 { indent = 0 } + ret = fmt.Sprintf("%*cBLOCK_END [%s]", indent, ' ', o.Code) + if o.StrConditionResult != "" { ret += fmt.Sprintf(" -> %s", o.StrConditionResult) } + return ret - //A block end can carry a value, for example if it's a count, any, all etc. XXX + // A block end can carry a value, for example if it's a count, any, all etc. XXX case o.Func: return ret + fmt.Sprintf("%s(%s) = %s", o.FuncName, strings.Join(o.Args, ", "), strings.Join(o.FuncResults, ", ")) case o.Comparison: if o.Negated { ret += "NOT " } + ret += fmt.Sprintf("%s == %s -> %s", o.Left, o.Right, o.StrConditionResult) + return ret case o.ConditionIn: return ret + fmt.Sprintf("%s in %s -> %s", o.Args[0], o.Args[1], o.StrConditionResult) @@ -91,18 +97,23 @@ func (o *OpOutput) String() string { if *o.ConditionResult { return ret + "OR -> false" } + return ret + "OR -> true" } + return ret + "OR(?)" case o.JumpIf && o.IfFalse: if o.ConditionResult != nil { if *o.ConditionResult { return ret + "AND -> true" } + return ret + "AND -> false" } + return ret + "AND(?)" } + return ret + "" } @@ -135,7 +146,7 @@ func (erp ExprRuntimeDebug) extractCode(ip int, program *vm.Program) string { func autoQuote(v any) string { switch x := v.(type) { case string: - //let's avoid printing long strings. it can happen ie. when we are debugging expr with `File()` or similar helpers + // let's avoid printing long strings. it can happen ie. when we are debugging expr with `File()` or similar helpers if len(x) > 40 { return fmt.Sprintf("%q", x[:40]+"...") } else { @@ -147,35 +158,40 @@ func autoQuote(v any) string { } func (erp ExprRuntimeDebug) ipDebug(ip int, vm *vm.VM, program *vm.Program, parts []string, outputs []OpOutput) ([]OpOutput, error) { - IdxOut := len(outputs) prevIdxOut := 0 currentDepth := 0 - //when there is a function call or comparison, we need to wait for the next instruction to get the result and "finalize" the previous one + // when there is a function call or comparison, we need to wait for the next instruction to get the result and "finalize" the previous one if IdxOut > 0 { prevIdxOut = IdxOut - 1 currentDepth = outputs[prevIdxOut].CodeDepth + if outputs[prevIdxOut].Func && !outputs[prevIdxOut].Finalized { stack := vm.Stack num_items := 1 + for i := len(stack) - 1; i >= 0 && num_items > 0; i-- { outputs[prevIdxOut].FuncResults = append(outputs[prevIdxOut].FuncResults, autoQuote(stack[i])) num_items-- } + outputs[prevIdxOut].Finalized = true } else if (outputs[prevIdxOut].Comparison || outputs[prevIdxOut].Condition) && !outputs[prevIdxOut].Finalized { stack := vm.Stack outputs[prevIdxOut].StrConditionResult = fmt.Sprintf("%+v", stack) + if val, ok := stack[0].(bool); ok { outputs[prevIdxOut].ConditionResult = new(bool) *outputs[prevIdxOut].ConditionResult = val } + outputs[prevIdxOut].Finalized = true } } erp.Logger.Tracef("[STEP %d:%s] (stack:%+v) (parts:%+v) {depth:%d}", ip, parts[1], vm.Stack, parts, currentDepth) + out := OpOutput{} out.CodeDepth = currentDepth out.Code = erp.extractCode(ip, program) @@ -188,27 +204,28 @@ func (erp ExprRuntimeDebug) ipDebug(ip int, vm *vm.VM, program *vm.Program, part case "OpEnd": out.CodeDepth -= IndentStep out.BlockEnd = true - //OpEnd can carry value, if it's any/all/count etc. + // OpEnd can carry value, if it's any/all/count etc. if len(vm.Stack) > 0 { out.StrConditionResult = fmt.Sprintf("%v", vm.Stack) } + outputs = append(outputs, out) case "OpNot": - //negate the previous condition + // negate the previous condition outputs[prevIdxOut].Negated = true - case "OpTrue": //generated when possible ? (1 == 1) + case "OpTrue": // generated when possible ? (1 == 1) out.Condition = true out.ConditionResult = new(bool) *out.ConditionResult = true out.StrConditionResult = "true" outputs = append(outputs, out) - case "OpFalse": //generated when possible ? (1 != 1) + case "OpFalse": // generated when possible ? (1 != 1) out.Condition = true out.ConditionResult = new(bool) *out.ConditionResult = false out.StrConditionResult = "false" outputs = append(outputs, out) - case "OpJumpIfTrue": //OR + case "OpJumpIfTrue": // OR stack := vm.Stack out.JumpIf = true out.IfTrue = true @@ -218,78 +235,88 @@ func (erp ExprRuntimeDebug) ipDebug(ip int, vm *vm.VM, program *vm.Program, part out.ConditionResult = new(bool) *out.ConditionResult = val } + outputs = append(outputs, out) - case "OpJumpIfFalse": //AND + case "OpJumpIfFalse": // AND stack := vm.Stack out.JumpIf = true out.IfFalse = true out.StrConditionResult = fmt.Sprintf("%v", stack[0]) + if val, ok := stack[0].(bool); ok { out.ConditionResult = new(bool) *out.ConditionResult = val } + outputs = append(outputs, out) - case "OpCall1": //Op for function calls + case "OpCall1": // Op for function calls out.Func = true out.FuncName = parts[3] stack := vm.Stack + num_items := 1 for i := len(stack) - 1; i >= 0 && num_items > 0; i-- { out.Args = append(out.Args, autoQuote(stack[i])) num_items-- } + outputs = append(outputs, out) - case "OpCall2": //Op for function calls + case "OpCall2": // Op for function calls out.Func = true out.FuncName = parts[3] stack := vm.Stack + num_items := 2 for i := len(stack) - 1; i >= 0 && num_items > 0; i-- { out.Args = append(out.Args, autoQuote(stack[i])) num_items-- } + outputs = append(outputs, out) - case "OpCall3": //Op for function calls + case "OpCall3": // Op for function calls out.Func = true out.FuncName = parts[3] stack := vm.Stack + num_items := 3 for i := len(stack) - 1; i >= 0 && num_items > 0; i-- { out.Args = append(out.Args, autoQuote(stack[i])) num_items-- } + outputs = append(outputs, out) - //double check OpCallFast and OpCallTyped + // double check OpCallFast and OpCallTyped case "OpCallFast", "OpCallTyped": // - case "OpCallN": //Op for function calls with more than 3 args + case "OpCallN": // Op for function calls with more than 3 args out.Func = true out.FuncName = parts[1] stack := vm.Stack - //for OpCallN, we get the number of args + // for OpCallN, we get the number of args if len(program.Arguments) >= ip { nb_args := program.Arguments[ip] if nb_args > 0 { - //we need to skip the top item on stack + // we need to skip the top item on stack for i := len(stack) - 2; i >= 0 && nb_args > 0; i-- { out.Args = append(out.Args, autoQuote(stack[i])) nb_args-- } } - } else { //let's blindly take the items on stack + } else { // let's blindly take the items on stack for _, val := range vm.Stack { out.Args = append(out.Args, autoQuote(val)) } } + outputs = append(outputs, out) - case "OpEqualString", "OpEqual", "OpEqualInt": //comparisons + case "OpEqualString", "OpEqual", "OpEqualInt": // comparisons stack := vm.Stack out.Comparison = true out.Left = autoQuote(stack[0]) out.Right = autoQuote(stack[1]) outputs = append(outputs, out) - case "OpIn": //in operator + case "OpIn": // in operator stack := vm.Stack out.Condition = true out.ConditionIn = true @@ -299,7 +326,7 @@ func (erp ExprRuntimeDebug) ipDebug(ip int, vm *vm.VM, program *vm.Program, part out.Args = append(out.Args, autoQuote(stack[0])) out.Args = append(out.Args, autoQuote(stack[1])) outputs = append(outputs, out) - case "OpContains": //kind OpIn , but reverse + case "OpContains": // kind OpIn , but reverse stack := vm.Stack out.Condition = true out.ConditionContains = true @@ -310,6 +337,7 @@ func (erp ExprRuntimeDebug) ipDebug(ip int, vm *vm.VM, program *vm.Program, part out.Args = append(out.Args, autoQuote(stack[1])) outputs = append(outputs, out) } + return outputs, nil } @@ -319,10 +347,12 @@ func (erp ExprRuntimeDebug) ipSeek(ip int) []string { if len(parts) == 0 { continue } + if parts[0] == strconv.Itoa(ip) { return parts } } + return nil } @@ -330,19 +360,23 @@ func Run(program *vm.Program, env interface{}, logger *log.Entry, debug bool) (a if debug { dbgInfo, ret, err := RunWithDebug(program, env, logger) DisplayExprDebug(program, dbgInfo, logger, ret) + return ret, err } + return expr.Run(program, env) } func cleanTextForDebug(text string) string { text = strings.Join(strings.Fields(text), " ") text = strings.Trim(text, " \t\n") + return text } func DisplayExprDebug(program *vm.Program, outputs []OpOutput, logger *log.Entry, ret any) { logger.Debugf("dbg(result=%v): %s", ret, cleanTextForDebug(string(program.Source()))) + for _, output := range outputs { logger.Debugf("%s", output.String()) } @@ -360,46 +394,55 @@ func RunWithDebug(program *vm.Program, env interface{}, logger *log.Entry) ([]Op erp.Lines = lines go func() { - //We must never return until the execution of the program is done + // We must never return until the execution of the program is done var err error + erp.Logger.Tracef("[START] ip 0") + ops := erp.ipSeek(0) if ops == nil { log.Warningf("error while debugging expr: failed getting ops for ip 0") } + if outputs, err = erp.ipDebug(0, vm, program, ops, outputs); err != nil { log.Warningf("error while debugging expr: error while debugging at ip 0") } + vm.Step() + for ip := range vm.Position() { ops := erp.ipSeek(ip) if ops == nil { erp.Logger.Tracef("[DONE] ip %d", ip) break } + if outputs, err = erp.ipDebug(ip, vm, program, ops, outputs); err != nil { log.Warningf("error while debugging expr: error while debugging at ip %d", ip) } + vm.Step() } }() var return_error error + ret, err := vm.Run(program, env) - //if the expr runtime failed, we don't need to wait for the debug to finish + // if the expr runtime failed, we don't need to wait for the debug to finish if err != nil { return_error = err } - //the overall result of expression is the result of last op ? + // the overall result of expression is the result of last op ? if len(outputs) > 0 { lastOutIdx := len(outputs) if lastOutIdx > 0 { lastOutIdx -= 1 } + switch val := ret.(type) { case bool: log.Tracef("completing with bool %t", ret) - //if outputs[lastOutIdx].Comparison { + // if outputs[lastOutIdx].Comparison { outputs[lastOutIdx].StrConditionResult = fmt.Sprintf("%v", ret) outputs[lastOutIdx].ConditionResult = new(bool) *outputs[lastOutIdx].ConditionResult = val @@ -412,5 +455,6 @@ func RunWithDebug(program *vm.Program, env interface{}, logger *log.Entry) ([]Op } else { log.Tracef("no output from expr runtime") } + return outputs, ret, return_error } diff --git a/pkg/exprhelpers/debugger_test.go b/pkg/exprhelpers/debugger_test.go index 32144454084..0852d7ab2de 100644 --- a/pkg/exprhelpers/debugger_test.go +++ b/pkg/exprhelpers/debugger_test.go @@ -1,3 +1,4 @@ +//go:build expr_debug package exprhelpers import ( diff --git a/pkg/exprhelpers/debuggerstub_test.go b/pkg/exprhelpers/debuggerstub_test.go new file mode 100644 index 00000000000..cc41c793b47 --- /dev/null +++ b/pkg/exprhelpers/debuggerstub_test.go @@ -0,0 +1,10 @@ +//go:build !expr_debug +package exprhelpers + +import ( + "testing" +) + +func TestFailWithoutExprDebug(t *testing.T) { + t.Fatal("To test pkg/exprhelpers, you need the expr_debug build tag") +} From 65292157757730f00705f319e5d41de2edb31cdf Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 16 Jan 2025 13:22:08 +0100 Subject: [PATCH 15/25] CI: golangci-lint 1.63 (#3396) --- .github/workflows/go-tests-windows.yml | 2 +- .github/workflows/go-tests.yml | 2 +- .golangci.yml | 11 +++++++++++ pkg/csplugin/listfiles_test.go | 19 +++++++++++-------- pkg/database/database.go | 2 +- pkg/exprhelpers/exprlib_test.go | 6 +----- pkg/fflag/features_test.go | 4 +--- pkg/setup/detect_test.go | 16 ++++++---------- 8 files changed, 33 insertions(+), 29 deletions(-) diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml index 3276dbb1bfd..44abbbe24a3 100644 --- a/.github/workflows/go-tests-windows.yml +++ b/.github/workflows/go-tests-windows.yml @@ -61,6 +61,6 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.62 + version: v1.63 args: --issues-exit-code=1 --timeout 10m only-new-issues: false diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index d882f88580e..649c47ebd26 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -189,6 +189,6 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.62 + version: v1.63 args: --issues-exit-code=1 --timeout 10m only-new-issues: false diff --git a/.golangci.yml b/.golangci.yml index 5995f14c512..12d35ed8737 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -460,3 +460,14 @@ issues: - revive path: "pkg/types/utils.go" text: "argument-limit: .*" + + # need some cleanup first: to create db in memory and share the client, not the config + - linters: + - usetesting + path: "pkg/apiserver/(.+)_test.go" + text: "os.MkdirTemp.* could be replaced by t.TempDir.*" + + - linters: + - usetesting + path: "pkg/apiserver/(.+)_test.go" + text: "os.CreateTemp.* could be replaced by os.CreateTemp.*" diff --git a/pkg/csplugin/listfiles_test.go b/pkg/csplugin/listfiles_test.go index c476d7a4e4a..32269f3f5f1 100644 --- a/pkg/csplugin/listfiles_test.go +++ b/pkg/csplugin/listfiles_test.go @@ -12,19 +12,22 @@ import ( ) func TestListFilesAtPath(t *testing.T) { - dir, err := os.MkdirTemp("", "test-listfiles") - require.NoError(t, err) - t.Cleanup(func() { - os.RemoveAll(dir) - }) - _, err = os.Create(filepath.Join(dir, "notification-gitter")) + dir := t.TempDir() + + f, err := os.Create(filepath.Join(dir, "notification-gitter")) require.NoError(t, err) - _, err = os.Create(filepath.Join(dir, "slack")) + require.NoError(t, f.Close()) + + f, err = os.Create(filepath.Join(dir, "slack")) require.NoError(t, err) + require.NoError(t, f.Close()) + err = os.Mkdir(filepath.Join(dir, "somedir"), 0o755) require.NoError(t, err) - _, err = os.Create(filepath.Join(dir, "somedir", "inner")) + + f, err = os.Create(filepath.Join(dir, "somedir", "inner")) require.NoError(t, err) + require.NoError(t, f.Close()) tests := []struct { name string diff --git a/pkg/database/database.go b/pkg/database/database.go index bb41dd3b645..80479710751 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -68,7 +68,7 @@ func NewClient(ctx context.Context, config *csconfig.DatabaseCfg) (*Client, erro return nil, err // unsupported database caught here } - if config.Type == "sqlite" { + if config.Type == "sqlite" && config.DbPath != ":memory:" { /*if it's the first startup, we want to touch and chmod file*/ if _, err = os.Stat(config.DbPath); os.IsNotExist(err) { f, err := os.OpenFile(config.DbPath, os.O_CREATE|os.O_RDWR, 0o600) diff --git a/pkg/exprhelpers/exprlib_test.go b/pkg/exprhelpers/exprlib_test.go index f2eb208ebfa..932db4b7da4 100644 --- a/pkg/exprhelpers/exprlib_test.go +++ b/pkg/exprhelpers/exprlib_test.go @@ -3,7 +3,6 @@ package exprhelpers import ( "context" "errors" - "os" "testing" "time" @@ -26,15 +25,12 @@ const TestFolder = "tests" func getDBClient(t *testing.T) *database.Client { t.Helper() - dbPath, err := os.CreateTemp("", "*sqlite") - require.NoError(t, err) - ctx := context.Background() testDBClient, err := database.NewClient(ctx, &csconfig.DatabaseCfg{ Type: "sqlite", DbName: "crowdsec", - DbPath: dbPath.Name(), + DbPath: ":memory:", }) require.NoError(t, err) diff --git a/pkg/fflag/features_test.go b/pkg/fflag/features_test.go index 144e7049362..bf8ddeca8fd 100644 --- a/pkg/fflag/features_test.go +++ b/pkg/fflag/features_test.go @@ -351,11 +351,9 @@ func TestSetFromYaml(t *testing.T) { } func TestSetFromYamlFile(t *testing.T) { - tmpfile, err := os.CreateTemp("", "test") + tmpfile, err := os.CreateTemp(t.TempDir(), "test") require.NoError(t, err) - defer os.Remove(tmpfile.Name()) - // write the config file _, err = tmpfile.WriteString("- experimental1") require.NoError(t, err) diff --git a/pkg/setup/detect_test.go b/pkg/setup/detect_test.go index 475f3af0928..72356bc1924 100644 --- a/pkg/setup/detect_test.go +++ b/pkg/setup/detect_test.go @@ -60,9 +60,14 @@ func TestSetupHelperProcess(t *testing.T) { func tempYAML(t *testing.T, content string) os.File { t.Helper() require := require.New(t) - file, err := os.CreateTemp("", "") + file, err := os.CreateTemp(t.TempDir(), "") require.NoError(err) + t.Cleanup(func() { + require.NoError(file.Close()) + require.NoError(os.Remove(file.Name())) + }) + _, err = file.WriteString(dedent.Dedent(content)) require.NoError(err) @@ -249,7 +254,6 @@ func TestListSupported(t *testing.T) { t.Parallel() f := tempYAML(t, tc.yml) - defer os.Remove(f.Name()) supported, err := setup.ListSupported(&f) cstest.RequireErrorContains(t, err, tc.expectedErr) @@ -375,7 +379,6 @@ func TestDetectSimpleRule(t *testing.T) { - false ugly: `) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{}) require.NoError(err) @@ -421,7 +424,6 @@ detect: for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { f := tempYAML(t, tc.config) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{}) cstest.RequireErrorContains(t, err, tc.expectedErr) @@ -514,7 +516,6 @@ detect: for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { f := tempYAML(t, tc.config) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{}) cstest.RequireErrorContains(t, err, tc.expectedErr) @@ -542,7 +543,6 @@ func TestDetectForcedUnit(t *testing.T) { journalctl_filter: - _SYSTEMD_UNIT=crowdsec-setup-forced.service `) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{ForcedUnits: []string{"crowdsec-setup-forced.service"}}) require.NoError(err) @@ -580,7 +580,6 @@ func TestDetectForcedProcess(t *testing.T) { when: - ProcessRunning("foobar") `) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{ForcedProcesses: []string{"foobar"}}) require.NoError(err) @@ -610,7 +609,6 @@ func TestDetectSkipService(t *testing.T) { when: - ProcessRunning("foobar") `) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{ForcedProcesses: []string{"foobar"}, SkipServices: []string{"wizard"}}) require.NoError(err) @@ -825,7 +823,6 @@ func TestDetectForcedOS(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { f := tempYAML(t, tc.config) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{ForcedOS: tc.forced}) cstest.RequireErrorContains(t, err, tc.expectedErr) @@ -1009,7 +1006,6 @@ func TestDetectDatasourceValidation(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { f := tempYAML(t, tc.config) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{}) cstest.RequireErrorContains(t, err, tc.expectedErr) require.Equal(tc.expected, detected) From b582730d061c10fbafc055f36998b3b5b6d1e1ae Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 16 Jan 2025 14:02:39 +0100 Subject: [PATCH 16/25] remove commands "cscli config backup/restore" (#3158) * remove "cscli config backup/restore" * remove backup/restore calls from deb,rpm pre/postinst and wizard * remove from prerm too --- cmd/crowdsec-cli/config_backup.go | 224 +---------------------- cmd/crowdsec-cli/config_restore.go | 275 +---------------------------- cmd/crowdsec-cli/copyfile.go | 82 --------- debian/postinst | 16 -- debian/preinst | 35 ---- debian/prerm | 3 +- rpm/SPECS/crowdsec.spec | 24 +-- test/bats/01_cscli.bats | 36 +--- test/bats/02_nolapi.bats | 12 -- test/bats/03_noagent.bats | 12 -- test/bats/04_nocapi.bats | 11 -- wizard.sh | 66 ++----- 12 files changed, 31 insertions(+), 765 deletions(-) delete mode 100644 cmd/crowdsec-cli/copyfile.go diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index faac786ebdc..0a58a8c1ab3 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -1,234 +1,18 @@ package main import ( - "encoding/json" - "errors" "fmt" - "os" - "path/filepath" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func (cli *cliConfig) backupHub(dirPath string) error { - hub, err := require.Hub(cli.cfg(), nil) - if err != nil { - return err - } - - for _, itemType := range cwhub.ItemTypes { - clog := log.WithField("type", itemType) - - itemMap := hub.GetItemMap(itemType) - if itemMap == nil { - clog.Infof("No %s to backup.", itemType) - continue - } - - itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itemType) - if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil { - return fmt.Errorf("error while creating %s: %w", itemDirectory, err) - } - - upstreamParsers := []string{} - - for k, v := range itemMap { - clog = clog.WithField("file", v.Name) - if !v.State.Installed { // only backup installed ones - clog.Debugf("[%s]: not installed", k) - continue - } - - // for the local/tainted ones, we back up the full file - if v.State.Tainted || v.State.IsLocal() || !v.State.UpToDate { - // we need to backup stages for parsers - if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS { - fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage) - if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil { - return fmt.Errorf("error while creating stage dir %s: %w", fstagedir, err) - } - } - - clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.State.IsLocal(), v.State.UpToDate) - - tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) - if err = CopyFile(v.State.LocalPath, tfile); err != nil { - return fmt.Errorf("failed copy %s %s to %s: %w", itemType, v.State.LocalPath, tfile, err) - } - - clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile) - - continue - } - - clog.Debugf("[%s]: from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate) - clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate) - upstreamParsers = append(upstreamParsers, v.Name) - } - // write the upstream items - upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType) - - upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ") - if err != nil { - return fmt.Errorf("failed to serialize upstream parsers: %w", err) - } - - err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0o644) - if err != nil { - return fmt.Errorf("unable to write to %s %s: %w", itemType, upstreamParsersFname, err) - } - - clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname) - } - - return nil -} - -/* - Backup crowdsec configurations to directory : - -- Main config (config.yaml) -- Profiles config (profiles.yaml) -- Simulation config (simulation.yaml) -- Backup of API credentials (local API and online API) -- List of scenarios, parsers, postoverflows and collections that are up-to-date -- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections -- Acquisition files (acquis.yaml, acquis.d/*.yaml) -*/ -func (cli *cliConfig) backup(dirPath string) error { - var err error - - cfg := cli.cfg() - - if dirPath == "" { - return errors.New("directory path can't be empty") - } - - log.Infof("Starting configuration backup") - - /*if parent directory doesn't exist, bail out. create final dir with Mkdir*/ - parentDir := filepath.Dir(dirPath) - if _, err = os.Stat(parentDir); err != nil { - return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err) - } - - if err = os.Mkdir(dirPath, 0o700); err != nil { - return fmt.Errorf("while creating %s: %w", dirPath, err) - } - - if cfg.ConfigPaths.SimulationFilePath != "" { - backupSimulation := filepath.Join(dirPath, "simulation.yaml") - if err = CopyFile(cfg.ConfigPaths.SimulationFilePath, backupSimulation); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", cfg.ConfigPaths.SimulationFilePath, backupSimulation, err) - } - - log.Infof("Saved simulation to %s", backupSimulation) - } - - /* - - backup AcquisitionFilePath - - backup the other files of acquisition directory - */ - if cfg.Crowdsec != nil && cfg.Crowdsec.AcquisitionFilePath != "" { - backupAcquisition := filepath.Join(dirPath, "acquis.yaml") - if err = CopyFile(cfg.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", cfg.Crowdsec.AcquisitionFilePath, backupAcquisition, err) - } - } - - acquisBackupDir := filepath.Join(dirPath, "acquis") - if err = os.Mkdir(acquisBackupDir, 0o700); err != nil { - return fmt.Errorf("error while creating %s: %w", acquisBackupDir, err) - } - - if cfg.Crowdsec != nil && len(cfg.Crowdsec.AcquisitionFiles) > 0 { - for _, acquisFile := range cfg.Crowdsec.AcquisitionFiles { - /*if it was the default one, it was already backup'ed*/ - if cfg.Crowdsec.AcquisitionFilePath == acquisFile { - continue - } - - targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile))) - if err != nil { - return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err) - } - - if err = CopyFile(acquisFile, targetFname); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err) - } - - log.Infof("Saved acquis %s to %s", acquisFile, targetFname) - } - } - - if ConfigFilePath != "" { - backupMain := fmt.Sprintf("%s/config.yaml", dirPath) - if err = CopyFile(ConfigFilePath, backupMain); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", ConfigFilePath, backupMain, err) - } - - log.Infof("Saved default yaml to %s", backupMain) - } - - if cfg.API != nil && cfg.API.Server != nil && cfg.API.Server.OnlineClient != nil && cfg.API.Server.OnlineClient.CredentialsFilePath != "" { - backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath) - if err = CopyFile(cfg.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", cfg.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err) - } - - log.Infof("Saved online API credentials to %s", backupCAPICreds) - } - - if cfg.API != nil && cfg.API.Client != nil && cfg.API.Client.CredentialsFilePath != "" { - backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath) - if err = CopyFile(cfg.API.Client.CredentialsFilePath, backupLAPICreds); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", cfg.API.Client.CredentialsFilePath, backupLAPICreds, err) - } - - log.Infof("Saved local API credentials to %s", backupLAPICreds) - } - - if cfg.API != nil && cfg.API.Server != nil && cfg.API.Server.ProfilesPath != "" { - backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath) - if err = CopyFile(cfg.API.Server.ProfilesPath, backupProfiles); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", cfg.API.Server.ProfilesPath, backupProfiles, err) - } - - log.Infof("Saved profiles to %s", backupProfiles) - } - - if err = cli.backupHub(dirPath); err != nil { - return fmt.Errorf("failed to backup hub config: %w", err) - } - - return nil -} - func (cli *cliConfig) newBackupCmd() *cobra.Command { cmd := &cobra.Command{ - Use: `backup "directory"`, - Short: "Backup current config", - Long: `Backup the current crowdsec configuration including : - -- Main config (config.yaml) -- Simulation config (simulation.yaml) -- Profiles config (profiles.yaml) -- List of scenarios, parsers, postoverflows and collections that are up-to-date -- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections -- Backup of API credentials (local API and online API)`, - Example: `cscli config backup ./my-backup`, - Args: cobra.ExactArgs(1), + Use: "backup", DisableAutoGenTag: true, - RunE: func(_ *cobra.Command, args []string) error { - if err := cli.backup(args[0]); err != nil { - return fmt.Errorf("failed to backup config: %w", err) - } - - return nil + RunE: func(_ *cobra.Command, _ []string) error { + configDir := cli.cfg().ConfigPaths.ConfigDir + return fmt.Errorf("'cscli config backup' has been removed, you can manually backup/restore %s instead", configDir) }, } diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index b5fbf36b2b4..75373475ed9 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -1,285 +1,18 @@ package main import ( - "context" - "encoding/json" "fmt" - "os" - "path/filepath" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" - "github.com/crowdsecurity/crowdsec/pkg/hubops" ) -func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error { - cfg := cli.cfg() - - hub, err := require.Hub(cfg, nil) - if err != nil { - return err - } - - contentProvider := require.HubDownloader(ctx, cfg) - - for _, itype := range cwhub.ItemTypes { - itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype) - if _, err = os.Stat(itemDirectory); err != nil { - log.Infof("no %s in backup", itype) - continue - } - /*restore the upstream items*/ - upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype) - - file, err := os.ReadFile(upstreamListFN) - if err != nil { - return fmt.Errorf("error while opening %s: %w", upstreamListFN, err) - } - - var upstreamList []string - - err = json.Unmarshal(file, &upstreamList) - if err != nil { - return fmt.Errorf("error parsing %s: %w", upstreamListFN, err) - } - - for _, toinstall := range upstreamList { - item := hub.GetItem(itype, toinstall) - if item == nil { - log.Errorf("Item %s/%s not found in hub", itype, toinstall) - continue - } - - plan := hubops.NewActionPlan(hub) - - if err = plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, false)); err != nil { - return err - } - - if err = plan.AddCommand(hubops.NewEnableCommand(item, false)); err != nil { - return err - } - - if err = plan.Execute(ctx, true, false, false); err != nil { - log.Errorf("Error while installing %s : %s", toinstall, err) - } - } - - /*restore the local and tainted items*/ - files, err := os.ReadDir(itemDirectory) - if err != nil { - return fmt.Errorf("failed enumerating files of %s: %w", itemDirectory, err) - } - - for _, file := range files { - // this was the upstream data - if file.Name() == fmt.Sprintf("upstream-%s.json", itype) { - continue - } - - if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS { - // we expect a stage here - if !file.IsDir() { - continue - } - - stage := file.Name() - stagedir := fmt.Sprintf("%s/%s/%s/", cfg.ConfigPaths.ConfigDir, itype, stage) - log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir) - - if err = os.MkdirAll(stagedir, os.ModePerm); err != nil { - return fmt.Errorf("error while creating stage directory %s: %w", stagedir, err) - } - - // find items - ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/") - if err != nil { - return fmt.Errorf("failed enumerating files of %s: %w", itemDirectory+"/"+stage, err) - } - - // finally copy item - for _, tfile := range ifiles { - log.Infof("Going to restore local/tainted [%s]", tfile.Name()) - sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name()) - - destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name()) - if err = CopyFile(sourceFile, destinationFile); err != nil { - return fmt.Errorf("failed copy %s %s to %s: %w", itype, sourceFile, destinationFile, err) - } - - log.Infof("restored %s to %s", sourceFile, destinationFile) - } - } else { - log.Infof("Going to restore local/tainted [%s]", file.Name()) - sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name()) - destinationFile := fmt.Sprintf("%s/%s/%s", cfg.ConfigPaths.ConfigDir, itype, file.Name()) - - if err = CopyFile(sourceFile, destinationFile); err != nil { - return fmt.Errorf("failed copy %s %s to %s: %w", itype, sourceFile, destinationFile, err) - } - - log.Infof("restored %s to %s", sourceFile, destinationFile) - } - } - } - - return nil -} - -/* - Restore crowdsec configurations to directory : - -- Main config (config.yaml) -- Profiles config (profiles.yaml) -- Simulation config (simulation.yaml) -- Backup of API credentials (local API and online API) -- List of scenarios, parsers, postoverflows and collections that are up-to-date -- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections -- Acquisition files (acquis.yaml, acquis.d/*.yaml) -*/ -func (cli *cliConfig) restore(ctx context.Context, dirPath string) error { - var err error - - cfg := cli.cfg() - - backupMain := fmt.Sprintf("%s/config.yaml", dirPath) - if _, err = os.Stat(backupMain); err == nil { - if cfg.ConfigPaths != nil && cfg.ConfigPaths.ConfigDir != "" { - if err = CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", cfg.ConfigPaths.ConfigDir)); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", backupMain, cfg.ConfigPaths.ConfigDir, err) - } - } - } - - // Now we have config.yaml, we should regenerate config struct to have rights paths etc - ConfigFilePath = fmt.Sprintf("%s/config.yaml", cfg.ConfigPaths.ConfigDir) - - log.Debug("Reloading configuration") - - csConfig, _, err = loadConfigFor("config") - if err != nil { - return fmt.Errorf("failed to reload configuration: %w", err) - } - - cfg = cli.cfg() - - backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath) - if _, err = os.Stat(backupCAPICreds); err == nil { - if err = CopyFile(backupCAPICreds, cfg.API.Server.OnlineClient.CredentialsFilePath); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", backupCAPICreds, cfg.API.Server.OnlineClient.CredentialsFilePath, err) - } - } - - backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath) - if _, err = os.Stat(backupLAPICreds); err == nil { - if err = CopyFile(backupLAPICreds, cfg.API.Client.CredentialsFilePath); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", backupLAPICreds, cfg.API.Client.CredentialsFilePath, err) - } - } - - backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath) - if _, err = os.Stat(backupProfiles); err == nil { - if err = CopyFile(backupProfiles, cfg.API.Server.ProfilesPath); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", backupProfiles, cfg.API.Server.ProfilesPath, err) - } - } - - backupSimulation := fmt.Sprintf("%s/simulation.yaml", dirPath) - if _, err = os.Stat(backupSimulation); err == nil { - if err = CopyFile(backupSimulation, cfg.ConfigPaths.SimulationFilePath); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", backupSimulation, cfg.ConfigPaths.SimulationFilePath, err) - } - } - - /*if there is a acquisition dir, restore its content*/ - if cfg.Crowdsec.AcquisitionDirPath != "" { - if err = os.MkdirAll(cfg.Crowdsec.AcquisitionDirPath, 0o700); err != nil { - return fmt.Errorf("error while creating %s: %w", cfg.Crowdsec.AcquisitionDirPath, err) - } - } - - // if there was a single one - backupAcquisition := fmt.Sprintf("%s/acquis.yaml", dirPath) - if _, err = os.Stat(backupAcquisition); err == nil { - log.Debugf("restoring backup'ed %s", backupAcquisition) - - if err = CopyFile(backupAcquisition, cfg.Crowdsec.AcquisitionFilePath); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", backupAcquisition, cfg.Crowdsec.AcquisitionFilePath, err) - } - } - - // if there are files in the acquis backup dir, restore them - acquisBackupDir := filepath.Join(dirPath, "acquis", "*.yaml") - if acquisFiles, err := filepath.Glob(acquisBackupDir); err == nil { - for _, acquisFile := range acquisFiles { - targetFname, err := filepath.Abs(cfg.Crowdsec.AcquisitionDirPath + "/" + filepath.Base(acquisFile)) - if err != nil { - return fmt.Errorf("while saving %s to %s: %w", acquisFile, targetFname, err) - } - - log.Debugf("restoring %s to %s", acquisFile, targetFname) - - if err = CopyFile(acquisFile, targetFname); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err) - } - } - } - - if cfg.Crowdsec != nil && len(cfg.Crowdsec.AcquisitionFiles) > 0 { - for _, acquisFile := range cfg.Crowdsec.AcquisitionFiles { - log.Infof("backup filepath from dir -> %s", acquisFile) - - // if it was the default one, it has already been backed up - if cfg.Crowdsec.AcquisitionFilePath == acquisFile { - log.Infof("skip this one") - continue - } - - targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile))) - if err != nil { - return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err) - } - - if err = CopyFile(acquisFile, targetFname); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err) - } - - log.Infof("Saved acquis %s to %s", acquisFile, targetFname) - } - } - - if err = cli.restoreHub(ctx, dirPath); err != nil { - return fmt.Errorf("failed to restore hub config: %w", err) - } - - return nil -} - func (cli *cliConfig) newRestoreCmd() *cobra.Command { cmd := &cobra.Command{ - Use: `restore "directory"`, - Short: `Restore config in backup "directory"`, - Long: `Restore the crowdsec configuration from specified backup "directory" including: - -- Main config (config.yaml) -- Simulation config (simulation.yaml) -- Profiles config (profiles.yaml) -- List of scenarios, parsers, postoverflows and collections that are up-to-date -- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections -- Backup of API credentials (local API and online API)`, - Args: cobra.ExactArgs(1), + Use: "restore", DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - dirPath := args[0] - - if err := cli.restore(cmd.Context(), dirPath); err != nil { - return fmt.Errorf("failed to restore config from %s: %w", dirPath, err) - } - - return nil + RunE: func(cmd *cobra.Command, _ []string) error { + configDir := cli.cfg().ConfigPaths.ConfigDir + return fmt.Errorf("'cscli config restore' has been removed, you can manually backup/restore %s instead", configDir) }, } diff --git a/cmd/crowdsec-cli/copyfile.go b/cmd/crowdsec-cli/copyfile.go deleted file mode 100644 index 272fb3f7851..00000000000 --- a/cmd/crowdsec-cli/copyfile.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "fmt" - "io" - "os" - "path/filepath" - - log "github.com/sirupsen/logrus" -) - -/*help to copy the file, ioutil doesn't offer the feature*/ - -func copyFileContents(src, dst string) (err error) { - in, err := os.Open(src) - if err != nil { - return - } - defer in.Close() - - out, err := os.Create(dst) - if err != nil { - return - } - - defer func() { - cerr := out.Close() - if err == nil { - err = cerr - } - }() - - if _, err = io.Copy(out, in); err != nil { - return - } - - err = out.Sync() - - return -} - -/*copy the file, ioutile doesn't offer the feature*/ -func CopyFile(sourceSymLink, destinationFile string) error { - sourceFile, err := filepath.EvalSymlinks(sourceSymLink) - if err != nil { - log.Infof("Not a symlink : %s", err) - - sourceFile = sourceSymLink - } - - sourceFileStat, err := os.Stat(sourceFile) - if err != nil { - return err - } - - if !sourceFileStat.Mode().IsRegular() { - // cannot copy non-regular files (e.g., directories, - // symlinks, devices, etc.) - return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String()) - } - - destinationFileStat, err := os.Stat(destinationFile) - if err != nil { - if !os.IsNotExist(err) { - return err - } - } else { - if !(destinationFileStat.Mode().IsRegular()) { - return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String()) - } - - if os.SameFile(sourceFileStat, destinationFileStat) { - return err - } - } - - if err = os.Link(sourceFile, destinationFile); err != nil { - err = copyFileContents(sourceFile, destinationFile) - } - - return err -} diff --git a/debian/postinst b/debian/postinst index 77f2511f556..d50a7c0bfe2 100644 --- a/debian/postinst +++ b/debian/postinst @@ -11,14 +11,6 @@ if [ "$1" = configure ]; then mkdir -p /var/lib/crowdsec/data fi - if [[ -d /var/lib/crowdsec/backup ]]; then - cscli config restore /var/lib/crowdsec/backup/backup.config - rm -rf /var/lib/crowdsec/backup - /usr/bin/cscli hub update - /usr/bin/cscli hub upgrade - systemctl start crowdsec - fi - . /usr/share/crowdsec/wizard.sh -n if ! [[ -f /etc/crowdsec/acquis.yaml ]]; then echo Creating /etc/crowdsec/acquis.yaml @@ -82,12 +74,6 @@ if [ "$1" = configure ]; then set -e fi - - if [[ -f /var/lib/crowdsec/data/crowdsec.db.backup ]]; then - cp /var/lib/crowdsec/data/crowdsec.db.backup /var/lib/crowdsec/data/crowdsec.db - rm -f /var/lib/crowdsec/data/crowdsec.db.backup - fi - systemctl --quiet is-enabled crowdsec || systemctl unmask crowdsec && systemctl enable crowdsec API=$(cscli config show --key "Config.API.Server") @@ -107,8 +93,6 @@ if [ "$1" = configure ]; then echo " * Detailed guides are available in our documentation: https://docs.crowdsec.net" echo " * Configuration items created by the community can be found at the Hub: https://hub.crowdsec.net" echo " * Gain insights into your use of CrowdSec with the help of the console https://app.crowdsec.net" - - fi echo "You can always run the configuration again interactively by using '/usr/share/crowdsec/wizard.sh -c'" diff --git a/debian/preinst b/debian/preinst index 217b836caa6..df5b56cef3f 100644 --- a/debian/preinst +++ b/debian/preinst @@ -5,39 +5,4 @@ set -e # Source debconf library. . /usr/share/debconf/confmodule - -OLD_MAJOR_VERSION=$(echo $2 | cut -d'.' -f1) -OLD_MINOR_VERSION=$(echo $2 | cut -d'.' -f2) -OLD_PATCH_VERSION=$(echo $2 | cut -d'.' -f3|cut -d'-' -f1) - -NEW_MAJOR_VERSION=$(echo $3 | cut -d'.' -f1) -NEW_MINOR_VERSION=$(echo $3 | cut -d'.' -f2) -NEW_PATCH_VERSION=$(echo $3 | cut -d'.' -f3|cut -d'-' -f1) - - - -if [ "$1" = upgrade ]; then - - OLD_MAJOR_VERSION=$(echo $2 | cut -d'.' -f1) - OLD_MINOR_VERSION=$(echo $2 | cut -d'.' -f2) - OLD_PATCH_VERSION=$(echo $2 | cut -d'.' -f3|cut -d'-' -f1) - - NEW_MAJOR_VERSION=$(echo $3 | cut -d'.' -f1) - NEW_MINOR_VERSION=$(echo $3 | cut -d'.' -f2) - NEW_PATCH_VERSION=$(echo $3 | cut -d'.' -f3|cut -d'-' -f1) - - - if [[ $OLD_MAJOR_VERSION -eq "1" ]] && [[ $OLD_MINOR_VERSION -eq "0" ]] && [[ $OLD_PATCH_VERSION -lt "9" ]]; then - if [[ -f /var/lib/crowdsec/data/crowdsec.db ]]; then - cp /var/lib/crowdsec/data/crowdsec.db /var/lib/crowdsec/data/crowdsec.db.backup - fi - fi - - if [[ $NEW_MAJOR_VERSION -gt $OLD_MAJOR_VERSION ]]; then - echo "Stopping crowdsec" - systemctl stop crowdsec || true - cscli config backup /var/lib/crowdsec/backup - fi -fi - echo "You can always run the configuration again interactively by using '/usr/share/crowdsec/wizard.sh -c'" diff --git a/debian/prerm b/debian/prerm index a463a6a1c80..10afcf1906d 100644 --- a/debian/prerm +++ b/debian/prerm @@ -1,9 +1,8 @@ if [ "$1" = "remove" ]; then - cscli dashboard remove -f -y --error || echo "Ignore the above error if you never installed the local dashboard." systemctl stop crowdsec systemctl disable crowdsec fi if [ "$1" = "upgrade" ]; then systemctl stop crowdsec -fi \ No newline at end of file +fi diff --git a/rpm/SPECS/crowdsec.spec b/rpm/SPECS/crowdsec.spec index ac438ad0c14..c24b3f2ac0d 100644 --- a/rpm/SPECS/crowdsec.spec +++ b/rpm/SPECS/crowdsec.spec @@ -143,18 +143,15 @@ rm -rf %{buildroot} #systemctl stop crowdsec || true -if [ $1 == 2 ];then - if [[ ! -d /var/lib/crowdsec/backup ]]; then - cscli config backup /var/lib/crowdsec/backup - fi -fi +#if [ $1 == 2 ]; then +# upgrade pre-install here +#fi %post -p /bin/bash #install if [ $1 == 1 ]; then - if [ ! -f "/var/lib/crowdsec/data/crowdsec.db" ] ; then touch /var/lib/crowdsec/data/crowdsec.db fi @@ -185,21 +182,6 @@ if [ $1 == 1 ]; then echo " * Detailed guides are available in our documentation: https://docs.crowdsec.net" echo " * Configuration items created by the community can be found at the Hub: https://hub.crowdsec.net" echo " * Gain insights into your use of CrowdSec with the help of the console https://app.crowdsec.net" - -#upgrade -elif [ $1 == 2 ] && [ -d /var/lib/crowdsec/backup ]; then - cscli config restore /var/lib/crowdsec/backup - if [ $? == 0 ]; then - rm -rf /var/lib/crowdsec/backup - fi - - if [[ -f %{_sysconfdir}/crowdsec/online_api_credentials.yaml ]] ; then - chmod 600 %{_sysconfdir}/crowdsec/online_api_credentials.yaml - fi - - if [[ -f %{_sysconfdir}/crowdsec/local_api_credentials.yaml ]] ; then - chmod 600 %{_sysconfdir}/crowdsec/local_api_credentials.yaml - fi fi %systemd_post %{name}.service diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats index 63c204a9e86..9af3c841759 100644 --- a/test/bats/01_cscli.bats +++ b/test/bats/01_cscli.bats @@ -172,41 +172,13 @@ teardown() { } @test "cscli config backup / restore" { - # test that we need a valid path - # disabled because in CI, the empty string is not passed as a parameter - #rune -1 cscli config backup "" - #assert_stderr --partial "failed to backup config: directory path can't be empty" + CONFIG_DIR=$(config_get '.config_paths.config_dir') rune -1 cscli config backup "/dev/null/blah" - assert_stderr --partial "failed to backup config: while creating /dev/null/blah: mkdir /dev/null/blah: not a directory" + assert_stderr --partial "'cscli config backup' has been removed, you can manually backup/restore $CONFIG_DIR instead" - # pick a dirpath - backupdir=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp -u) - - # succeed the first time - rune -0 cscli config backup "$backupdir" - assert_stderr --partial "Starting configuration backup" - - # don't overwrite an existing backup - rune -1 cscli config backup "$backupdir" - assert_stderr --partial "failed to backup config" - assert_stderr --partial "file exists" - - SIMULATION_YAML="$(config_get '.config_paths.simulation_path')" - - # restore - rm "$SIMULATION_YAML" - rune -0 cscli config restore "$backupdir" - assert_file_exists "$SIMULATION_YAML" - - # cleanup - rm -rf -- "${backupdir:?}" - - # backup: detect missing files - rm "$SIMULATION_YAML" - rune -1 cscli config backup "$backupdir" - assert_stderr --regexp "failed to backup config: failed copy .* to .*: stat .*: no such file or directory" - rm -rf -- "${backupdir:?}" + rune -1 cscli config restore "/dev/null/blah" + assert_stderr --partial "'cscli config restore' has been removed, you can manually backup/restore $CONFIG_DIR instead" } @test "'cscli completion' with or without configuration file" { diff --git a/test/bats/02_nolapi.bats b/test/bats/02_nolapi.bats index cefa6d798b4..70495a0ed91 100644 --- a/test/bats/02_nolapi.bats +++ b/test/bats/02_nolapi.bats @@ -66,18 +66,6 @@ teardown() { refute_output --partial "Local API Server" } -@test "cscli config backup" { - config_disable_lapi - backupdir=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp -u) - rune -0 cscli config backup "$backupdir" - assert_stderr --partial "Starting configuration backup" - rune -1 cscli config backup "$backupdir" - rm -rf -- "${backupdir:?}" - - assert_stderr --partial "failed to backup config" - assert_stderr --partial "file exists" -} - @test "lapi status shouldn't be ok without api.server" { config_disable_lapi rune -1 cscli machines list diff --git a/test/bats/03_noagent.bats b/test/bats/03_noagent.bats index 6be5101cee2..972b84977ad 100644 --- a/test/bats/03_noagent.bats +++ b/test/bats/03_noagent.bats @@ -60,18 +60,6 @@ teardown() { refute_output --partial "Crowdsec" } -@test "no agent: cscli config backup" { - config_disable_agent - backupdir=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp -u) - rune -0 cscli config backup "$backupdir" - assert_stderr --partial "Starting configuration backup" - rune -1 cscli config backup "$backupdir" - - assert_stderr --partial "failed to backup config" - assert_stderr --partial "file exists" - rm -rf -- "${backupdir:?}" -} - @test "no agent: lapi status should be ok" { config_disable_agent ./instance-crowdsec start diff --git a/test/bats/04_nocapi.bats b/test/bats/04_nocapi.bats index 8d0018a9a4a..eaeb0939112 100644 --- a/test/bats/04_nocapi.bats +++ b/test/bats/04_nocapi.bats @@ -51,17 +51,6 @@ teardown() { assert_output --regexp "Global:.*Crowdsec.*cscli:.*Local API Server:" } -@test "no agent: cscli config backup" { - config_disable_capi - backupdir=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp -u) - rune -0 cscli config backup "$backupdir" - assert_stderr --partial "Starting configuration backup" - rune -1 cscli config backup "$backupdir" - assert_stderr --partial "failed to backup config" - assert_stderr --partial "file exists" - rm -rf -- "${backupdir:?}" -} - @test "without capi: cscli lapi status -> success" { config_disable_capi ./instance-crowdsec start diff --git a/wizard.sh b/wizard.sh index 57311d40cdb..4da970cd695 100755 --- a/wizard.sh +++ b/wizard.sh @@ -21,11 +21,8 @@ DOCKER_MODE="false" CROWDSEC_LIB_DIR="/var/lib/crowdsec" CROWDSEC_USR_DIR="/usr/local/lib/crowdsec" CROWDSEC_DATA_DIR="${CROWDSEC_LIB_DIR}/data" -CROWDSEC_DB_PATH="${CROWDSEC_DATA_DIR}/crowdsec.db" CROWDSEC_PATH="/etc/crowdsec" CROWDSEC_CONFIG_PATH="${CROWDSEC_PATH}" -CROWDSEC_LOG_FILE="/var/log/crowdsec.log" -LAPI_LOG_FILE="/var/log/crowdsec_api.log" CROWDSEC_PLUGIN_DIR="${CROWDSEC_USR_DIR}/plugins" CROWDSEC_CONSOLE_DIR="${CROWDSEC_PATH}/console" @@ -35,8 +32,6 @@ CSCLI_BIN="./cmd/crowdsec-cli/cscli" CLIENT_SECRETS="local_api_credentials.yaml" LAPI_SECRETS="online_api_credentials.yaml" -CONSOLE_FILE="console.yaml" - BIN_INSTALL_PATH="/usr/local/bin" CROWDSEC_BIN_INSTALLED="${BIN_INSTALL_PATH}/crowdsec" @@ -91,9 +86,6 @@ SENTINEL_PLUGIN_CONFIG="./cmd/notification-sentinel/sentinel.yaml" FILE_PLUGIN_CONFIG="./cmd/notification-file/file.yaml" -BACKUP_DIR=$(mktemp -d) -rm -rf -- "$BACKUP_DIR" - log_info() { msg=$1 date=$(date "+%Y-%m-%d %H:%M:%S") @@ -426,27 +418,20 @@ install_crowdsec() { mkdir -p "${CROWDSEC_CONFIG_PATH}/contexts" || exit mkdir -p "${CROWDSEC_CONSOLE_DIR}" || exit - # tmp - mkdir -p /tmp/data mkdir -p /etc/crowdsec/hub/ - install -v -m 600 -D "./config/${CLIENT_SECRETS}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 600 -D "./config/${LAPI_SECRETS}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - - ## end tmp - install -v -m 600 -D ./config/config.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/dev.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/user.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/acquis.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/profiles.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/simulation.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/"${CONSOLE_FILE}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/context.yaml "${CROWDSEC_CONSOLE_DIR}" 1> /dev/null || exit + # Don't overwrite existing files + [[ ! -f "${CROWDSEC_CONFIG_PATH}/${CLIENT_SECRETS}" ]] && install -v -m 600 -D "./config/${CLIENT_SECRETS}" "${CROWDSEC_CONFIG_PATH}" >/dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/${LAPI_SECRETS}" ]] && install -v -m 600 -D "./config/${LAPI_SECRETS}" "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/config.yaml" ]] && install -v -m 600 -D ./config/config.yaml "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/dev.yaml" ]] && install -v -m 644 -D ./config/dev.yaml "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/user.yaml" ]] && install -v -m 644 -D ./config/user.yaml "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/acquis.yaml" ]] && install -v -m 644 -D ./config/acquis.yaml "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/profiles.yaml" ]] && install -v -m 644 -D ./config/profiles.yaml "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/simulation.yaml" ]] && install -v -m 644 -D ./config/simulation.yaml "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/console.yaml" ]] && install -v -m 644 -D ./config/console.yaml "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/context.yaml" ]] && install -v -m 644 -D ./config/context.yaml "${CROWDSEC_CONSOLE_DIR}" > /dev/null || exit - DATA=${CROWDSEC_DATA_DIR} CFG=${CROWDSEC_CONFIG_PATH} envsubst '$CFG $DATA' < ./config/user.yaml > ${CROWDSEC_CONFIG_PATH}"/user.yaml" || log_fatal "unable to generate user configuration file" - if [[ ${DOCKER_MODE} == "false" ]]; then - CFG=${CROWDSEC_CONFIG_PATH} BIN=${CROWDSEC_BIN_INSTALLED} envsubst '$CFG $BIN' < ./config/crowdsec.service > "${SYSTEMD_PATH_FILE}" || log_fatal "unable to crowdsec systemd file" - fi install_bins if [[ ${DOCKER_MODE} == "false" ]]; then @@ -471,23 +456,12 @@ update_full() { log_err "Cscli binary '$CSCLI_BIN' not found. Please build it with 'make build'" && exit fi - log_info "Backing up existing configuration" - ${CSCLI_BIN_INSTALLED} config backup ${BACKUP_DIR} - log_info "Saving default database content if exist" - if [[ -f "/var/lib/crowdsec/data/crowdsec.db" ]]; then - cp /var/lib/crowdsec/data/crowdsec.db ${BACKUP_DIR}/crowdsec.db - fi - log_info "Cleanup existing crowdsec configuration" + log_info "Removing old binaries" uninstall_crowdsec log_info "Installing crowdsec" install_crowdsec - log_info "Restoring configuration" + log_info "Updating hub" ${CSCLI_BIN_INSTALLED} hub update - ${CSCLI_BIN_INSTALLED} config restore ${BACKUP_DIR} - log_info "Restoring saved database if exist" - if [[ -f "${BACKUP_DIR}/crowdsec.db" ]]; then - cp ${BACKUP_DIR}/crowdsec.db /var/lib/crowdsec/data/crowdsec.db - fi log_info "Finished, restarting" systemctl restart crowdsec || log_fatal "Failed to restart crowdsec" } @@ -565,15 +539,6 @@ uninstall_crowdsec() { ${CSCLI_BIN} dashboard remove -f -y >/dev/null delete_bins - # tmp - rm -rf /tmp/data/ - ## end tmp - - find /etc/crowdsec -maxdepth 1 -mindepth 1 | grep -v "bouncer" | xargs rm -rf || echo "" - rm -f ${CROWDSEC_LOG_FILE} || echo "" - rm -f ${LAPI_LOG_FILE} || echo "" - rm -f ${CROWDSEC_DB_PATH} || echo "" - rm -rf ${CROWDSEC_LIB_DIR} || echo "" rm -rf ${CROWDSEC_USR_DIR} || echo "" rm -f ${SYSTEMD_PATH_FILE} || echo "" log_info "crowdsec successfully uninstalled" @@ -765,12 +730,11 @@ usage() { echo " ./wizard.sh --unattended Install in unattended mode, no question will be asked and defaults will be followed" echo " ./wizard.sh --docker-mode Will install crowdsec without systemd and generate random machine-id" echo " ./wizard.sh -n|--noop Do nothing" - - exit 0 } if [[ $# -eq 0 ]]; then -usage + usage + exit 0 fi while [[ $# -gt 0 ]] From fe931af5cae0e7a5739e2073e5ae7f2f4ccbdebd Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 16 Jan 2025 14:03:53 +0100 Subject: [PATCH 17/25] lint: gocritic/captLocal (don't capitalize local variables) (#3402) * lint: gocritic/captLocal (don't capitalize local variables) * lint (whitespace) --- .golangci.yml | 1 - cmd/crowdsec-cli/clidecision/decisions.go | 4 +- pkg/acquisition/acquisition.go | 20 ++++---- pkg/acquisition/modules/appsec/appsec.go | 22 +++++++-- .../modules/cloudwatch/cloudwatch.go | 39 +++++++++++++++- pkg/acquisition/modules/docker/docker.go | 4 +- pkg/acquisition/modules/file/file.go | 4 +- pkg/acquisition/modules/http/http.go | 6 ++- .../modules/journalctl/journalctl.go | 4 +- pkg/acquisition/modules/kafka/kafka.go | 7 ++- pkg/acquisition/modules/kinesis/kinesis.go | 4 +- .../modules/kubernetesaudit/k8s_audit.go | 4 +- pkg/acquisition/modules/loki/loki.go | 4 +- pkg/acquisition/modules/syslog/syslog.go | 4 +- .../wineventlog/wineventlog_windows.go | 4 +- pkg/apiclient/client.go | 4 +- pkg/csprofiles/csprofiles.go | 44 +++++++++--------- pkg/database/flush.go | 16 +++---- pkg/exprhelpers/crowdsec_cti.go | 24 +++++----- pkg/exprhelpers/helpers.go | 46 +++++++++++++------ pkg/leakybucket/bayesian.go | 4 +- pkg/leakybucket/overflow_filter.go | 4 +- 22 files changed, 172 insertions(+), 101 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 12d35ed8737..7df08cf717c 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -192,7 +192,6 @@ linters-settings: - unnamedResult - sloppyReassign - appendCombine - - captLocal - typeUnparen - commentFormatting - deferInLoop # diff --git a/cmd/crowdsec-cli/clidecision/decisions.go b/cmd/crowdsec-cli/clidecision/decisions.go index 307cabffe51..b5865bab6e0 100644 --- a/cmd/crowdsec-cli/clidecision/decisions.go +++ b/cmd/crowdsec-cli/clidecision/decisions.go @@ -170,7 +170,7 @@ func (cli *cliDecisions) NewCommand() *cobra.Command { return cmd } -func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOpts, NoSimu *bool, contained *bool, printMachine bool) error { +func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOpts, noSimu *bool, contained *bool, printMachine bool) error { var err error *filter.ScopeEquals, err = clialert.SanitizeScope(*filter.ScopeEquals, *filter.IPEquals, *filter.RangeEquals) @@ -181,7 +181,7 @@ func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOp filter.ActiveDecisionEquals = new(bool) *filter.ActiveDecisionEquals = true - if NoSimu != nil && *NoSimu { + if noSimu != nil && *noSimu { filter.IncludeSimulated = new(bool) } /* nullify the empty entries to avoid bad filter */ diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 06a4918592b..d3928270598 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -365,13 +365,13 @@ func copyEvent(evt types.Event, line string) types.Event { return evtCopy } -func transform(transformChan chan types.Event, output chan types.Event, AcquisTomb *tomb.Tomb, transformRuntime *vm.Program, logger *log.Entry) { +func transform(transformChan chan types.Event, output chan types.Event, acquisTomb *tomb.Tomb, transformRuntime *vm.Program, logger *log.Entry) { defer trace.CatchPanic("crowdsec/acquis") logger.Infof("transformer started") for { select { - case <-AcquisTomb.Dying(): + case <-acquisTomb.Dying(): logger.Debugf("transformer is dying") return case evt := <-transformChan: @@ -420,7 +420,7 @@ func transform(transformChan chan types.Event, output chan types.Event, AcquisTo } } -func StartAcquisition(ctx context.Context, sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error { +func StartAcquisition(ctx context.Context, sources []DataSource, output chan types.Event, acquisTomb *tomb.Tomb) error { // Don't wait if we have no sources, as it will hang forever if len(sources) == 0 { return nil @@ -430,7 +430,7 @@ func StartAcquisition(ctx context.Context, sources []DataSource, output chan typ subsrc := sources[i] // ensure its a copy log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc) - AcquisTomb.Go(func() error { + acquisTomb.Go(func() error { defer trace.CatchPanic("crowdsec/acquis") var err error @@ -449,21 +449,21 @@ func StartAcquisition(ctx context.Context, sources []DataSource, output chan typ "datasource": subsrc.GetName(), }) - AcquisTomb.Go(func() error { - transform(outChan, output, AcquisTomb, transformRuntime, transformLogger) + acquisTomb.Go(func() error { + transform(outChan, output, acquisTomb, transformRuntime, transformLogger) return nil }) } if subsrc.GetMode() == configuration.TAIL_MODE { - err = subsrc.StreamingAcquisition(ctx, outChan, AcquisTomb) + err = subsrc.StreamingAcquisition(ctx, outChan, acquisTomb) } else { - err = subsrc.OneShotAcquisition(ctx, outChan, AcquisTomb) + err = subsrc.OneShotAcquisition(ctx, outChan, acquisTomb) } if err != nil { // if one of the acqusition returns an error, we kill the others to properly shutdown - AcquisTomb.Kill(err) + acquisTomb.Kill(err) } return nil @@ -471,7 +471,7 @@ func StartAcquisition(ctx context.Context, sources []DataSource, output chan typ } /*return only when acquisition is over (cat) or never (tail)*/ - err := AcquisTomb.Wait() + err := acquisTomb.Wait() return err } diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index 2f7861b32ff..86dbfe38b71 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -155,14 +155,14 @@ func (w *AppsecSource) GetAggregMetrics() []prometheus.Collector { return []prometheus.Collector{AppsecReqCounter, AppsecBlockCounter, AppsecRuleHits, AppsecOutbandParsingHistogram, AppsecInbandParsingHistogram, AppsecGlobalParsingHistogram} } -func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { err := w.UnmarshalConfig(yamlConfig) if err != nil { return fmt.Errorf("unable to parse appsec configuration: %w", err) } w.logger = logger - w.metricsLevel = MetricsLevel + w.metricsLevel = metricsLevel w.logger.Tracef("Appsec configuration: %+v", w.config) if w.config.AuthCacheDuration == nil { @@ -180,7 +180,7 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLe w.InChan = make(chan appsec.ParsedRequest) appsecCfg := appsec.AppsecConfig{Logger: w.logger.WithField("component", "appsec_config")} - //we keep the datasource name + // we keep the datasource name appsecCfg.Name = w.config.Name // let's load the associated appsec_config: @@ -275,6 +275,7 @@ func (w *AppsecSource) StreamingAcquisition(ctx context.Context, out chan types. for _, runner := range w.AppsecRunners { runner.outChan = out + t.Go(func() error { defer trace.CatchPanic("crowdsec/acquis/appsec/live/runner") return runner.Run(t) @@ -285,16 +286,20 @@ func (w *AppsecSource) StreamingAcquisition(ctx context.Context, out chan types. if w.config.ListenSocket != "" { w.logger.Infof("creating unix socket %s", w.config.ListenSocket) _ = os.RemoveAll(w.config.ListenSocket) + listener, err := net.Listen("unix", w.config.ListenSocket) if err != nil { return fmt.Errorf("appsec server failed: %w", err) } + defer listener.Close() + if w.config.CertFilePath != "" && w.config.KeyFilePath != "" { err = w.server.ServeTLS(listener, w.config.CertFilePath, w.config.KeyFilePath) } else { err = w.server.Serve(listener) } + if err != nil && !errors.Is(err, http.ErrServerClosed) { return fmt.Errorf("appsec server failed: %w", err) } @@ -304,8 +309,10 @@ func (w *AppsecSource) StreamingAcquisition(ctx context.Context, out chan types. }) t.Go(func() error { var err error + if w.config.ListenAddr != "" { w.logger.Infof("creating TCP server on %s", w.config.ListenAddr) + if w.config.CertFilePath != "" && w.config.KeyFilePath != "" { err = w.server.ListenAndServeTLS(w.config.CertFilePath, w.config.KeyFilePath) } else { @@ -324,6 +331,7 @@ func (w *AppsecSource) StreamingAcquisition(ctx context.Context, out chan types. // xx let's clean up the appsec runners :) appsec.AppsecRulesDetails = make(map[int]appsec.RulesDetails) w.server.Shutdown(ctx) + return nil }) @@ -354,11 +362,13 @@ func (w *AppsecSource) IsAuth(apiKey string) bool { } req.Header.Add("X-Api-Key", apiKey) + resp, err := client.Do(req) if err != nil { log.Errorf("Error performing request: %s", err) return false } + defer resp.Body.Close() return resp.StatusCode == http.StatusOK @@ -371,17 +381,21 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { apiKey := r.Header.Get(appsec.APIKeyHeaderName) clientIP := r.Header.Get(appsec.IPHeaderName) remoteIP := r.RemoteAddr + if apiKey == "" { w.logger.Errorf("Unauthorized request from '%s' (real IP = %s)", remoteIP, clientIP) rw.WriteHeader(http.StatusUnauthorized) + return } + expiration, exists := w.AuthCache.Get(apiKey) // if the apiKey is not in cache or has expired, just recheck the auth if !exists || time.Now().After(expiration) { if !w.IsAuth(apiKey) { rw.WriteHeader(http.StatusUnauthorized) w.logger.Errorf("Unauthorized request from '%s' (real IP = %s)", remoteIP, clientIP) + return } @@ -394,8 +408,10 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { if err != nil { w.logger.Errorf("%s", err) rw.WriteHeader(http.StatusInternalServerError) + return } + parsedRequest.AppsecEngine = w.config.Name logger := w.logger.WithFields(log.Fields{ diff --git a/pkg/acquisition/modules/cloudwatch/cloudwatch.go b/pkg/acquisition/modules/cloudwatch/cloudwatch.go index ba267c9050b..5739ebc3124 100644 --- a/pkg/acquisition/modules/cloudwatch/cloudwatch.go +++ b/pkg/acquisition/modules/cloudwatch/cloudwatch.go @@ -154,13 +154,13 @@ func (cw *CloudwatchSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (cw *CloudwatchSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (cw *CloudwatchSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { err := cw.UnmarshalConfig(yamlConfig) if err != nil { return err } - cw.metricsLevel = MetricsLevel + cw.metricsLevel = metricsLevel cw.logger = logger.WithField("group", cw.Config.GroupName) @@ -330,9 +330,12 @@ func (cw *CloudwatchSource) WatchLogGroupForStreams(ctx context.Context, out cha LastIngestionTime := time.Unix(0, *event.LastIngestionTime*int64(time.Millisecond)) if LastIngestionTime.Before(oldest) { cw.logger.Tracef("stop iteration, %s reached oldest age, stop (%s < %s)", *event.LogStreamName, LastIngestionTime, time.Now().UTC().Add(-*cw.Config.MaxStreamAge)) + hasMoreStreams = false + return false } + cw.logger.Tracef("stream %s is elligible for monitoring", *event.LogStreamName) // the stream has been updated recently, check if we should monitor it var expectMode int @@ -341,6 +344,7 @@ func (cw *CloudwatchSource) WatchLogGroupForStreams(ctx context.Context, out cha } else { expectMode = types.TIMEMACHINE } + monitorStream := LogStreamTailConfig{ GroupName: cw.Config.GroupName, StreamName: *event.LogStreamName, @@ -354,16 +358,20 @@ func (cw *CloudwatchSource) WatchLogGroupForStreams(ctx context.Context, out cha out <- monitorStream } } + if lastPage { cw.logger.Tracef("reached last page") + hasMoreStreams = false } + return true }, ) if err != nil { return fmt.Errorf("while describing group %s: %w", cw.Config.GroupName, err) } + cw.logger.Tracef("after DescribeLogStreamsPagesWithContext") } } @@ -373,12 +381,14 @@ func (cw *CloudwatchSource) WatchLogGroupForStreams(ctx context.Context, out cha // LogStreamManager receives the potential streams to monitor, and starts a go routine when needed func (cw *CloudwatchSource) LogStreamManager(ctx context.Context, in chan LogStreamTailConfig, outChan chan types.Event) error { cw.logger.Debugf("starting to monitor streams for %s", cw.Config.GroupName) + pollDeadStreamInterval := time.NewTicker(def_PollDeadStreamInterval) for { select { case newStream := <-in: //nolint:govet // copylocks won't matter if the tomb is not initialized shouldCreate := true + cw.logger.Tracef("received new streams to monitor : %s/%s", newStream.GroupName, newStream.StreamName) if cw.Config.StreamName != nil && newStream.StreamName != *cw.Config.StreamName { @@ -402,12 +412,16 @@ func (cw *CloudwatchSource) LogStreamManager(ctx context.Context, in chan LogStr if !stream.t.Alive() { cw.logger.Debugf("stream %s already exists, but is dead", newStream.StreamName) cw.monitoredStreams = append(cw.monitoredStreams[:idx], cw.monitoredStreams[idx+1:]...) + if cw.metricsLevel != configuration.METRICS_NONE { openedStreams.With(prometheus.Labels{"group": newStream.GroupName}).Dec() } + break } + shouldCreate = false + break } } @@ -417,19 +431,23 @@ func (cw *CloudwatchSource) LogStreamManager(ctx context.Context, in chan LogStr if cw.metricsLevel != configuration.METRICS_NONE { openedStreams.With(prometheus.Labels{"group": newStream.GroupName}).Inc() } + newStream.t = tomb.Tomb{} newStream.logger = cw.logger.WithField("stream", newStream.StreamName) cw.logger.Debugf("starting tail of stream %s", newStream.StreamName) newStream.t.Go(func() error { return cw.TailLogStream(ctx, &newStream, outChan) }) + cw.monitoredStreams = append(cw.monitoredStreams, &newStream) } case <-pollDeadStreamInterval.C: newMonitoredStreams := cw.monitoredStreams[:0] + for idx, stream := range cw.monitoredStreams { if !cw.monitoredStreams[idx].t.Alive() { cw.logger.Debugf("remove dead stream %s", stream.StreamName) + if cw.metricsLevel != configuration.METRICS_NONE { openedStreams.With(prometheus.Labels{"group": cw.monitoredStreams[idx].GroupName}).Dec() } @@ -437,20 +455,25 @@ func (cw *CloudwatchSource) LogStreamManager(ctx context.Context, in chan LogStr newMonitoredStreams = append(newMonitoredStreams, stream) } } + cw.monitoredStreams = newMonitoredStreams case <-cw.t.Dying(): cw.logger.Infof("LogStreamManager for %s is dying, %d alive streams", cw.Config.GroupName, len(cw.monitoredStreams)) + for idx, stream := range cw.monitoredStreams { if cw.monitoredStreams[idx].t.Alive() { cw.logger.Debugf("killing stream %s", stream.StreamName) cw.monitoredStreams[idx].t.Kill(nil) + if err := cw.monitoredStreams[idx].t.Wait(); err != nil { cw.logger.Debugf("error while waiting for death of %s : %s", stream.StreamName, err) } } } + cw.monitoredStreams = nil cw.logger.Debugf("routine cleanup done, return") + return nil } } @@ -458,12 +481,14 @@ func (cw *CloudwatchSource) LogStreamManager(ctx context.Context, in chan LogStr func (cw *CloudwatchSource) TailLogStream(ctx context.Context, cfg *LogStreamTailConfig, outChan chan types.Event) error { var startFrom *string + lastReadMessage := time.Now().UTC() ticker := time.NewTicker(cfg.PollStreamInterval) // resume at existing index if we already had streamIndexMutex.Lock() v := cw.streamIndexes[cfg.GroupName+"+"+cfg.StreamName] streamIndexMutex.Unlock() + if v != "" { cfg.logger.Debugf("restarting on index %s", v) startFrom = &v @@ -474,7 +499,9 @@ func (cw *CloudwatchSource) TailLogStream(ctx context.Context, cfg *LogStreamTai select { case <-ticker.C: cfg.logger.Tracef("entering loop") + hasMorePages := true + for hasMorePages { /*for the first call, we only consume the last item*/ cfg.logger.Tracef("calling GetLogEventsPagesWithContext") @@ -489,36 +516,44 @@ func (cw *CloudwatchSource) TailLogStream(ctx context.Context, cfg *LogStreamTai func(page *cloudwatchlogs.GetLogEventsOutput, lastPage bool) bool { cfg.logger.Tracef("%d results, last:%t", len(page.Events), lastPage) startFrom = page.NextForwardToken + if page.NextForwardToken != nil { streamIndexMutex.Lock() cw.streamIndexes[cfg.GroupName+"+"+cfg.StreamName] = *page.NextForwardToken streamIndexMutex.Unlock() } + if lastPage { /*wait another ticker to check on new log availability*/ cfg.logger.Tracef("last page") + hasMorePages = false } + if len(page.Events) > 0 { lastReadMessage = time.Now().UTC() } + for _, event := range page.Events { evt, err := cwLogToEvent(event, cfg) if err != nil { cfg.logger.Warningf("cwLogToEvent error, discarded event : %s", err) } else { cfg.logger.Debugf("pushing message : %s", evt.Line.Raw) + if cw.metricsLevel != configuration.METRICS_NONE { linesRead.With(prometheus.Labels{"group": cfg.GroupName, "stream": cfg.StreamName}).Inc() } outChan <- evt } } + return true }, ) if err != nil { newerr := fmt.Errorf("while reading %s/%s: %w", cfg.GroupName, cfg.StreamName, err) cfg.logger.Warningf("err : %s", newerr) + return newerr } cfg.logger.Tracef("done reading GetLogEventsPagesWithContext") diff --git a/pkg/acquisition/modules/docker/docker.go b/pkg/acquisition/modules/docker/docker.go index 798eba29440..582da3d53a1 100644 --- a/pkg/acquisition/modules/docker/docker.go +++ b/pkg/acquisition/modules/docker/docker.go @@ -136,9 +136,9 @@ func (d *DockerSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (d *DockerSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (d *DockerSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { d.logger = logger - d.metricsLevel = MetricsLevel + d.metricsLevel = metricsLevel err := d.UnmarshalConfig(yamlConfig) if err != nil { diff --git a/pkg/acquisition/modules/file/file.go b/pkg/acquisition/modules/file/file.go index 9f439b0c82e..697a3d35dc2 100644 --- a/pkg/acquisition/modules/file/file.go +++ b/pkg/acquisition/modules/file/file.go @@ -102,9 +102,9 @@ func (f *FileSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { f.logger = logger - f.metricsLevel = MetricsLevel + f.metricsLevel = metricsLevel err := f.UnmarshalConfig(yamlConfig) if err != nil { diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 3e4f26915fd..97e220570ff 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -157,9 +157,9 @@ func (hc *HttpConfiguration) Validate() error { return nil } -func (h *HTTPSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (h *HTTPSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { h.logger = logger - h.metricsLevel = MetricsLevel + h.metricsLevel = metricsLevel err := h.UnmarshalConfig(yamlConfig) if err != nil { @@ -339,12 +339,14 @@ func (h *HTTPSource) RunServer(out chan types.Event, t *tomb.Tomb) error { if r.Method != http.MethodPost { h.logger.Errorf("method not allowed: %s", r.Method) http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) + return } if err := authorizeRequest(r, &h.Config); err != nil { h.logger.Errorf("failed to authorize request from '%s': %s", r.RemoteAddr, err) http.Error(w, "Unauthorized", http.StatusUnauthorized) + return } diff --git a/pkg/acquisition/modules/journalctl/journalctl.go b/pkg/acquisition/modules/journalctl/journalctl.go index 47d90e2b3a0..f72878d9b3c 100644 --- a/pkg/acquisition/modules/journalctl/journalctl.go +++ b/pkg/acquisition/modules/journalctl/journalctl.go @@ -210,9 +210,9 @@ func (j *JournalCtlSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (j *JournalCtlSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (j *JournalCtlSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { j.logger = logger - j.metricsLevel = MetricsLevel + j.metricsLevel = metricsLevel err := j.UnmarshalConfig(yamlConfig) if err != nil { diff --git a/pkg/acquisition/modules/kafka/kafka.go b/pkg/acquisition/modules/kafka/kafka.go index 77fc44e310d..f213b85814c 100644 --- a/pkg/acquisition/modules/kafka/kafka.go +++ b/pkg/acquisition/modules/kafka/kafka.go @@ -85,9 +85,9 @@ func (k *KafkaSource) UnmarshalConfig(yamlConfig []byte) error { return err } -func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { k.logger = logger - k.metricsLevel = MetricsLevel + k.metricsLevel = metricsLevel k.logger.Debugf("start configuring %s source", dataSourceName) @@ -160,6 +160,7 @@ func (k *KafkaSource) ReadMessage(ctx context.Context, out chan types.Event) err k.logger.Errorln(fmt.Errorf("while reading %s message: %w", dataSourceName, err)) continue } + k.logger.Tracef("got message: %s", string(m.Value)) l := types.Line{ Raw: string(m.Value), @@ -170,9 +171,11 @@ func (k *KafkaSource) ReadMessage(ctx context.Context, out chan types.Event) err Module: k.GetName(), } k.logger.Tracef("line with message read from topic '%s': %+v", k.Config.Topic, l) + if k.metricsLevel != configuration.METRICS_NONE { linesRead.With(prometheus.Labels{"topic": k.Config.Topic}).Inc() } + evt := types.MakeEvent(k.Config.UseTimeMachine, types.LOG, true) evt.Line = l out <- evt diff --git a/pkg/acquisition/modules/kinesis/kinesis.go b/pkg/acquisition/modules/kinesis/kinesis.go index b166a706ca9..16c91ad06bc 100644 --- a/pkg/acquisition/modules/kinesis/kinesis.go +++ b/pkg/acquisition/modules/kinesis/kinesis.go @@ -161,9 +161,9 @@ func (k *KinesisSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (k *KinesisSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (k *KinesisSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { k.logger = logger - k.metricsLevel = MetricsLevel + k.metricsLevel = metricsLevel err := k.UnmarshalConfig(yamlConfig) if err != nil { diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go index aaa83a3bbb2..f8c325b5c4b 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go @@ -97,9 +97,9 @@ func (ka *KubernetesAuditSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (ka *KubernetesAuditSource) Configure(config []byte, logger *log.Entry, MetricsLevel int) error { +func (ka *KubernetesAuditSource) Configure(config []byte, logger *log.Entry, metricsLevel int) error { ka.logger = logger - ka.metricsLevel = MetricsLevel + ka.metricsLevel = metricsLevel err := ka.UnmarshalConfig(config) if err != nil { diff --git a/pkg/acquisition/modules/loki/loki.go b/pkg/acquisition/modules/loki/loki.go index c57e6a67c94..47493d8cdfe 100644 --- a/pkg/acquisition/modules/loki/loki.go +++ b/pkg/acquisition/modules/loki/loki.go @@ -120,10 +120,10 @@ func (l *LokiSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (l *LokiSource) Configure(config []byte, logger *log.Entry, MetricsLevel int) error { +func (l *LokiSource) Configure(config []byte, logger *log.Entry, metricsLevel int) error { l.Config = LokiConfiguration{} l.logger = logger - l.metricsLevel = MetricsLevel + l.metricsLevel = metricsLevel err := l.UnmarshalConfig(config) if err != nil { return err diff --git a/pkg/acquisition/modules/syslog/syslog.go b/pkg/acquisition/modules/syslog/syslog.go index fb6a04600c1..df805d08cae 100644 --- a/pkg/acquisition/modules/syslog/syslog.go +++ b/pkg/acquisition/modules/syslog/syslog.go @@ -124,10 +124,10 @@ func (s *SyslogSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (s *SyslogSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (s *SyslogSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { s.logger = logger s.logger.Infof("Starting syslog datasource configuration") - s.metricsLevel = MetricsLevel + s.metricsLevel = metricsLevel err := s.UnmarshalConfig(yamlConfig) if err != nil { return err diff --git a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go index 8283bcc21a2..22186ea96cb 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go @@ -287,9 +287,9 @@ func (w *WinEventLogSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { w.logger = logger - w.metricsLevel = MetricsLevel + w.metricsLevel = metricsLevel err := w.UnmarshalConfig(yamlConfig) if err != nil { diff --git a/pkg/apiclient/client.go b/pkg/apiclient/client.go index 47d97a28344..ec473beca77 100644 --- a/pkg/apiclient/client.go +++ b/pkg/apiclient/client.go @@ -125,8 +125,8 @@ func NewClient(config *Config) (*ApiClient, error) { return c, nil } -func NewDefaultClient(URL *url.URL, prefix string, userAgent string, client *http.Client) (*ApiClient, error) { - transport, baseURL := createTransport(URL) +func NewDefaultClient(url *url.URL, prefix string, userAgent string, client *http.Client) (*ApiClient, error) { + transport, baseURL := createTransport(url) if client == nil { client = &http.Client{} diff --git a/pkg/csprofiles/csprofiles.go b/pkg/csprofiles/csprofiles.go index 52cda1ed2e1..c509fb448e3 100644 --- a/pkg/csprofiles/csprofiles.go +++ b/pkg/csprofiles/csprofiles.go @@ -96,17 +96,17 @@ func NewProfile(profilesCfg []*csconfig.ProfileCfg) ([]*Runtime, error) { return profilesRuntime, nil } -func (Profile *Runtime) GenerateDecisionFromProfile(Alert *models.Alert) ([]*models.Decision, error) { +func (profile *Runtime) GenerateDecisionFromProfile(alert *models.Alert) ([]*models.Decision, error) { var decisions []*models.Decision - for _, refDecision := range Profile.Cfg.Decisions { + for _, refDecision := range profile.Cfg.Decisions { decision := models.Decision{} /*the reference decision from profile is in simulated mode */ if refDecision.Simulated != nil && *refDecision.Simulated { decision.Simulated = new(bool) *decision.Simulated = true /*the event is already in simulation mode */ - } else if Alert.Simulated != nil && *Alert.Simulated { + } else if alert.Simulated != nil && *alert.Simulated { decision.Simulated = new(bool) *decision.Simulated = true } @@ -116,7 +116,7 @@ func (Profile *Runtime) GenerateDecisionFromProfile(Alert *models.Alert) ([]*mod if refDecision.Scope != nil && *refDecision.Scope != "" { *decision.Scope = *refDecision.Scope } else { - *decision.Scope = *Alert.Source.Scope + *decision.Scope = *alert.Source.Scope } /*some fields are populated from the reference object : duration, scope, type*/ @@ -125,19 +125,19 @@ func (Profile *Runtime) GenerateDecisionFromProfile(Alert *models.Alert) ([]*mod *decision.Duration = *refDecision.Duration } - if Profile.Cfg.DurationExpr != "" && Profile.RuntimeDurationExpr != nil { + if profile.Cfg.DurationExpr != "" && profile.RuntimeDurationExpr != nil { profileDebug := false - if Profile.Cfg.Debug != nil && *Profile.Cfg.Debug { + if profile.Cfg.Debug != nil && *profile.Cfg.Debug { profileDebug = true } - duration, err := exprhelpers.Run(Profile.RuntimeDurationExpr, map[string]interface{}{"Alert": Alert}, Profile.Logger, profileDebug) + duration, err := exprhelpers.Run(profile.RuntimeDurationExpr, map[string]interface{}{"Alert": alert}, profile.Logger, profileDebug) if err != nil { - Profile.Logger.Warningf("Failed to run duration_expr : %v", err) + profile.Logger.Warningf("Failed to run duration_expr : %v", err) } else { durationStr := fmt.Sprint(duration) if _, err := time.ParseDuration(durationStr); err != nil { - Profile.Logger.Warningf("Failed to parse expr duration result '%s'", duration) + profile.Logger.Warningf("Failed to parse expr duration result '%s'", duration) } else { *decision.Duration = durationStr } @@ -149,7 +149,7 @@ func (Profile *Runtime) GenerateDecisionFromProfile(Alert *models.Alert) ([]*mod /*for the others, let's populate it from the alert and its source*/ decision.Value = new(string) - *decision.Value = *Alert.Source.Value + *decision.Value = *alert.Source.Value decision.Origin = new(string) *decision.Origin = types.CrowdSecOrigin @@ -158,7 +158,7 @@ func (Profile *Runtime) GenerateDecisionFromProfile(Alert *models.Alert) ([]*mod } decision.Scenario = new(string) - *decision.Scenario = *Alert.Scenario + *decision.Scenario = *alert.Scenario decisions = append(decisions, &decision) } @@ -166,21 +166,21 @@ func (Profile *Runtime) GenerateDecisionFromProfile(Alert *models.Alert) ([]*mod } // EvaluateProfile is going to evaluate an Alert against a profile to generate Decisions -func (Profile *Runtime) EvaluateProfile(Alert *models.Alert) ([]*models.Decision, bool, error) { +func (profile *Runtime) EvaluateProfile(alert *models.Alert) ([]*models.Decision, bool, error) { var decisions []*models.Decision matched := false - for eIdx, expression := range Profile.RuntimeFilters { + for eIdx, expression := range profile.RuntimeFilters { debugProfile := false - if Profile.Cfg.Debug != nil && *Profile.Cfg.Debug { + if profile.Cfg.Debug != nil && *profile.Cfg.Debug { debugProfile = true } - output, err := exprhelpers.Run(expression, map[string]interface{}{"Alert": Alert}, Profile.Logger, debugProfile) + output, err := exprhelpers.Run(expression, map[string]interface{}{"Alert": alert}, profile.Logger, debugProfile) if err != nil { - Profile.Logger.Warningf("failed to run profile expr for %s: %v", Profile.Cfg.Name, err) - return nil, matched, fmt.Errorf("while running expression %s: %w", Profile.Cfg.Filters[eIdx], err) + profile.Logger.Warningf("failed to run profile expr for %s: %v", profile.Cfg.Name, err) + return nil, matched, fmt.Errorf("while running expression %s: %w", profile.Cfg.Filters[eIdx], err) } switch out := output.(type) { @@ -188,22 +188,22 @@ func (Profile *Runtime) EvaluateProfile(Alert *models.Alert) ([]*models.Decision if out { matched = true /*the expression matched, create the associated decision*/ - subdecisions, err := Profile.GenerateDecisionFromProfile(Alert) + subdecisions, err := profile.GenerateDecisionFromProfile(alert) if err != nil { - return nil, matched, fmt.Errorf("while generating decision from profile %s: %w", Profile.Cfg.Name, err) + return nil, matched, fmt.Errorf("while generating decision from profile %s: %w", profile.Cfg.Name, err) } decisions = append(decisions, subdecisions...) } else { - Profile.Logger.Debugf("Profile %s filter is unsuccessful", Profile.Cfg.Name) + profile.Logger.Debugf("Profile %s filter is unsuccessful", profile.Cfg.Name) - if Profile.Cfg.OnFailure == "break" { + if profile.Cfg.OnFailure == "break" { break } } default: - return nil, matched, fmt.Errorf("unexpected type %t (%v) while running '%s'", output, output, Profile.Cfg.Filters[eIdx]) + return nil, matched, fmt.Errorf("unexpected type %t (%v) while running '%s'", output, output, profile.Cfg.Filters[eIdx]) } } diff --git a/pkg/database/flush.go b/pkg/database/flush.go index 8f646ddc961..4a3a93a406c 100644 --- a/pkg/database/flush.go +++ b/pkg/database/flush.go @@ -222,7 +222,7 @@ func (c *Client) FlushAgentsAndBouncers(ctx context.Context, agentsCfg *csconfig return nil } -func (c *Client) FlushAlerts(ctx context.Context, MaxAge string, MaxItems int) error { +func (c *Client) FlushAlerts(ctx context.Context, maxAge string, maxItems int) error { var ( deletedByAge int deletedByNbItem int @@ -247,22 +247,22 @@ func (c *Client) FlushAlerts(ctx context.Context, MaxAge string, MaxItems int) e c.Log.Debugf("FlushAlerts (Total alerts): %d", totalAlerts) - if MaxAge != "" { + if maxAge != "" { filter := map[string][]string{ - "created_before": {MaxAge}, + "created_before": {maxAge}, } nbDeleted, err := c.DeleteAlertWithFilter(ctx, filter) if err != nil { c.Log.Warningf("FlushAlerts (max age): %s", err) - return fmt.Errorf("unable to flush alerts with filter until=%s: %w", MaxAge, err) + return fmt.Errorf("unable to flush alerts with filter until=%s: %w", maxAge, err) } c.Log.Debugf("FlushAlerts (deleted max age alerts): %d", nbDeleted) deletedByAge = nbDeleted } - if MaxItems > 0 { + if maxItems > 0 { // We get the highest id for the alerts // We subtract MaxItems to avoid deleting alerts that are not old enough // This gives us the oldest alert that we want to keep @@ -282,7 +282,7 @@ func (c *Client) FlushAlerts(ctx context.Context, MaxAge string, MaxItems int) e } if len(lastAlert) != 0 { - maxid := lastAlert[0].ID - MaxItems + maxid := lastAlert[0].ID - maxItems c.Log.Debugf("FlushAlerts (max id): %d", maxid) @@ -299,12 +299,12 @@ func (c *Client) FlushAlerts(ctx context.Context, MaxAge string, MaxItems int) e if deletedByNbItem > 0 { c.Log.Infof("flushed %d/%d alerts because the max number of alerts has been reached (%d max)", - deletedByNbItem, totalAlerts, MaxItems) + deletedByNbItem, totalAlerts, maxItems) } if deletedByAge > 0 { c.Log.Infof("flushed %d/%d alerts because they were created %s ago or more", - deletedByAge, totalAlerts, MaxAge) + deletedByAge, totalAlerts, maxAge) } return nil diff --git a/pkg/exprhelpers/crowdsec_cti.go b/pkg/exprhelpers/crowdsec_cti.go index 9b9eac4b95c..900bd7824a8 100644 --- a/pkg/exprhelpers/crowdsec_cti.go +++ b/pkg/exprhelpers/crowdsec_cti.go @@ -29,29 +29,29 @@ var ( var ctiClient *cticlient.CrowdsecCTIClient -func InitCrowdsecCTI(Key *string, TTL *time.Duration, Size *int, LogLevel *log.Level) error { - if Key == nil || *Key == "" { +func InitCrowdsecCTI(key *string, ttl *time.Duration, size *int, logLevel *log.Level) error { + if key == nil || *key == "" { log.Warningf("CTI API key not set or empty, CTI will not be available") return cticlient.ErrDisabled } - CTIApiKey = *Key - if Size == nil { - Size = new(int) - *Size = 1000 + CTIApiKey = *key + if size == nil { + size = new(int) + *size = 1000 } - if TTL == nil { - TTL = new(time.Duration) - *TTL = 5 * time.Minute + if ttl == nil { + ttl = new(time.Duration) + *ttl = 5 * time.Minute } clog := log.New() if err := types.ConfigureLogger(clog); err != nil { return fmt.Errorf("while configuring datasource logger: %w", err) } - if LogLevel != nil { - clog.SetLevel(*LogLevel) + if logLevel != nil { + clog.SetLevel(*logLevel) } subLogger := clog.WithField("type", "crowdsec-cti") - CrowdsecCTIInitCache(*Size, *TTL) + CrowdsecCTIInitCache(*size, *ttl) ctiClient = cticlient.NewCrowdsecCTIClient(cticlient.WithAPIKey(CTIApiKey), cticlient.WithLogger(subLogger)) CTIApiEnabled = true return nil diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index 9bc991a8f2d..96de0020ccc 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -129,32 +129,34 @@ func Init(databaseClient *database.Client) error { dataFileRegex = make(map[string][]*regexp.Regexp) dataFileRe2 = make(map[string][]*re2.Regexp) dbClient = databaseClient + XMLCacheInit() + return nil } -func RegexpCacheInit(filename string, CacheCfg types.DataSource) error { +func RegexpCacheInit(filename string, cacheCfg types.DataSource) error { // cache is explicitly disabled - if CacheCfg.Cache != nil && !*CacheCfg.Cache { + if cacheCfg.Cache != nil && !*cacheCfg.Cache { return nil } // cache is implicitly disabled if no cache config is provided - if CacheCfg.Strategy == nil && CacheCfg.TTL == nil && CacheCfg.Size == nil { + if cacheCfg.Strategy == nil && cacheCfg.TTL == nil && cacheCfg.Size == nil { return nil } // cache is enabled - if CacheCfg.Size == nil { - CacheCfg.Size = ptr.Of(50) + if cacheCfg.Size == nil { + cacheCfg.Size = ptr.Of(50) } - gc := gcache.New(*CacheCfg.Size) + gc := gcache.New(*cacheCfg.Size) - if CacheCfg.Strategy == nil { - CacheCfg.Strategy = ptr.Of("LRU") + if cacheCfg.Strategy == nil { + cacheCfg.Strategy = ptr.Of("LRU") } - switch *CacheCfg.Strategy { + switch *cacheCfg.Strategy { case "LRU": gc = gc.LRU() case "LFU": @@ -162,11 +164,11 @@ func RegexpCacheInit(filename string, CacheCfg types.DataSource) error { case "ARC": gc = gc.ARC() default: - return fmt.Errorf("unknown cache strategy '%s'", *CacheCfg.Strategy) + return fmt.Errorf("unknown cache strategy '%s'", *cacheCfg.Strategy) } - if CacheCfg.TTL != nil { - gc.Expiration(*CacheCfg.TTL) + if cacheCfg.TTL != nil { + gc.Expiration(*cacheCfg.TTL) } cache := gc.Build() @@ -240,6 +242,7 @@ func Distinct(params ...any) (any, error) { if rt := reflect.TypeOf(params[0]).Kind(); rt != reflect.Slice && rt != reflect.Array { return nil, nil } + array := params[0].([]interface{}) if array == nil { return []interface{}{}, nil @@ -254,6 +257,7 @@ func Distinct(params ...any) (any, error) { ret = append(ret, val) } } + return ret, nil } @@ -282,8 +286,10 @@ func flatten(args []interface{}, v reflect.Value) []interface{} { } func existsInFileMaps(filename string, ftype string) (bool, error) { - ok := false var err error + + ok := false + switch ftype { case "regex", "regexp": if fflag.Re2RegexpInfileSupport.IsEnabled() { @@ -296,10 +302,11 @@ func existsInFileMaps(filename string, ftype string) (bool, error) { default: err = fmt.Errorf("unknown data type '%s' for : '%s'", ftype, filename) } + return ok, err } -//Expr helpers +// Expr helpers // func Get(arr []string, index int) string { func Get(params ...any) (any, error) { @@ -315,10 +322,12 @@ func Get(params ...any) (any, error) { func Atof(params ...any) (any, error) { x := params[0].(string) log.Debugf("debug atof %s", x) + ret, err := strconv.ParseFloat(x, 64) if err != nil { log.Warningf("Atof : can't convert float '%s' : %v", x, err) } + return ret, nil } @@ -340,22 +349,28 @@ func Distance(params ...any) (any, error) { long1 := params[1].(string) lat2 := params[2].(string) long2 := params[3].(string) + lat1f, err := strconv.ParseFloat(lat1, 64) if err != nil { log.Warningf("lat1 is not a float : %v", err) + return 0.0, fmt.Errorf("lat1 is not a float : %v", err) } + long1f, err := strconv.ParseFloat(long1, 64) if err != nil { log.Warningf("long1 is not a float : %v", err) + return 0.0, fmt.Errorf("long1 is not a float : %v", err) } + lat2f, err := strconv.ParseFloat(lat2, 64) if err != nil { log.Warningf("lat2 is not a float : %v", err) return 0.0, fmt.Errorf("lat2 is not a float : %v", err) } + long2f, err := strconv.ParseFloat(long2, 64) if err != nil { log.Warningf("long2 is not a float : %v", err) @@ -363,7 +378,7 @@ func Distance(params ...any) (any, error) { return 0.0, fmt.Errorf("long2 is not a float : %v", err) } - //either set of coordinates is 0,0, return 0 to avoid FPs + // either set of coordinates is 0,0, return 0 to avoid FPs if (lat1f == 0.0 && long1f == 0.0) || (lat2f == 0.0 && long2f == 0.0) { log.Warningf("one of the coordinates is 0,0, returning 0") return 0.0, nil @@ -373,6 +388,7 @@ func Distance(params ...any) (any, error) { second := haversine.Coord{Lat: lat2f, Lon: long2f} _, km := haversine.Distance(first, second) + return km, nil } diff --git a/pkg/leakybucket/bayesian.go b/pkg/leakybucket/bayesian.go index 357d51f597b..30e1b396ef8 100644 --- a/pkg/leakybucket/bayesian.go +++ b/pkg/leakybucket/bayesian.go @@ -31,9 +31,9 @@ type BayesianBucket struct { DumbProcessor } -func updateProbability(prior, probGivenEvil, ProbGivenBenign float32) float32 { +func updateProbability(prior, probGivenEvil, probGivenBenign float32) float32 { numerator := probGivenEvil * prior - denominator := numerator + ProbGivenBenign*(1-prior) + denominator := numerator + probGivenBenign*(1-prior) return numerator / denominator } diff --git a/pkg/leakybucket/overflow_filter.go b/pkg/leakybucket/overflow_filter.go index 01dd491ed41..b37e431fadf 100644 --- a/pkg/leakybucket/overflow_filter.go +++ b/pkg/leakybucket/overflow_filter.go @@ -36,10 +36,10 @@ func NewOverflowFilter(g *BucketFactory) (*OverflowFilter, error) { return &u, nil } -func (u *OverflowFilter) OnBucketOverflow(Bucket *BucketFactory) func(*Leaky, types.RuntimeAlert, *types.Queue) (types.RuntimeAlert, *types.Queue) { +func (u *OverflowFilter) OnBucketOverflow(bucket *BucketFactory) func(*Leaky, types.RuntimeAlert, *types.Queue) (types.RuntimeAlert, *types.Queue) { return func(l *Leaky, s types.RuntimeAlert, q *types.Queue) (types.RuntimeAlert, *types.Queue) { el, err := exprhelpers.Run(u.FilterRuntime, map[string]interface{}{ - "queue": q, "signal": s, "leaky": l}, l.logger, Bucket.Debug) + "queue": q, "signal": s, "leaky": l}, l.logger, bucket.Debug) if err != nil { l.logger.Errorf("Failed running overflow filter: %s", err) return s, q From 49fb24c3b15cf04fded18489f4fd1c6794cb365c Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 16 Jan 2025 16:13:10 +0100 Subject: [PATCH 18/25] lint: enable errcheck; add allowlist and explicit checks (#3403) * lint: enable errcheck with explicit allow list * add explicit error checks * windows tests * windows nolint --- .golangci.yml | 35 ++++++++++-- cmd/crowdsec-cli/climachine/add.go | 8 ++- cmd/crowdsec-cli/completion.go | 8 +-- cmd/crowdsec/serve.go | 2 +- pkg/acquisition/modules/appsec/appsec.go | 5 +- .../modules/kubernetesaudit/k8s_audit.go | 4 +- .../wineventlog/wineventlog_windows_test.go | 56 +++++++++++-------- pkg/cticlient/example/fire.go | 10 +++- pkg/exprhelpers/helpers.go | 18 +++--- pkg/leakybucket/manager_load.go | 4 +- pkg/parser/node.go | 4 +- pkg/parser/stage.go | 6 +- pkg/setup/install.go | 4 +- 13 files changed, 112 insertions(+), 52 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 7df08cf717c..fe77aec2d3c 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -5,6 +5,37 @@ run: - expr_debug linters-settings: + errcheck: + # Report about not checking of errors in type assertions: `a := b.(MyStruct)`. + # Such cases aren't reported by default. + # Default: false + check-type-assertions: false + # List of functions to exclude from checking, where each entry is a single function to exclude. + # See https://github.com/kisielk/errcheck#excluding-functions for details. + exclude-functions: + - (*bytes.Buffer).ReadFrom # TODO: + - io.Copy # TODO: + - (net/http.ResponseWriter).Write # TODO: + - (*os/exec.Cmd).Start + - (*os/exec.Cmd).Wait + - (*os.Process).Kill + - (*text/template.Template).ExecuteTemplate + - syscall.FreeLibrary + - golang.org/x/sys/windows.CloseHandle + - golang.org/x/sys/windows.ResetEvent + - (*golang.org/x/sys/windows/svc/eventlog.Log).Info + - (*golang.org/x/sys/windows/svc/mgr.Mgr).Disconnect + + - (github.com/bluele/gcache.Cache).Set + - (github.com/gin-gonic/gin.ResponseWriter).WriteString + - (*github.com/segmentio/kafka-go.Reader).SetOffsetAt + - (*gopkg.in/tomb.v2.Tomb).Wait + + - (*github.com/crowdsecurity/crowdsec/pkg/appsec.ReqDumpFilter).FilterArgs + - (*github.com/crowdsecurity/crowdsec/pkg/appsec.ReqDumpFilter).FilterBody + - (*github.com/crowdsecurity/crowdsec/pkg/appsec.ReqDumpFilter).FilterHeaders + - (*github.com/crowdsecurity/crowdsec/pkg/longpollclient.LongPollClient).Stop + gci: sections: - standard @@ -318,10 +349,6 @@ issues: - govet text: "shadow: declaration of \"(err|ctx)\" shadows declaration" - - linters: - - errcheck - text: "Error return value of `.*` is not checked" - # Will fix, trivial - just beware of merge conflicts - linters: diff --git a/cmd/crowdsec-cli/climachine/add.go b/cmd/crowdsec-cli/climachine/add.go index 4f28119dde6..b2595583823 100644 --- a/cmd/crowdsec-cli/climachine/add.go +++ b/cmd/crowdsec-cli/climachine/add.go @@ -73,7 +73,9 @@ func (cli *cliMachines) add(ctx context.Context, args []string, machinePassword qs := &survey.Password{ Message: "Please provide a password for the machine:", } - survey.AskOne(qs, &machinePassword) + if err := survey.AskOne(qs, &machinePassword); err != nil { + return err + } } password := strfmt.Password(machinePassword) @@ -147,9 +149,9 @@ cscli machines add -f- --auto > /tmp/mycreds.yaml`, flags.VarP(&password, "password", "p", "machine password to login to the API") flags.StringVarP(&dumpFile, "file", "f", "", "output file destination (defaults to "+csconfig.DefaultConfigPath("local_api_credentials.yaml")+")") flags.StringVarP(&apiURL, "url", "u", "", "URL of the local API") - flags.BoolVarP(&interactive, "interactive", "i", false, "interfactive mode to enter the password") + flags.BoolVarP(&interactive, "interactive", "i", false, "interactive mode to enter the password") flags.BoolVarP(&autoAdd, "auto", "a", false, "automatically generate password (and username if not provided)") - flags.BoolVar(&force, "force", false, "will force add the machine if it already exist") + flags.BoolVar(&force, "force", false, "will force add the machine if it already exists") return cmd } diff --git a/cmd/crowdsec-cli/completion.go b/cmd/crowdsec-cli/completion.go index 7b6531f5516..fb60f9afab0 100644 --- a/cmd/crowdsec-cli/completion.go +++ b/cmd/crowdsec-cli/completion.go @@ -71,13 +71,13 @@ func NewCompletionCmd() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { switch args[0] { case "bash": - cmd.Root().GenBashCompletion(os.Stdout) + _ = cmd.Root().GenBashCompletion(os.Stdout) case "zsh": - cmd.Root().GenZshCompletion(os.Stdout) + _ = cmd.Root().GenZshCompletion(os.Stdout) case "powershell": - cmd.Root().GenPowerShellCompletion(os.Stdout) + _ = cmd.Root().GenPowerShellCompletion(os.Stdout) case "fish": - cmd.Root().GenFishCompletion(os.Stdout, true) + _ = cmd.Root().GenFishCompletion(os.Stdout, true) } }, } diff --git a/cmd/crowdsec/serve.go b/cmd/crowdsec/serve.go index 62b721befdb..0f7a84ce5c7 100644 --- a/cmd/crowdsec/serve.go +++ b/cmd/crowdsec/serve.go @@ -419,7 +419,7 @@ func Serve(cConfig *csconfig.Config, agentReady chan bool) error { } if cConfig.Common != nil && cConfig.Common.Daemonize { - csdaemon.Notify(csdaemon.Ready, log.StandardLogger()) + _ = csdaemon.Notify(csdaemon.Ready, log.StandardLogger()) // wait for signals return HandleSignals(cConfig) } diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index 86dbfe38b71..a4c2c5124b3 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -330,7 +330,10 @@ func (w *AppsecSource) StreamingAcquisition(ctx context.Context, out chan types. w.logger.Info("Shutting down Appsec server") // xx let's clean up the appsec runners :) appsec.AppsecRulesDetails = make(map[int]appsec.RulesDetails) - w.server.Shutdown(ctx) + + if err := w.server.Shutdown(ctx); err != nil { + w.logger.Errorf("Error shutting down Appsec server: %s", err.Error()) + } return nil }) diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go index f8c325b5c4b..b0650d3906e 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go @@ -154,7 +154,9 @@ func (ka *KubernetesAuditSource) StreamingAcquisition(ctx context.Context, out c }) <-t.Dying() ka.logger.Infof("Stopping k8s-audit server on %s:%d%s", ka.config.ListenAddr, ka.config.ListenPort, ka.config.WebhookPath) - ka.server.Shutdown(ctx) + if err := ka.server.Shutdown(ctx); err != nil { + ka.logger.Errorf("Error shutting down k8s-audit server: %s", err.Error()) + } return nil }) diff --git a/pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go b/pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go index 2f6fe15450f..b4998de76c4 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go @@ -7,18 +7,22 @@ import ( "testing" "time" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sys/windows/svc/eventlog" "gopkg.in/tomb.v2" + + "github.com/crowdsecurity/go-cs-lib/cstest" + + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" ) func TestBadConfiguration(t *testing.T) { - exprhelpers.Init(nil) + err := exprhelpers.Init(nil) + require.NoError(t, err) tests := []struct { config string @@ -62,7 +66,8 @@ xpath_query: test`, } func TestQueryBuilder(t *testing.T) { - exprhelpers.Init(nil) + err := exprhelpers.Init(nil) + require.NoError(t, err) tests := []struct { config string @@ -111,23 +116,26 @@ event_level: bla`, } subLogger := log.WithField("type", "windowseventlog") for _, test := range tests { - f := WinEventLogSource{} - f.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) - q, err := f.buildXpathQuery() - if test.expectedErr != "" { - if err == nil { - t.Fatalf("expected error '%s' but got none", test.expectedErr) + t.Run(test.config, func(t *testing.T) { + f := WinEventLogSource{} + + err := f.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) + cstest.AssertErrorContains(t, err, test.expectedErr) + if test.expectedErr != "" { + return } - assert.Contains(t, err.Error(), test.expectedErr) - } else { + + q, err := f.buildXpathQuery() require.NoError(t, err) assert.Equal(t, test.expectedQuery, q) - } + }) } } func TestLiveAcquisition(t *testing.T) { - exprhelpers.Init(nil) + err := exprhelpers.Init(nil) + require.NoError(t, err) + ctx := context.Background() tests := []struct { @@ -185,8 +193,13 @@ event_ids: to := &tomb.Tomb{} c := make(chan types.Event) f := WinEventLogSource{} - f.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) - f.StreamingAcquisition(ctx, c, to) + + err := f.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) + require.NoError(t, err) + + err = f.StreamingAcquisition(ctx, c, to) + require.NoError(t, err) + time.Sleep(time.Second) lines := test.expectedLines go func() { @@ -261,7 +274,8 @@ func TestOneShotAcquisition(t *testing.T) { }, } - exprhelpers.Init(nil) + err := exprhelpers.Init(nil) + require.NoError(t, err) for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -269,15 +283,13 @@ func TestOneShotAcquisition(t *testing.T) { to := &tomb.Tomb{} c := make(chan types.Event) f := WinEventLogSource{} - err := f.ConfigureByDSN(test.dsn, map[string]string{"type": "wineventlog"}, log.WithField("type", "windowseventlog"), "") + err := f.ConfigureByDSN(test.dsn, map[string]string{"type": "wineventlog"}, log.WithField("type", "windowseventlog"), "") + cstest.AssertErrorContains(t, err, test.expectedConfigureErr) if test.expectedConfigureErr != "" { - assert.Contains(t, err.Error(), test.expectedConfigureErr) return } - require.NoError(t, err) - go func() { for { select { diff --git a/pkg/cticlient/example/fire.go b/pkg/cticlient/example/fire.go index e52922571ef..598175ce02c 100644 --- a/pkg/cticlient/example/fire.go +++ b/pkg/cticlient/example/fire.go @@ -57,6 +57,12 @@ func main() { }) } } - csvWriter.Write(csvHeader) - csvWriter.WriteAll(allItems) + + if err = csvWriter.Write(csvHeader); err != nil { + panic(err) + } + + if err = csvWriter.WriteAll(allItems); err != nil { + panic(err) + } } diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index 96de0020ccc..d0f6f2cfe22 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -29,8 +29,6 @@ import ( "github.com/umahmood/haversine" "github.com/wasilibs/go-re2" - "github.com/crowdsecurity/go-cs-lib/ptr" - "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" @@ -146,17 +144,19 @@ func RegexpCacheInit(filename string, cacheCfg types.DataSource) error { } // cache is enabled - if cacheCfg.Size == nil { - cacheCfg.Size = ptr.Of(50) + size := 50 + if cacheCfg.Size != nil { + size = *cacheCfg.Size } - gc := gcache.New(*cacheCfg.Size) + gc := gcache.New(size) - if cacheCfg.Strategy == nil { - cacheCfg.Strategy = ptr.Of("LRU") + strategy := "LRU" + if cacheCfg.Strategy != nil { + strategy = *cacheCfg.Strategy } - switch *cacheCfg.Strategy { + switch strategy { case "LRU": gc = gc.LRU() case "LFU": @@ -164,7 +164,7 @@ func RegexpCacheInit(filename string, cacheCfg types.DataSource) error { case "ARC": gc = gc.ARC() default: - return fmt.Errorf("unknown cache strategy '%s'", *cacheCfg.Strategy) + return fmt.Errorf("unknown cache strategy '%s'", strategy) } if cacheCfg.TTL != nil { diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index 5e8bab8486e..1ed9c2d2980 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -458,7 +458,9 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error { } if data.Type == "regexp" { // cache only makes sense for regexp - exprhelpers.RegexpCacheInit(data.DestPath, *data) + if err := exprhelpers.RegexpCacheInit(data.DestPath, *data); err != nil { + bucketFactory.logger.Error(err.Error()) + } } } diff --git a/pkg/parser/node.go b/pkg/parser/node.go index 62a1ff6c4e2..1229a0f4470 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -353,7 +353,9 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri clog.Warningf("unexpected type %t (%v) while running '%s'", output, output, stash.Key) continue } - cache.SetKey(stash.Name, key, value, &stash.TTLVal) + if err = cache.SetKey(stash.Name, key, value, &stash.TTLVal); err != nil { + clog.Warningf("failed to store data in cache: %s", err.Error()) + } } } diff --git a/pkg/parser/stage.go b/pkg/parser/stage.go index b98db350254..ddc07ca7f1d 100644 --- a/pkg/parser/stage.go +++ b/pkg/parser/stage.go @@ -114,10 +114,12 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) ( for _, data := range node.Data { err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath, data.Type) if err != nil { - log.Error(err) + log.Error(err.Error()) } if data.Type == "regexp" { //cache only makes sense for regexp - exprhelpers.RegexpCacheInit(data.DestPath, *data) + if err = exprhelpers.RegexpCacheInit(data.DestPath, *data); err != nil { + log.Error(err.Error()) + } } } diff --git a/pkg/setup/install.go b/pkg/setup/install.go index 3d1540f23be..556ddab4c9a 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -192,7 +192,9 @@ func marshalAcquisDocuments(ads []AcquisDocument, toDir string) (string, error) return "", fmt.Errorf("while writing to %s: %w", ad.AcquisFilename, err) } - f.Sync() + if err = f.Sync(); err != nil { + return "", fmt.Errorf("while syncing %s: %w", ad.AcquisFilename, err) + } continue } From 7d12b806cdc5508eca914c226ce848e63853090f Mon Sep 17 00:00:00 2001 From: Laurence Jones Date: Sat, 18 Jan 2025 13:13:46 +0100 Subject: [PATCH 19/25] enhance: Log appsec error on writing response to remediation (#3412) * enhance: Log appsec error on writing response to remediation * fix: note to self dont write code at midnight --- pkg/acquisition/modules/appsec/appsec.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index a4c2c5124b3..78225d5f8c3 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -446,6 +446,8 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { logger.Errorf("unable to serialize response: %s", err) rw.WriteHeader(http.StatusInternalServerError) } else { - rw.Write(body) + if _, err := rw.Write(body); err != nil { + logger.Errorf("unable to write response: %s", err) + } } } From bd7e1b50c32d3569c2f5b9d41c774fbe926f21b2 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 20 Jan 2025 10:30:36 +0100 Subject: [PATCH 20/25] cscli: cliconfig - remove global variables and gratuitous pointer (#3414) --- .../{config_backup.go => cliconfig/backup.go} | 2 +- cmd/crowdsec-cli/{ => cliconfig}/config.go | 14 ++++++++++---- .../feature_flags.go} | 4 ++-- .../{config_restore.go => cliconfig/restore.go} | 2 +- .../{config_show.go => cliconfig/show.go} | 2 +- .../{config_showyaml.go => cliconfig/showyaml.go} | 8 ++++---- cmd/crowdsec-cli/clisupport/support.go | 2 +- cmd/crowdsec-cli/main.go | 3 ++- cmd/crowdsec-cli/require/require.go | 2 +- pkg/csconfig/config.go | 9 +++++---- 10 files changed, 28 insertions(+), 20 deletions(-) rename cmd/crowdsec-cli/{config_backup.go => cliconfig/backup.go} (95%) rename cmd/crowdsec-cli/{ => cliconfig}/config.go (58%) rename cmd/crowdsec-cli/{config_feature_flags.go => cliconfig/feature_flags.go} (96%) rename cmd/crowdsec-cli/{config_restore.go => cliconfig/restore.go} (95%) rename cmd/crowdsec-cli/{config_show.go => cliconfig/show.go} (99%) rename cmd/crowdsec-cli/{config_showyaml.go => cliconfig/showyaml.go} (62%) diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/cliconfig/backup.go similarity index 95% rename from cmd/crowdsec-cli/config_backup.go rename to cmd/crowdsec-cli/cliconfig/backup.go index 0a58a8c1ab3..5cd34fcf07f 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/cliconfig/backup.go @@ -1,4 +1,4 @@ -package main +package cliconfig import ( "fmt" diff --git a/cmd/crowdsec-cli/config.go b/cmd/crowdsec-cli/cliconfig/config.go similarity index 58% rename from cmd/crowdsec-cli/config.go rename to cmd/crowdsec-cli/cliconfig/config.go index 4cf8916ad4b..22095ac7d5b 100644 --- a/cmd/crowdsec-cli/config.go +++ b/cmd/crowdsec-cli/cliconfig/config.go @@ -1,20 +1,26 @@ -package main +package cliconfig import ( "github.com/spf13/cobra" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) +type configGetter func() *csconfig.Config + +type mergedConfigGetter func() string + type cliConfig struct { cfg configGetter } -func NewCLIConfig(cfg configGetter) *cliConfig { +func New(cfg configGetter) *cliConfig { return &cliConfig{ cfg: cfg, } } -func (cli *cliConfig) NewCommand() *cobra.Command { +func (cli *cliConfig) NewCommand(mergedConfigGetter mergedConfigGetter) *cobra.Command { cmd := &cobra.Command{ Use: "config [command]", Short: "Allows to view current config", @@ -23,7 +29,7 @@ func (cli *cliConfig) NewCommand() *cobra.Command { } cmd.AddCommand(cli.newShowCmd()) - cmd.AddCommand(cli.newShowYAMLCmd()) + cmd.AddCommand(cli.newShowYAMLCmd(mergedConfigGetter)) cmd.AddCommand(cli.newBackupCmd()) cmd.AddCommand(cli.newRestoreCmd()) cmd.AddCommand(cli.newFeatureFlagsCmd()) diff --git a/cmd/crowdsec-cli/config_feature_flags.go b/cmd/crowdsec-cli/cliconfig/feature_flags.go similarity index 96% rename from cmd/crowdsec-cli/config_feature_flags.go rename to cmd/crowdsec-cli/cliconfig/feature_flags.go index 760e2194bb3..c03db10ccce 100644 --- a/cmd/crowdsec-cli/config_feature_flags.go +++ b/cmd/crowdsec-cli/cliconfig/feature_flags.go @@ -1,4 +1,4 @@ -package main +package cliconfig import ( "fmt" @@ -86,7 +86,7 @@ func (cli *cliConfig) featureFlags(showRetired bool) error { fmt.Println("To enable a feature you can: ") fmt.Println(" - set the environment variable CROWDSEC_FEATURE_ to true") - featurePath, err := filepath.Abs(csconfig.GetFeatureFilePath(ConfigFilePath)) + featurePath, err := filepath.Abs(csconfig.GetFeatureFilePath(cli.cfg().FilePath)) if err != nil { // we already read the file, shouldn't happen return err diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/cliconfig/restore.go similarity index 95% rename from cmd/crowdsec-cli/config_restore.go rename to cmd/crowdsec-cli/cliconfig/restore.go index 75373475ed9..d368b27ea30 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/cliconfig/restore.go @@ -1,4 +1,4 @@ -package main +package cliconfig import ( "fmt" diff --git a/cmd/crowdsec-cli/config_show.go b/cmd/crowdsec-cli/cliconfig/show.go similarity index 99% rename from cmd/crowdsec-cli/config_show.go rename to cmd/crowdsec-cli/cliconfig/show.go index 3d17d264574..90c0ab71069 100644 --- a/cmd/crowdsec-cli/config_show.go +++ b/cmd/crowdsec-cli/cliconfig/show.go @@ -1,4 +1,4 @@ -package main +package cliconfig import ( "encoding/json" diff --git a/cmd/crowdsec-cli/config_showyaml.go b/cmd/crowdsec-cli/cliconfig/showyaml.go similarity index 62% rename from cmd/crowdsec-cli/config_showyaml.go rename to cmd/crowdsec-cli/cliconfig/showyaml.go index 10549648d09..2e46a0171ab 100644 --- a/cmd/crowdsec-cli/config_showyaml.go +++ b/cmd/crowdsec-cli/cliconfig/showyaml.go @@ -1,4 +1,4 @@ -package main +package cliconfig import ( "fmt" @@ -6,19 +6,19 @@ import ( "github.com/spf13/cobra" ) -func (cli *cliConfig) showYAML() error { +func (cli *cliConfig) showYAML(mergedConfig string) error { fmt.Println(mergedConfig) return nil } -func (cli *cliConfig) newShowYAMLCmd() *cobra.Command { +func (cli *cliConfig) newShowYAMLCmd(mergedConfigGetter mergedConfigGetter) *cobra.Command { cmd := &cobra.Command{ Use: "show-yaml", Short: "Displays merged config.yaml + config.yaml.local", Args: cobra.NoArgs, DisableAutoGenTag: true, RunE: func(_ *cobra.Command, _ []string) error { - return cli.showYAML() + return cli.showYAML(mergedConfigGetter()) }, } diff --git a/cmd/crowdsec-cli/clisupport/support.go b/cmd/crowdsec-cli/clisupport/support.go index 5f6032a17bd..eb3e03df253 100644 --- a/cmd/crowdsec-cli/clisupport/support.go +++ b/cmd/crowdsec-cli/clisupport/support.go @@ -290,7 +290,7 @@ func (cli *cliSupport) dumpConfigYAML(zw *zip.Writer) error { cfg := cli.cfg() - config, err := os.ReadFile(*cfg.FilePath) + config, err := os.ReadFile(cfg.FilePath) if err != nil { return fmt.Errorf("could not read config file: %w", err) } diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 936211be7ff..a17bafb96d8 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -17,6 +17,7 @@ import ( "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clialert" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clibouncer" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clicapi" + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/cliconfig" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/cliconsole" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clidecision" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/cliexplain" @@ -256,7 +257,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall cmd.AddCommand(NewCLIDoc().NewCommand(cmd)) cmd.AddCommand(NewCLIVersion().NewCommand()) - cmd.AddCommand(NewCLIConfig(cli.cfg).NewCommand()) + cmd.AddCommand(cliconfig.New(cli.cfg).NewCommand(func() string { return mergedConfig })) cmd.AddCommand(clihub.New(cli.cfg).NewCommand()) cmd.AddCommand(climetrics.New(cli.cfg).NewCommand()) cmd.AddCommand(NewCLIDashboard(cli.cfg).NewCommand()) diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index dd98cd092cb..beffa29f3eb 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -27,7 +27,7 @@ func LAPI(c *csconfig.Config) error { func CAPI(c *csconfig.Config) error { if c.API.Server.OnlineClient == nil { - return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath) + return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", c.FilePath) } return nil diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index 3bbdf607187..b0784e5e6f3 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -30,7 +30,7 @@ var globalConfig = Config{} // Config contains top-level defaults -> overridden by configuration file -> overridden by CLI flags type Config struct { // just a path to ourselves :p - FilePath *string `yaml:"-"` + FilePath string `yaml:"-"` Self []byte `yaml:"-"` Common *CommonCfg `yaml:"common,omitempty"` Prometheus *PrometheusCfg `yaml:"prometheus,omitempty"` @@ -45,9 +45,10 @@ type Config struct { Hub *LocalHubCfg `yaml:"-"` } -func NewConfig(configFile string, disableAgent bool, disableAPI bool, inCli bool) (*Config, string, error) { +// NewConfig +func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) { patcher := yamlpatch.NewPatcher(configFile, ".local") - patcher.SetQuiet(inCli) + patcher.SetQuiet(quiet) fcontent, err := patcher.MergedPatchContent() if err != nil { @@ -56,7 +57,7 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, inCli bool configData := csstring.StrictExpand(string(fcontent), os.LookupEnv) cfg := Config{ - FilePath: &configFile, + FilePath: configFile, DisableAgent: disableAgent, DisableAPI: disableAPI, } From ff9b350e0e23d4addba01c7904237206a8045fec Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 20 Jan 2025 10:31:55 +0100 Subject: [PATCH 21/25] deb build: get version with dpkg-parsechangelog -S (#3405) --- debian/rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/rules b/debian/rules index 5b8d6fc51f8..ec80caff985 100755 --- a/debian/rules +++ b/debian/rules @@ -1,6 +1,6 @@ #!/usr/bin/make -f -export DEB_VERSION=$(shell dpkg-parsechangelog | grep -E '^Version:' | cut -f 2 -d ' ') +export DEB_VERSION=$(shell dpkg-parsechangelog -SVersion) export BUILD_VERSION=v${DEB_VERSION}-debian-pragmatic export GO111MODULE=on From d6b3841f13aa578c199ef25a5842f2187977e85f Mon Sep 17 00:00:00 2001 From: AlteredCoder <64792091+AlteredCoder@users.noreply.github.com> Date: Mon, 20 Jan 2025 11:21:25 +0100 Subject: [PATCH 22/25] pkg/cticlient: Add missing field in SmokeItem and FireItem (#3413) * pkg/cticlient: Add missing field in SmokeItem and FireItem --- pkg/cticlient/types.go | 32 ++++++++++++++++++++++++++++++++ pkg/cticlient/types_test.go | 12 ++++++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/pkg/cticlient/types.go b/pkg/cticlient/types.go index 954d24641b4..5ea29d6c5b0 100644 --- a/pkg/cticlient/types.go +++ b/pkg/cticlient/types.go @@ -64,6 +64,9 @@ type CTIReferences struct { type SmokeItem struct { IpRangeScore int `json:"ip_range_score"` Ip string `json:"ip"` + Reputation string `json:"reputation"` + BackgroundNoise string `json:"background_noise"` + Confidence string `json:"confidence"` IpRange *string `json:"ip_range"` AsName *string `json:"as_name"` AsNum *int `json:"as_num"` @@ -77,6 +80,7 @@ type SmokeItem struct { BackgroundNoiseScore *int `json:"background_noise_score"` Scores CTIScores `json:"scores"` References []CTIReferences `json:"references"` + CVEs []string `json:"cves"` IsOk bool `json:"-"` } @@ -120,6 +124,10 @@ type FireItem struct { BackgroundNoiseScore *int `json:"background_noise_score"` Scores CTIScores `json:"scores"` References []CTIReferences `json:"references"` + CVEs []string `json:"cves"` + Reputation string `json:"reputation"` + BackgroundNoise string `json:"background_noise"` + Confidence string `json:"confidence"` State string `json:"state"` Expiration CustomTime `json:"expiration"` } @@ -209,6 +217,18 @@ func (c *SmokeItem) GetFalsePositives() []string { return ret } +func (c *SmokeItem) GetClassifications() []string { + ret := make([]string, 0) + + if c.Classifications.Classifications != nil { + for _, b := range c.Classifications.Classifications { + ret = append(ret, b.Name) + } + } + + return ret +} + func (c *SmokeItem) IsFalsePositive() bool { if c.Classifications.FalsePositives != nil { if len(c.Classifications.FalsePositives) > 0 { @@ -282,6 +302,18 @@ func (c *FireItem) GetFalsePositives() []string { return ret } +func (c *FireItem) GetClassifications() []string { + ret := make([]string, 0) + + if c.Classifications.Classifications != nil { + for _, b := range c.Classifications.Classifications { + ret = append(ret, b.Name) + } + } + + return ret +} + func (c *FireItem) IsFalsePositive() bool { if c.Classifications.FalsePositives != nil { if len(c.Classifications.FalsePositives) > 0 { diff --git a/pkg/cticlient/types_test.go b/pkg/cticlient/types_test.go index a7308af35e0..9c7840de324 100644 --- a/pkg/cticlient/types_test.go +++ b/pkg/cticlient/types_test.go @@ -40,8 +40,14 @@ func getSampleSmokeItem() SmokeItem { DaysAge: 1, }, Classifications: CTIClassifications{ - FalsePositives: []CTIClassification{}, - Classifications: []CTIClassification{}, + FalsePositives: []CTIClassification{}, + Classifications: []CTIClassification{ + { + Name: "profile:likely_botnet", + Label: "Likely Botnet", + Description: "IP appears to be a botnet.", + }, + }, }, AttackDetails: []*CTIAttackDetails{ { @@ -101,6 +107,7 @@ func TestBasicSmokeItem(t *testing.T) { assert.Equal(t, 3, item.GetBackgroundNoiseScore()) assert.Equal(t, []string{}, item.GetFalsePositives()) assert.False(t, item.IsFalsePositive()) + assert.Equal(t, []string{"profile:likely_botnet"}, item.GetClassifications()) } func TestEmptySmokeItem(t *testing.T) { @@ -112,4 +119,5 @@ func TestEmptySmokeItem(t *testing.T) { assert.Equal(t, 0, item.GetBackgroundNoiseScore()) assert.Equal(t, []string{}, item.GetFalsePositives()) assert.False(t, item.IsFalsePositive()) + assert.Equal(t, []string{}, item.GetClassifications()) } From e1a1b1e565f4806efb313a6bf04a624a0354f52d Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 20 Jan 2025 14:16:01 +0100 Subject: [PATCH 23/25] deb packaging: don't duplicate crowdsec.service if /lib is not linked to /usr/lib (#3415) We don't need to explicitly copy the file, but the standard location is /usr/lib, not /lib - hence we must avoid duplicating it by copying explicitly if /lib is not linked to /usr/lib. Also we don't need preinst anymore and the message is in postinst as well. --- debian/install | 1 - debian/preinst | 8 -------- 2 files changed, 9 deletions(-) delete mode 100644 debian/preinst diff --git a/debian/install b/debian/install index fa422cac8d9..2d4cc6e1a7f 100644 --- a/debian/install +++ b/debian/install @@ -3,7 +3,6 @@ config/profiles.yaml etc/crowdsec/ config/simulation.yaml etc/crowdsec/ config/patterns/* etc/crowdsec/patterns -config/crowdsec.service lib/systemd/system # Referenced configs: cmd/notification-slack/slack.yaml etc/crowdsec/notifications/ diff --git a/debian/preinst b/debian/preinst deleted file mode 100644 index df5b56cef3f..00000000000 --- a/debian/preinst +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -e - -# Source debconf library. -. /usr/share/debconf/confmodule - -echo "You can always run the configuration again interactively by using '/usr/share/crowdsec/wizard.sh -c'" From 013fd8b198f63b7f63f6d2ad302d8c1cc16f7792 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 20 Jan 2025 15:01:34 +0100 Subject: [PATCH 24/25] remove dependency from github.com/gofrs/uuid (#3406) --- go.mod | 1 - pkg/longpollclient/client.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index d002aa43d9b..44dcd9c6753 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,6 @@ require ( github.com/go-openapi/validate v0.20.0 github.com/go-sql-driver/mysql v1.6.0 github.com/goccy/go-yaml v1.11.0 - github.com/gofrs/uuid v4.0.0+incompatible github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.1 github.com/golang/protobuf v1.5.4 // indirect diff --git a/pkg/longpollclient/client.go b/pkg/longpollclient/client.go index 5c395185b20..6a668e07d84 100644 --- a/pkg/longpollclient/client.go +++ b/pkg/longpollclient/client.go @@ -10,7 +10,7 @@ import ( "net/url" "time" - "github.com/gofrs/uuid" + "github.com/google/uuid" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" ) From 29b2252b50c6acab330148725c88f1e5416226a4 Mon Sep 17 00:00:00 2001 From: blotus Date: Mon, 20 Jan 2025 15:57:26 +0100 Subject: [PATCH 25/25] Improve post-installation message (#3407) --- debian/postinst | 16 ++++++++++++---- rpm/SPECS/crowdsec.spec | 16 ++++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/debian/postinst b/debian/postinst index d50a7c0bfe2..ed537325c44 100644 --- a/debian/postinst +++ b/debian/postinst @@ -89,10 +89,18 @@ if [ "$1" = configure ]; then echo "This port is configured through /etc/crowdsec/config.yaml and /etc/crowdsec/local_api_credentials.yaml" fi - echo "Get started with CrowdSec:" - echo " * Detailed guides are available in our documentation: https://docs.crowdsec.net" - echo " * Configuration items created by the community can be found at the Hub: https://hub.crowdsec.net" - echo " * Gain insights into your use of CrowdSec with the help of the console https://app.crowdsec.net" + GREEN='\033[0;32m' + BOLD='\033[1m' + RESET='\033[0m' + + echo -e "${BOLD}Get started with CrowdSec:${RESET}" + echo -e " * Go further by following our ${BOLD}post installation steps${RESET} : ${GREEN}${BOLD}https://docs.crowdsec.net/u/getting_started/next_steps${RESET}" + echo -e "====================================================================================================================" + echo -e " * Install a ${BOLD}remediation component${RESET} to block attackers: ${GREEN}${BOLD}https://docs.crowdsec.net/u/bouncers/intro${RESET}" + echo -e "====================================================================================================================" + echo -e " * Find more ${BOLD}collections${RESET}, ${BOLD}parsers${RESET} and ${BOLD}scenarios${RESET} created by the community with the Hub: ${GREEN}${BOLD}https://hub.crowdsec.net${RESET}" + echo -e "====================================================================================================================" + echo -e " * Subscribe to ${BOLD}additional blocklists${RESET}, ${BOLD}visualize${RESET} your alerts and more with the console: ${GREEN}${BOLD}https://app.crowdsec.net${RESET}" fi echo "You can always run the configuration again interactively by using '/usr/share/crowdsec/wizard.sh -c'" diff --git a/rpm/SPECS/crowdsec.spec b/rpm/SPECS/crowdsec.spec index c24b3f2ac0d..eba022d9bda 100644 --- a/rpm/SPECS/crowdsec.spec +++ b/rpm/SPECS/crowdsec.spec @@ -178,10 +178,18 @@ if [ $1 == 1 ]; then cscli hub update CSCLI_BIN_INSTALLED="/usr/bin/cscli" SILENT=true install_collection - echo "Get started with CrowdSec:" - echo " * Detailed guides are available in our documentation: https://docs.crowdsec.net" - echo " * Configuration items created by the community can be found at the Hub: https://hub.crowdsec.net" - echo " * Gain insights into your use of CrowdSec with the help of the console https://app.crowdsec.net" + GREEN='\033[0;32m' + BOLD='\033[1m' + RESET='\033[0m' + + echo -e "${BOLD}Get started with CrowdSec:${RESET}" + echo -e " * Go further by following our ${BOLD}post installation steps${RESET} : ${GREEN}${BOLD}https://docs.crowdsec.net/u/getting_started/next_steps${RESET}" + echo -e "====================================================================================================================" + echo -e " * Install a ${BOLD}remediation component${RESET} to block attackers: ${GREEN}${BOLD}https://docs.crowdsec.net/u/bouncers/intro${RESET}" + echo -e "====================================================================================================================" + echo -e " * Find more ${BOLD}collections${RESET}, ${BOLD}parsers${RESET} and ${BOLD}scenarios${RESET} created by the community with the Hub: ${GREEN}${BOLD}https://hub.crowdsec.net${RESET}" + echo -e "====================================================================================================================" + echo -e " * Subscribe to ${BOLD}additional blocklists${RESET}, ${BOLD}visualize${RESET} your alerts and more with the console: ${GREEN}${BOLD}https://app.crowdsec.net${RESET}" fi %systemd_post %{name}.service