Merge branch 'master' into naive-rfc3339

.github/workflows/go-tests-windows.yml

@@ -61,6 +61,6 @@ jobs:
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v6
       with:
-        version: v1.62
+        version: v1.63
         args: --issues-exit-code=1 --timeout 10m
         only-new-issues: false

.github/workflows/go-tests.yml

@@ -143,11 +143,11 @@ jobs:
         go generate ./...
         protoc --version
         if [[ $(git status --porcelain) ]]; then
-          echo "Error: Uncommitted changes found after running 'make generate'. Please commit all generated code."
+          echo "Error: Uncommitted changes found after running 'go generate'. Please commit all generated code."
           git diff
           exit 1
         else
-          echo "No changes detected after running 'make generate'."
+          echo "No changes detected after running 'go generate'."
         fi
 
     - name: Create localstack streams
@@ -189,6 +189,6 @@ jobs:
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v6
       with:
-        version: v1.62
+        version: v1.63
         args: --issues-exit-code=1 --timeout 10m
         only-new-issues: false

.golangci.yml

@@ -1,6 +1,41 @@
 # https://github.com/golangci/golangci-lint/blob/master/.golangci.reference.yml
 
+run:
+  build-tags:
+    - expr_debug
+
 linters-settings:
+  errcheck:
+    # Report about not checking of errors in type assertions: `a := b.(MyStruct)`.
+    # Such cases aren't reported by default.
+    # Default: false
+    check-type-assertions: false
+    # List of functions to exclude from checking, where each entry is a single function to exclude.
+    # See https://github.com/kisielk/errcheck#excluding-functions for details.
+    exclude-functions:
+     - (*bytes.Buffer).ReadFrom   # TODO:
+     - io.Copy                    # TODO:
+     - (net/http.ResponseWriter).Write    # TODO:
+     - (*os/exec.Cmd).Start
+     - (*os/exec.Cmd).Wait
+     - (*os.Process).Kill
+     - (*text/template.Template).ExecuteTemplate
+     - syscall.FreeLibrary
+     - golang.org/x/sys/windows.CloseHandle
+     - golang.org/x/sys/windows.ResetEvent
+     - (*golang.org/x/sys/windows/svc/eventlog.Log).Info
+     - (*golang.org/x/sys/windows/svc/mgr.Mgr).Disconnect
+
+     - (github.com/bluele/gcache.Cache).Set
+     - (github.com/gin-gonic/gin.ResponseWriter).WriteString
+     - (*github.com/segmentio/kafka-go.Reader).SetOffsetAt
+     - (*gopkg.in/tomb.v2.Tomb).Wait
+
+     - (*github.com/crowdsecurity/crowdsec/pkg/appsec.ReqDumpFilter).FilterArgs
+     - (*github.com/crowdsecurity/crowdsec/pkg/appsec.ReqDumpFilter).FilterBody
+     - (*github.com/crowdsecurity/crowdsec/pkg/appsec.ReqDumpFilter).FilterHeaders
+     - (*github.com/crowdsecurity/crowdsec/pkg/longpollclient.LongPollClient).Stop
+
   gci:
     sections:
      - standard
@@ -188,7 +223,6 @@ linters-settings:
       - unnamedResult
       - sloppyReassign
       - appendCombine
-      - captLocal
       - typeUnparen
       - commentFormatting
       - deferInLoop             #
@@ -315,10 +349,6 @@ issues:
         - govet
       text: "shadow: declaration of \"(err|ctx)\" shadows declaration"
 
-    - linters:
-        - errcheck
-      text: "Error return value of `.*` is not checked"
-
     # Will fix, trivial - just beware of merge conflicts
 
     - linters:
@@ -456,3 +486,14 @@ issues:
         - revive
       path: "pkg/types/utils.go"
       text: "argument-limit: .*"
+
+    # need some cleanup first: to create db in memory and share the client, not the config
+    - linters:
+        - usetesting
+      path: "pkg/apiserver/(.+)_test.go"
+      text: "os.MkdirTemp.* could be replaced by t.TempDir.*"
+
+    - linters:
+        - usetesting
+      path: "pkg/apiserver/(.+)_test.go"
+      text: "os.CreateTemp.* could be replaced by os.CreateTemp.*"

cmd/crowdsec-cli/clialert/table.go

@@ -86,7 +86,7 @@ func alertDecisionsTable(out io.Writer, wantColor string, alert *models.Alert) {
 	}
 
 	if foundActive {
-		fmt.Printf(" - Active Decisions  :\n")
+		t.Writer.SetTitle("Active Decisions")
 		t.Render() // Send output
 	}
 }

cmd/crowdsec-cli/clibouncer/inspect.go

@@ -47,7 +47,7 @@ func (cli *cliBouncers) inspectHuman(out io.Writer, bouncer *ent.Bouncer) {
 		t.AppendRow(table.Row{"Feature Flags", ff})
 	}
 
-	io.WriteString(out, t.Render()+"\n")
+	fmt.Fprint(out, t.Render())
 }
 
 func (cli *cliBouncers) inspect(bouncer *ent.Bouncer) error {

cmd/crowdsec-cli/clibouncer/list.go

@@ -37,7 +37,7 @@ func (cli *cliBouncers) listHuman(out io.Writer, bouncers ent.Bouncers) {
 		t.AppendRow(table.Row{b.Name, b.IPAddress, revoked, lastPull, b.Type, b.Version, b.AuthType})
 	}
 
-	io.WriteString(out, t.Render()+"\n")
+	fmt.Fprintln(out, t.Render())
 }
 
 func (cli *cliBouncers) listCSV(out io.Writer, bouncers ent.Bouncers) error {
@@ -71,7 +71,6 @@ func (cli *cliBouncers) listCSV(out io.Writer, bouncers ent.Bouncers) error {
 func (cli *cliBouncers) List(ctx context.Context, out io.Writer, db *database.Client) error {
 	// XXX: must use the provided db object, the one in the struct might be nil
 	// (calling List directly skips the PersistentPreRunE)
-
 	bouncers, err := db.ListBouncers(ctx)
 	if err != nil {
 		return fmt.Errorf("unable to list bouncers: %w", err)

cmd/crowdsec-cli/cliconfig/backup.go

@@ -0,0 +1,20 @@
+package cliconfig
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+func (cli *cliConfig) newBackupCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               "backup",
+		DisableAutoGenTag: true,
+		RunE: func(_ *cobra.Command, _ []string) error {
+			configDir := cli.cfg().ConfigPaths.ConfigDir
+			return fmt.Errorf("'cscli config backup' has been removed, you can manually backup/restore %s instead", configDir)
+		},
+	}
+
+	return cmd
+}

cmd/crowdsec-cli/config.go → cmd/crowdsec-cli/cliconfig/config.go

@@ -1,20 +1,26 @@
-package main
+package cliconfig
 
 import (
 	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
+type configGetter func() *csconfig.Config
+
+type mergedConfigGetter func() string
+
 type cliConfig struct {
 	cfg configGetter
 }
 
-func NewCLIConfig(cfg configGetter) *cliConfig {
+func New(cfg configGetter) *cliConfig {
 	return &cliConfig{
 		cfg: cfg,
 	}
 }
 
-func (cli *cliConfig) NewCommand() *cobra.Command {
+func (cli *cliConfig) NewCommand(mergedConfigGetter mergedConfigGetter) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:               "config [command]",
 		Short:             "Allows to view current config",
@@ -23,7 +29,7 @@ func (cli *cliConfig) NewCommand() *cobra.Command {
 	}
 
 	cmd.AddCommand(cli.newShowCmd())
-	cmd.AddCommand(cli.newShowYAMLCmd())
+	cmd.AddCommand(cli.newShowYAMLCmd(mergedConfigGetter))
 	cmd.AddCommand(cli.newBackupCmd())
 	cmd.AddCommand(cli.newRestoreCmd())
 	cmd.AddCommand(cli.newFeatureFlagsCmd())

cmd/crowdsec-cli/config_feature_flags.go → cmd/crowdsec-cli/cliconfig/feature_flags.go

@@ -1,4 +1,4 @@
-package main
+package cliconfig
 
 import (
 	"fmt"
@@ -86,7 +86,7 @@ func (cli *cliConfig) featureFlags(showRetired bool) error {
 	fmt.Println("To enable a feature you can: ")
 	fmt.Println("  - set the environment variable CROWDSEC_FEATURE_<uppercase_feature_name> to true")
 
-	featurePath, err := filepath.Abs(csconfig.GetFeatureFilePath(ConfigFilePath))
+	featurePath, err := filepath.Abs(csconfig.GetFeatureFilePath(cli.cfg().FilePath))
 	if err != nil {
 		// we already read the file, shouldn't happen
 		return err

cmd/crowdsec-cli/cliconfig/restore.go

@@ -0,0 +1,20 @@
+package cliconfig
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+func (cli *cliConfig) newRestoreCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               "restore",
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			configDir := cli.cfg().ConfigPaths.ConfigDir
+			return fmt.Errorf("'cscli config restore' has been removed, you can manually backup/restore %s instead", configDir)
+		},
+	}
+
+	return cmd
+}

cmd/crowdsec-cli/config_show.go → cmd/crowdsec-cli/cliconfig/show.go

@@ -1,4 +1,4 @@
-package main
+package cliconfig
 
 import (
 	"encoding/json"

cmd/crowdsec-cli/config_showyaml.go → cmd/crowdsec-cli/cliconfig/showyaml.go

@@ -1,24 +1,24 @@
-package main
+package cliconfig
 
 import (
 	"fmt"
 
 	"github.com/spf13/cobra"
 )
 
-func (cli *cliConfig) showYAML() error {
+func (cli *cliConfig) showYAML(mergedConfig string) error {
 	fmt.Println(mergedConfig)
 	return nil
 }
 
-func (cli *cliConfig) newShowYAMLCmd() *cobra.Command {
+func (cli *cliConfig) newShowYAMLCmd(mergedConfigGetter mergedConfigGetter) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:               "show-yaml",
 		Short:             "Displays merged config.yaml + config.yaml.local",
 		Args:              cobra.NoArgs,
 		DisableAutoGenTag: true,
 		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.showYAML()
+			return cli.showYAML(mergedConfigGetter())
 		},
 	}
 

cmd/crowdsec-cli/clidecision/decisions.go

@@ -170,7 +170,7 @@ func (cli *cliDecisions) NewCommand() *cobra.Command {
 	return cmd
 }
 
-func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOpts, NoSimu *bool, contained *bool, printMachine bool) error {
+func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOpts, noSimu *bool, contained *bool, printMachine bool) error {
 	var err error
 
 	*filter.ScopeEquals, err = clialert.SanitizeScope(*filter.ScopeEquals, *filter.IPEquals, *filter.RangeEquals)
@@ -181,7 +181,7 @@ func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOp
 	filter.ActiveDecisionEquals = new(bool)
 	*filter.ActiveDecisionEquals = true
 
-	if NoSimu != nil && *NoSimu {
+	if noSimu != nil && *noSimu {
 		filter.IncludeSimulated = new(bool)
 	}
 	/* nullify the empty entries to avoid bad filter */

cmd/crowdsec-cli/clihub/hub.go

@@ -177,11 +177,15 @@ func (cli *cliHub) upgrade(ctx context.Context, yes bool, dryRun bool, force boo
 
 	for _, itemType := range cwhub.ItemTypes {
 		for _, item := range hub.GetInstalledByType(itemType, true) {
-			plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force))
+			if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil {
+				return err
+			}
 		}
 	}
 
-	plan.AddCommand(hubops.NewDataRefreshCommand(force))
+	if err := plan.AddCommand(hubops.NewDataRefreshCommand(force)); err != nil {
+		return err
+	}
 
 	verbose := (cfg.Cscli.Output == "raw")
 
@@ -209,7 +213,11 @@ func (cli *cliHub) newUpgradeCmd() *cobra.Command {
 		Long: `
 Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available.
 `,
-		// TODO: Example
+		Example: `# Upgrade all the collections, scenarios etc. to the latest version in the downloaded index. Update data files too.
+cscli hub upgrade
+
+# Upgrade tainted items as well; force re-download of data files.
+cscli hub upgrade --force`,
 		Args:              cobra.NoArgs,
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, _ []string) error {

cmd/crowdsec-cli/clihub/items.go

@@ -63,7 +63,7 @@ func ListItems(out io.Writer, wantColor string, itemTypes []string, items map[st
 				continue
 			}
 
-			listHubItemTable(out, wantColor, "\n"+strings.ToUpper(itemType), items[itemType])
+			listHubItemTable(out, wantColor, strings.ToUpper(itemType), items[itemType])
 
 			nothingToDisplay = false
 		}
@@ -105,7 +105,7 @@ func ListItems(out io.Writer, wantColor string, itemTypes []string, items map[st
 			return fmt.Errorf("failed to parse: %w", err)
 		}
 
-		out.Write(x)
+		fmt.Fprint(out, string(x))
 	case "raw":
 		csvwriter := csv.NewWriter(out)
 

cmd/crowdsec-cli/clihub/utils_table.go

@@ -20,6 +20,6 @@ func listHubItemTable(out io.Writer, wantColor string, title string, items []*cw
 		t.AppendRow(table.Row{item.Name, status, item.State.LocalVersion, item.State.LocalPath})
 	}
 
-	io.WriteString(out, title+"\n")
-	io.WriteString(out, t.Render()+"\n")
+	t.SetTitle(title)
+	fmt.Fprintln(out, t.Render())
 }

cmd/crowdsec-cli/clihubtest/explain.go

@@ -14,9 +14,12 @@ func (cli *cliHubTest) explain(testName string, details bool, skipOk bool) error
 		return fmt.Errorf("can't load test: %+v", err)
 	}
 
+	cfg := cli.cfg()
+	patternDir := cfg.ConfigPaths.PatternDir
+
 	err = test.ParserAssert.LoadTest(test.ParserResultFile)
 	if err != nil {
-		if err = test.Run(); err != nil {
+		if err = test.Run(patternDir); err != nil {
 			return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
 		}
 
@@ -27,7 +30,7 @@ func (cli *cliHubTest) explain(testName string, details bool, skipOk bool) error
 
 	err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
 	if err != nil {
-		if err = test.Run(); err != nil {
+		if err = test.Run(patternDir); err != nil {
 			return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
 		}
 

cmd/crowdsec-cli/clihubtest/run.go

@@ -42,12 +42,14 @@ func (cli *cliHubTest) run(runAll bool, nucleiTargetHost string, appSecHost stri
 	// set timezone to avoid DST issues
 	os.Setenv("TZ", "UTC")
 
+	patternDir := cfg.ConfigPaths.PatternDir
+
 	for _, test := range hubPtr.Tests {
 		if cfg.Cscli.Output == "human" {
 			log.Infof("Running test '%s'", test.Name)
 		}
 
-		err := test.Run()
+		err := test.Run(patternDir)
 		if err != nil {
 			log.Errorf("running test '%s' failed: %+v", test.Name, err)
 		}