From a1d26bdc5be716a80a80861745c2aefbe518db3f Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 26 Dec 2024 15:21:52 +0100 Subject: [PATCH] cscli: improved hub management (#3352) --- cmd/crowdsec-cli/clihub/hub.go | 69 ++-- cmd/crowdsec-cli/clihub/items.go | 43 -- cmd/crowdsec-cli/clihub/utils_table.go | 60 --- cmd/crowdsec-cli/cliitem/inspect.go | 57 +++ cmd/crowdsec-cli/cliitem/item.go | 213 ++++++---- .../item_metrics.go => cliitem/metrics.go} | 8 +- cmd/crowdsec-cli/cliitem/metrics_table.go | 71 ++++ cmd/crowdsec-cli/clisetup/setup.go | 15 +- cmd/crowdsec-cli/config_restore.go | 13 +- cmd/crowdsec-cli/require/require.go | 2 +- docker/test/tests/test_hub.py | 2 +- docker/test/tests/test_hub_collections.py | 13 +- go.mod | 2 +- go.sum | 6 +- pkg/csconfig/cscli.go | 1 + pkg/cwhub/cwhub.go | 8 +- pkg/cwhub/cwhub_test.go | 8 +- pkg/cwhub/dataset.go | 72 ---- pkg/cwhub/doc.go | 17 +- pkg/cwhub/fetch.go | 108 +++++ pkg/cwhub/hub.go | 38 +- pkg/cwhub/hub_test.go | 8 +- pkg/cwhub/item.go | 227 +++++++---- pkg/cwhub/iteminstall.go | 73 ---- pkg/cwhub/iteminstall_test.go | 10 +- pkg/cwhub/itemlink.go | 78 ---- pkg/cwhub/itemremove.go | 138 ------- pkg/cwhub/itemupgrade.go | 254 ------------ pkg/cwhub/itemupgrade_test.go | 4 + pkg/cwhub/remote.go | 13 +- pkg/cwhub/sync.go | 12 +- pkg/emoji/emoji.go | 4 + pkg/hubops/colorize.go | 38 ++ pkg/hubops/datarefresh.go | 75 ++++ pkg/hubops/disable.go | 121 ++++++ pkg/hubops/download.go | 212 ++++++++++ pkg/hubops/enable.go | 113 ++++++ pkg/hubops/plan.go | 250 ++++++++++++ pkg/hubops/purge.go | 88 ++++ pkg/hubtest/hubtest_item.go | 7 +- pkg/setup/detect_test.go | 2 +- pkg/setup/install.go | 51 +-- test/bats/07_setup.bats | 31 +- test/bats/20_hub.bats | 40 +- test/bats/20_hub_collections.bats | 381 ----------------- test/bats/20_hub_collections_dep.bats | 26 +- test/bats/20_hub_items.bats | 70 ++-- test/bats/20_hub_parsers.bats | 383 ------------------ test/bats/20_hub_postoverflows.bats | 383 ------------------ test/bats/20_hub_scenarios.bats | 383 ------------------ test/bats/cscli-hubtype-inspect.bats | 93 +++++ test/bats/cscli-hubtype-install.bats | 269 ++++++++++++ test/bats/cscli-hubtype-list.bats | 130 ++++++ test/bats/cscli-hubtype-remove.bats | 245 +++++++++++ test/bats/cscli-hubtype-upgrade.bats | 253 ++++++++++++ test/bats/cscli-parsers.bats | 44 ++ test/bats/cscli-postoverflows.bats | 44 ++ test/bats/hub-index.bats | 357 ++++++++++++++++ test/bin/remove-all-hub-items | 2 +- test/lib/config/config-local | 2 +- test/lib/setup_file.sh | 24 +- 61 files changed, 3115 insertions(+), 2649 deletions(-) create mode 100644 cmd/crowdsec-cli/cliitem/inspect.go rename cmd/crowdsec-cli/{clihub/item_metrics.go => cliitem/metrics.go} (96%) create mode 100644 cmd/crowdsec-cli/cliitem/metrics_table.go delete mode 100644 pkg/cwhub/dataset.go create mode 100644 pkg/cwhub/fetch.go delete mode 100644 pkg/cwhub/iteminstall.go delete mode 100644 pkg/cwhub/itemlink.go delete mode 100644 pkg/cwhub/itemremove.go delete mode 100644 pkg/cwhub/itemupgrade.go create mode 100644 pkg/hubops/colorize.go create mode 100644 pkg/hubops/datarefresh.go create mode 100644 pkg/hubops/disable.go create mode 100644 pkg/hubops/download.go create mode 100644 pkg/hubops/enable.go create mode 100644 pkg/hubops/plan.go create mode 100644 pkg/hubops/purge.go delete mode 100644 test/bats/20_hub_collections.bats delete mode 100644 test/bats/20_hub_parsers.bats delete mode 100644 test/bats/20_hub_postoverflows.bats delete mode 100644 test/bats/20_hub_scenarios.bats create mode 100644 test/bats/cscli-hubtype-inspect.bats create mode 100644 test/bats/cscli-hubtype-install.bats create mode 100644 test/bats/cscli-hubtype-list.bats create mode 100644 test/bats/cscli-hubtype-remove.bats create mode 100644 test/bats/cscli-hubtype-upgrade.bats create mode 100644 test/bats/cscli-parsers.bats create mode 100644 test/bats/cscli-postoverflows.bats create mode 100644 test/bats/hub-index.bats diff --git a/cmd/crowdsec-cli/clihub/hub.go b/cmd/crowdsec-cli/clihub/hub.go index f189d6a2e13..49ccd761285 100644 --- a/cmd/crowdsec-cli/clihub/hub.go +++ b/cmd/crowdsec-cli/clihub/hub.go @@ -5,15 +5,18 @@ import ( "encoding/json" "fmt" "io" + "os" "github.com/fatih/color" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v3" + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/reload" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/hubops" ) type configGetter = func() *csconfig.Config @@ -55,11 +58,11 @@ func (cli *cliHub) List(out io.Writer, hub *cwhub.Hub, all bool) error { cfg := cli.cfg() for _, v := range hub.Warnings { - log.Info(v) + fmt.Fprintln(os.Stderr, v) } for _, line := range hub.ItemStats() { - log.Info(line) + fmt.Fprintln(os.Stderr, line) } items := make(map[string][]*cwhub.Item) @@ -100,7 +103,7 @@ func (cli *cliHub) newListCmd() *cobra.Command { } flags := cmd.Flags() - flags.BoolVarP(&all, "all", "a", false, "List disabled items as well") + flags.BoolVarP(&all, "all", "a", false, "List all available items, including those not installed") return cmd } @@ -108,7 +111,6 @@ func (cli *cliHub) newListCmd() *cobra.Command { func (cli *cliHub) update(ctx context.Context, withContent bool) error { local := cli.cfg().Hub remote := require.RemoteHub(ctx, cli.cfg()) - remote.EmbedItemContent = withContent // don't use require.Hub because if there is no index file, it would fail hub, err := cwhub.NewHub(local, remote, log.StandardLogger()) @@ -116,7 +118,7 @@ func (cli *cliHub) update(ctx context.Context, withContent bool) error { return err } - if err := hub.Update(ctx); err != nil { + if err := hub.Update(ctx, withContent); err != nil { return fmt.Errorf("failed to update hub: %w", err) } @@ -125,7 +127,7 @@ func (cli *cliHub) update(ctx context.Context, withContent bool) error { } for _, v := range hub.Warnings { - log.Info(v) + fmt.Fprintln(os.Stderr, v) } return nil @@ -140,10 +142,18 @@ func (cli *cliHub) newUpdateCmd() *cobra.Command { Long: ` Fetches the .index.json file from the hub, containing the list of available configs. `, + Example: `# Download the last version of the index file. +cscli hub update + +# Download a 4x bigger version with all item contents (effectively pre-caching item downloads, but not data files). +cscli hub update --with-content`, Args: cobra.NoArgs, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { - return cli.update(cmd.Context(), withContent) + if cmd.Flags().Changed("with-content") { + return cli.update(cmd.Context(), withContent) + } + return cli.update(cmd.Context(), cli.cfg().Cscli.HubWithContent) }, } @@ -153,36 +163,43 @@ Fetches the .index.json file from the hub, containing the list of available conf return cmd } -func (cli *cliHub) upgrade(ctx context.Context, force bool) error { - hub, err := require.Hub(cli.cfg(), require.RemoteHub(ctx, cli.cfg()), log.StandardLogger()) +func (cli *cliHub) upgrade(ctx context.Context, yes bool, dryRun bool, force bool) error { + cfg := cli.cfg() + + hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger()) if err != nil { return err } + plan := hubops.NewActionPlan(hub) + for _, itemType := range cwhub.ItemTypes { - updated := 0 + for _, item := range hub.GetInstalledByType(itemType, true) { + plan.AddCommand(hubops.NewDownloadCommand(item, force)) + } + } - log.Infof("Upgrading %s", itemType) + plan.AddCommand(hubops.NewDataRefreshCommand(force)) - for _, item := range hub.GetInstalledByType(itemType, true) { - didUpdate, err := item.Upgrade(ctx, force) - if err != nil { - return err - } + verbose := (cfg.Cscli.Output == "raw") - if didUpdate { - updated++ - } - } + if err := plan.Execute(ctx, yes, dryRun, verbose); err != nil { + return err + } - log.Infof("Upgraded %d %s", updated, itemType) + if plan.ReloadNeeded { + fmt.Println("\n" + reload.Message) } return nil } func (cli *cliHub) newUpgradeCmd() *cobra.Command { - var force bool + var ( + yes bool + dryRun bool + force bool + ) cmd := &cobra.Command{ Use: "upgrade", @@ -190,15 +207,19 @@ func (cli *cliHub) newUpgradeCmd() *cobra.Command { Long: ` Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available. `, + // TODO: Example Args: cobra.NoArgs, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { - return cli.upgrade(cmd.Context(), force) + return cli.upgrade(cmd.Context(), yes, dryRun, force) }, } flags := cmd.Flags() - flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated files") + flags.BoolVar(&yes, "yes", false, "Confirm execution without prompt") + flags.BoolVar(&dryRun, "dry-run", false, "Don't install or remove anything; print the execution plan") + flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated items; always update data files") + cmd.MarkFlagsMutuallyExclusive("yes", "dry-run") return cmd } diff --git a/cmd/crowdsec-cli/clihub/items.go b/cmd/crowdsec-cli/clihub/items.go index ef3127033ac..f63dc4bedd7 100644 --- a/cmd/crowdsec-cli/clihub/items.go +++ b/cmd/crowdsec-cli/clihub/items.go @@ -5,13 +5,9 @@ import ( "encoding/json" "fmt" "io" - "os" - "path/filepath" "slices" "strings" - "gopkg.in/yaml.v3" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -145,42 +141,3 @@ func ListItems(out io.Writer, wantColor string, itemTypes []string, items map[st return nil } - -func InspectItem(item *cwhub.Item, wantMetrics bool, output string, prometheusURL string, wantColor string) error { - switch output { - case "human", "raw": - enc := yaml.NewEncoder(os.Stdout) - enc.SetIndent(2) - - if err := enc.Encode(item); err != nil { - return fmt.Errorf("unable to encode item: %w", err) - } - case "json": - b, err := json.MarshalIndent(*item, "", " ") - if err != nil { - return fmt.Errorf("unable to serialize item: %w", err) - } - - fmt.Print(string(b)) - } - - if output != "human" { - return nil - } - - if item.State.Tainted { - fmt.Println() - fmt.Printf(`This item is tainted. Use "%s %s inspect --diff %s" to see why.`, filepath.Base(os.Args[0]), item.Type, item.Name) - fmt.Println() - } - - if wantMetrics { - fmt.Printf("\nCurrent metrics: \n") - - if err := showMetrics(prometheusURL, item, wantColor); err != nil { - return err - } - } - - return nil -} diff --git a/cmd/crowdsec-cli/clihub/utils_table.go b/cmd/crowdsec-cli/clihub/utils_table.go index 98f14341b10..4693161005b 100644 --- a/cmd/crowdsec-cli/clihub/utils_table.go +++ b/cmd/crowdsec-cli/clihub/utils_table.go @@ -3,7 +3,6 @@ package clihub import ( "fmt" "io" - "strconv" "github.com/jedib0t/go-pretty/v6/table" @@ -24,62 +23,3 @@ func listHubItemTable(out io.Writer, wantColor string, title string, items []*cw io.WriteString(out, title+"\n") io.WriteString(out, t.Render()+"\n") } - -func appsecMetricsTable(out io.Writer, wantColor string, itemName string, metrics map[string]int) { - t := cstable.NewLight(out, wantColor).Writer - t.AppendHeader(table.Row{"Inband Hits", "Outband Hits"}) - - t.AppendRow(table.Row{ - strconv.Itoa(metrics["inband_hits"]), - strconv.Itoa(metrics["outband_hits"]), - }) - - io.WriteString(out, fmt.Sprintf("\n - (AppSec Rule) %s:\n", itemName)) - io.WriteString(out, t.Render()+"\n") -} - -func scenarioMetricsTable(out io.Writer, wantColor string, itemName string, metrics map[string]int) { - if metrics["instantiation"] == 0 { - return - } - - t := cstable.New(out, wantColor).Writer - t.AppendHeader(table.Row{"Current Count", "Overflows", "Instantiated", "Poured", "Expired"}) - - t.AppendRow(table.Row{ - strconv.Itoa(metrics["curr_count"]), - strconv.Itoa(metrics["overflow"]), - strconv.Itoa(metrics["instantiation"]), - strconv.Itoa(metrics["pour"]), - strconv.Itoa(metrics["underflow"]), - }) - - io.WriteString(out, fmt.Sprintf("\n - (Scenario) %s:\n", itemName)) - io.WriteString(out, t.Render()+"\n") -} - -func parserMetricsTable(out io.Writer, wantColor string, itemName string, metrics map[string]map[string]int) { - t := cstable.New(out, wantColor).Writer - t.AppendHeader(table.Row{"Parsers", "Hits", "Parsed", "Unparsed"}) - - // don't show table if no hits - showTable := false - - for source, stats := range metrics { - if stats["hits"] > 0 { - t.AppendRow(table.Row{ - source, - strconv.Itoa(stats["hits"]), - strconv.Itoa(stats["parsed"]), - strconv.Itoa(stats["unparsed"]), - }) - - showTable = true - } - } - - if showTable { - io.WriteString(out, fmt.Sprintf("\n - (Parser) %s:\n", itemName)) - io.WriteString(out, t.Render()+"\n") - } -} diff --git a/cmd/crowdsec-cli/cliitem/inspect.go b/cmd/crowdsec-cli/cliitem/inspect.go new file mode 100644 index 00000000000..596674aa788 --- /dev/null +++ b/cmd/crowdsec-cli/cliitem/inspect.go @@ -0,0 +1,57 @@ +package cliitem + +import ( + "fmt" + "encoding/json" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func inspectItem(hub *cwhub.Hub, item *cwhub.Item, wantMetrics bool, output string, prometheusURL string, wantColor string) error { + // This is dirty... + // We want to show current dependencies (from content), not latest (from index). + // The item is modifed but after this function the whole hub should be thrown away. + // A cleaner way would be to copy the struct first. + item.Dependencies = item.CurrentDependencies() + + switch output { + case "human", "raw": + enc := yaml.NewEncoder(os.Stdout) + enc.SetIndent(2) + + if err := enc.Encode(item); err != nil { + return fmt.Errorf("unable to encode item: %w", err) + } + case "json": + b, err := json.MarshalIndent(*item, "", " ") + if err != nil { + return fmt.Errorf("unable to serialize item: %w", err) + } + + fmt.Print(string(b)) + } + + if output != "human" { + return nil + } + + if item.State.Tainted { + fmt.Println() + fmt.Printf(`This item is tainted. Use "%s %s inspect --diff %s" to see why.`, filepath.Base(os.Args[0]), item.Type, item.Name) + fmt.Println() + } + + if wantMetrics { + fmt.Printf("\nCurrent metrics: \n") + + if err := showMetrics(prometheusURL, hub, item, wantColor); err != nil { + return err + } + } + + return nil +} diff --git a/cmd/crowdsec-cli/cliitem/item.go b/cmd/crowdsec-cli/cliitem/item.go index 28828eb9c95..05e52d18dd3 100644 --- a/cmd/crowdsec-cli/cliitem/item.go +++ b/cmd/crowdsec-cli/cliitem/item.go @@ -20,6 +20,7 @@ import ( "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/hubops" ) type cliHelp struct { @@ -67,7 +68,7 @@ func (cli cliItem) NewCommand() *cobra.Command { return cmd } -func (cli cliItem) install(ctx context.Context, args []string, downloadOnly bool, force bool, ignoreError bool) error { +func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun bool, downloadOnly bool, force bool, ignoreError bool) error { cfg := cli.cfg() hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger()) @@ -75,6 +76,8 @@ func (cli cliItem) install(ctx context.Context, args []string, downloadOnly bool return err } + plan := hubops.NewActionPlan(hub) + for _, name := range args { item := hub.GetItem(cli.name, name) if item == nil { @@ -88,22 +91,38 @@ func (cli cliItem) install(ctx context.Context, args []string, downloadOnly bool continue } - if err := item.Install(ctx, force, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", item.Name, err) + if err = plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil { + return err + } + + if !downloadOnly { + if err = plan.AddCommand(hubops.NewEnableCommand(item, force)); err != nil { + return err } + } + } + + verbose := (cfg.Cscli.Output == "raw") - log.Errorf("Error while installing '%s': %s", item.Name, err) + if err := plan.Execute(ctx, yes, dryRun, verbose); err != nil { + if !ignoreError { + return err } + + log.Error(err) } - log.Info(reload.Message) + if plan.ReloadNeeded { + fmt.Println("\n" + reload.Message) + } return nil } func (cli cliItem) newInstallCmd() *cobra.Command { var ( + yes bool + dryRun bool downloadOnly bool force bool ignoreError bool @@ -120,20 +139,23 @@ func (cli cliItem) newInstallCmd() *cobra.Command { return compAllItems(cli.name, args, toComplete, cli.cfg) }, RunE: func(cmd *cobra.Command, args []string) error { - return cli.install(cmd.Context(), args, downloadOnly, force, ignoreError) + return cli.install(cmd.Context(), args, yes, dryRun, downloadOnly, force, ignoreError) }, } flags := cmd.Flags() + flags.BoolVarP(&yes, "yes", "y", false, "Confirm execution without prompt") + flags.BoolVar(&dryRun, "dry-run", false, "Don't install or remove anything; print the execution plan") flags.BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") flags.BoolVar(&force, "force", false, "Force install: overwrite tainted and outdated files") flags.BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple "+cli.name) + cmd.MarkFlagsMutuallyExclusive("yes", "dry-run") return cmd } // return the names of the installed parents of an item, used to check if we can remove it -func istalledParentNames(item *cwhub.Item) []string { +func installedParentNames(item *cwhub.Item) []string { ret := make([]string, 0) for _, parent := range item.Ancestors() { @@ -145,11 +167,8 @@ func istalledParentNames(item *cwhub.Item) []string { return ret } -func (cli cliItem) remove(args []string, purge bool, force bool, all bool) error { - hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger()) - if err != nil { - return err - } +func (cli cliItem) removePlan(hub *cwhub.Hub, args []string, purge bool, force bool, all bool) (*hubops.ActionPlan, error) { + plan := hubops.NewActionPlan(hub) if all { itemGetter := hub.GetInstalledByType @@ -157,43 +176,31 @@ func (cli cliItem) remove(args []string, purge bool, force bool, all bool) error itemGetter = hub.GetItemsByType } - removed := 0 - for _, item := range itemGetter(cli.name, true) { - didRemove, err := item.Remove(purge, force) - if err != nil { - return err + if err := plan.AddCommand(hubops.NewDisableCommand(item, force)); err != nil { + return nil, err } - - if didRemove { - log.Infof("Removed %s", item.Name) - - removed++ + if purge { + if err := plan.AddCommand(hubops.NewPurgeCommand(item, force)); err != nil { + return nil, err + } } } - log.Infof("Removed %d %s", removed, cli.name) - - if removed > 0 { - log.Info(reload.Message) - } - - return nil + return plan, nil } if len(args) == 0 { - return fmt.Errorf("specify at least one %s to remove or '--all'", cli.singular) + return nil, fmt.Errorf("specify at least one %s to remove or '--all'", cli.singular) } - removed := 0 - for _, itemName := range args { item := hub.GetItem(cli.name, itemName) if item == nil { - return fmt.Errorf("can't find '%s' in %s", itemName, cli.name) + return nil, fmt.Errorf("can't find '%s' in %s", itemName, cli.name) } - parents := istalledParentNames(item) + parents := installedParentNames(item) if !force && len(parents) > 0 { log.Warningf("%s belongs to collections: %s", item.Name, parents) @@ -202,22 +209,43 @@ func (cli cliItem) remove(args []string, purge bool, force bool, all bool) error continue } - didRemove, err := item.Remove(purge, force) - if err != nil { - return err - } + if err := plan.AddCommand(hubops.NewDisableCommand(item, force)); err != nil { + return nil, err - if didRemove { - log.Infof("Removed %s", item.Name) + } + if purge { + if err := plan.AddCommand(hubops.NewPurgeCommand(item, force)); err != nil { + return nil, err - removed++ + } } } - log.Infof("Removed %d %s", removed, cli.name) + return plan, nil +} - if removed > 0 { - log.Info(reload.Message) + +func (cli cliItem) remove(ctx context.Context, args []string, yes bool, dryRun bool, purge bool, force bool, all bool) error { + cfg := cli.cfg() + + hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger()) + if err != nil { + return err + } + + plan, err := cli.removePlan(hub, args, purge, force, all) + if err != nil { + return err + } + + verbose := (cfg.Cscli.Output == "raw") + + if err := plan.Execute(ctx, yes, dryRun, verbose); err != nil { + return err + } + + if plan.ReloadNeeded { + fmt.Println("\n" + reload.Message) } return nil @@ -225,9 +253,11 @@ func (cli cliItem) remove(args []string, purge bool, force bool, all bool) error func (cli cliItem) newRemoveCmd() *cobra.Command { var ( - purge bool - force bool - all bool + yes bool + dryRun bool + purge bool + force bool + all bool ) cmd := &cobra.Command{ @@ -240,76 +270,78 @@ func (cli cliItem) newRemoveCmd() *cobra.Command { ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cli.name, args, toComplete, cli.cfg) }, - RunE: func(_ *cobra.Command, args []string) error { - return cli.remove(args, purge, force, all) + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) > 0 && all { + return errors.New("can't specify items and '--all' at the same time") + } + + return cli.remove(cmd.Context(), args, yes, dryRun, purge, force, all) }, } flags := cmd.Flags() + flags.BoolVarP(&yes, "yes", "y", false, "Confirm execution without prompt") + flags.BoolVar(&dryRun, "dry-run", false, "Don't install or remove anything; print the execution plan") flags.BoolVar(&purge, "purge", false, "Delete source file too") flags.BoolVar(&force, "force", false, "Force remove: remove tainted and outdated files") flags.BoolVar(&all, "all", false, "Remove all the "+cli.name) + cmd.MarkFlagsMutuallyExclusive("yes", "dry-run") return cmd } -func (cli cliItem) upgrade(ctx context.Context, args []string, force bool, all bool) error { - cfg := cli.cfg() - - hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger()) - if err != nil { - return err - } +func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bool) (*hubops.ActionPlan, error) { + plan := hubops.NewActionPlan(hub) if all { - updated := 0 - for _, item := range hub.GetInstalledByType(cli.name, true) { - didUpdate, err := item.Upgrade(ctx, force) - if err != nil { - return err + if err := plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil { + return nil, err } - - if didUpdate { - updated++ - } - } - - log.Infof("Updated %d %s", updated, cli.name) - - if updated > 0 { - log.Info(reload.Message) } - return nil + return plan, nil } if len(args) == 0 { - return fmt.Errorf("specify at least one %s to upgrade or '--all'", cli.singular) + return nil, fmt.Errorf("specify at least one %s to upgrade or '--all'", cli.singular) } - updated := 0 - for _, itemName := range args { item := hub.GetItem(cli.name, itemName) if item == nil { - return fmt.Errorf("can't find '%s' in %s", itemName, cli.name) + return nil, fmt.Errorf("can't find '%s' in %s", itemName, cli.name) } - didUpdate, err := item.Upgrade(ctx, force) - if err != nil { - return err + if err := plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil { + return nil, err } + } - if didUpdate { - log.Infof("Updated %s", item.Name) + return plan, nil +} - updated++ - } +func (cli cliItem) upgrade(ctx context.Context, args []string, yes bool, dryRun bool, force bool, all bool) error { + cfg := cli.cfg() + + hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger()) + if err != nil { + return err + } + + plan, err := cli.upgradePlan(hub, args, force, all) + if err != nil { + return err + } + + verbose := (cfg.Cscli.Output == "raw") + + if err := plan.Execute(ctx, yes, dryRun, verbose); err != nil { + return err } - if updated > 0 { - log.Info(reload.Message) + if plan.ReloadNeeded { + fmt.Println("\n" + reload.Message) } return nil @@ -317,6 +349,8 @@ func (cli cliItem) upgrade(ctx context.Context, args []string, force bool, all b func (cli cliItem) newUpgradeCmd() *cobra.Command { var ( + yes bool + dryRun bool all bool force bool ) @@ -331,13 +365,16 @@ func (cli cliItem) newUpgradeCmd() *cobra.Command { return compInstalledItems(cli.name, args, toComplete, cli.cfg) }, RunE: func(cmd *cobra.Command, args []string) error { - return cli.upgrade(cmd.Context(), args, force, all) + return cli.upgrade(cmd.Context(), args, yes, dryRun, force, all) }, } flags := cmd.Flags() + flags.BoolVarP(&yes, "yes", "y", false, "Confirm execution without prompt") + flags.BoolVar(&dryRun, "dry-run", false, "Don't install or remove anything; print the execution plan") flags.BoolVarP(&all, "all", "a", false, "Upgrade all the "+cli.name) flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated files") + cmd.MarkFlagsMutuallyExclusive("yes", "dry-run") return cmd } @@ -376,7 +413,7 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff continue } - if err = clihub.InspectItem(item, !noMetrics, cfg.Cscli.Output, cfg.Cscli.PrometheusUrl, cfg.Cscli.Color); err != nil { + if err = inspectItem(hub, item, !noMetrics, cfg.Cscli.Output, cfg.Cscli.PrometheusUrl, cfg.Cscli.Color); err != nil { return err } diff --git a/cmd/crowdsec-cli/clihub/item_metrics.go b/cmd/crowdsec-cli/cliitem/metrics.go similarity index 96% rename from cmd/crowdsec-cli/clihub/item_metrics.go rename to cmd/crowdsec-cli/cliitem/metrics.go index ac9c18640fa..4999ea38078 100644 --- a/cmd/crowdsec-cli/clihub/item_metrics.go +++ b/cmd/crowdsec-cli/cliitem/metrics.go @@ -1,4 +1,4 @@ -package clihub +package cliitem import ( "fmt" @@ -17,7 +17,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func showMetrics(prometheusURL string, hubItem *cwhub.Item, wantColor string) error { +func showMetrics(prometheusURL string, hub *cwhub.Hub, hubItem *cwhub.Item, wantColor string) error { switch hubItem.Type { case cwhub.PARSERS: metrics, err := getParserMetric(prometheusURL, hubItem.Name) @@ -32,8 +32,8 @@ func showMetrics(prometheusURL string, hubItem *cwhub.Item, wantColor string) er } scenarioMetricsTable(color.Output, wantColor, hubItem.Name, metrics) case cwhub.COLLECTIONS: - for _, sub := range hubItem.SubItems() { - if err := showMetrics(prometheusURL, sub, wantColor); err != nil { + for sub := range hubItem.CurrentDependencies().SubItems(hub) { + if err := showMetrics(prometheusURL, hub, sub, wantColor); err != nil { return err } } diff --git a/cmd/crowdsec-cli/cliitem/metrics_table.go b/cmd/crowdsec-cli/cliitem/metrics_table.go new file mode 100644 index 00000000000..378394bad85 --- /dev/null +++ b/cmd/crowdsec-cli/cliitem/metrics_table.go @@ -0,0 +1,71 @@ +package cliitem + +import ( + "fmt" + "io" + "strconv" + + "github.com/jedib0t/go-pretty/v6/table" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/cstable" +) + + +func appsecMetricsTable(out io.Writer, wantColor string, itemName string, metrics map[string]int) { + t := cstable.NewLight(out, wantColor).Writer + t.AppendHeader(table.Row{"Inband Hits", "Outband Hits"}) + + t.AppendRow(table.Row{ + strconv.Itoa(metrics["inband_hits"]), + strconv.Itoa(metrics["outband_hits"]), + }) + + io.WriteString(out, fmt.Sprintf("\n - (AppSec Rule) %s:\n", itemName)) + io.WriteString(out, t.Render()+"\n") +} + +func scenarioMetricsTable(out io.Writer, wantColor string, itemName string, metrics map[string]int) { + if metrics["instantiation"] == 0 { + return + } + + t := cstable.New(out, wantColor).Writer + t.AppendHeader(table.Row{"Current Count", "Overflows", "Instantiated", "Poured", "Expired"}) + + t.AppendRow(table.Row{ + strconv.Itoa(metrics["curr_count"]), + strconv.Itoa(metrics["overflow"]), + strconv.Itoa(metrics["instantiation"]), + strconv.Itoa(metrics["pour"]), + strconv.Itoa(metrics["underflow"]), + }) + + io.WriteString(out, fmt.Sprintf("\n - (Scenario) %s:\n", itemName)) + io.WriteString(out, t.Render()+"\n") +} + +func parserMetricsTable(out io.Writer, wantColor string, itemName string, metrics map[string]map[string]int) { + t := cstable.New(out, wantColor).Writer + t.AppendHeader(table.Row{"Parsers", "Hits", "Parsed", "Unparsed"}) + + // don't show table if no hits + showTable := false + + for source, stats := range metrics { + if stats["hits"] > 0 { + t.AppendRow(table.Row{ + source, + strconv.Itoa(stats["hits"]), + strconv.Itoa(stats["parsed"]), + strconv.Itoa(stats["unparsed"]), + }) + + showTable = true + } + } + + if showTable { + io.WriteString(out, fmt.Sprintf("\n - (Parser) %s:\n", itemName)) + io.WriteString(out, t.Render()+"\n") + } +} diff --git a/cmd/crowdsec-cli/clisetup/setup.go b/cmd/crowdsec-cli/clisetup/setup.go index 269cdfb78e9..4cb423e484c 100644 --- a/cmd/crowdsec-cli/clisetup/setup.go +++ b/cmd/crowdsec-cli/clisetup/setup.go @@ -94,7 +94,10 @@ func (cli *cliSetup) newDetectCmd() *cobra.Command { } func (cli *cliSetup) newInstallHubCmd() *cobra.Command { - var dryRun bool + var ( + yes bool + dryRun bool + ) cmd := &cobra.Command{ Use: "install-hub [setup_file] [flags]", @@ -102,12 +105,14 @@ func (cli *cliSetup) newInstallHubCmd() *cobra.Command { Args: cobra.ExactArgs(1), DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return cli.install(cmd.Context(), dryRun, args[0]) + return cli.install(cmd.Context(), yes, dryRun, args[0]) }, } flags := cmd.Flags() + flags.BoolVarP(&yes, "yes", "y", false, "confirm execution without prompt") flags.BoolVar(&dryRun, "dry-run", false, "don't install anything; print out what would have been") + cmd.MarkFlagsMutuallyExclusive("yes", "dry-run") return cmd } @@ -276,7 +281,7 @@ func (cli *cliSetup) dataSources(fromFile string, toDir string) error { return nil } -func (cli *cliSetup) install(ctx context.Context, dryRun bool, fromFile string) error { +func (cli *cliSetup) install(ctx context.Context, yes bool, dryRun bool, fromFile string) error { input, err := os.ReadFile(fromFile) if err != nil { return fmt.Errorf("while reading file %s: %w", fromFile, err) @@ -289,7 +294,9 @@ func (cli *cliSetup) install(ctx context.Context, dryRun bool, fromFile string) return err } - return setup.InstallHubItems(ctx, hub, input, dryRun) + verbose := (cfg.Cscli.Output == "raw") + + return setup.InstallHubItems(ctx, hub, input, yes, dryRun, verbose) } func (cli *cliSetup) validate(fromFile string) error { diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index c32328485ec..8884fa448d2 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -12,6 +12,7 @@ import ( "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/hubops" ) func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error { @@ -50,7 +51,17 @@ func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error { continue } - if err = item.Install(ctx, false, false); err != nil { + plan := hubops.NewActionPlan(hub) + + if err = plan.AddCommand(hubops.NewDownloadCommand(item, false)); err != nil { + return err + } + + if err = plan.AddCommand(hubops.NewEnableCommand(item, false)); err != nil { + return err + } + + if err = plan.Execute(ctx, true, false, false); err != nil { log.Errorf("Error while installing %s : %s", toinstall, err) } } diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index 191eee55bc5..7b3410021c1 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -116,7 +116,7 @@ func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) } if err := hub.Load(); err != nil { - return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err) + return nil, fmt.Errorf("failed to read hub index: %w. Run 'sudo cscli hub update' to download the index again", err) } return hub, nil diff --git a/docker/test/tests/test_hub.py b/docker/test/tests/test_hub.py index 2365e3a9cef..e70555ea855 100644 --- a/docker/test/tests/test_hub.py +++ b/docker/test/tests/test_hub.py @@ -17,7 +17,7 @@ def test_preinstalled_hub(crowdsec, flavor): with crowdsec(flavor=flavor) as cs: cs.wait_for_log("*Starting processing data*") cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK) - res = cs.cont.exec_run('cscli hub list -o json') + res = cs.cont.exec_run('cscli hub list -o json', stderr=False) assert res.exit_code == 0 j = json.loads(res.output) collections = {c['name']: c for c in j['collections']} diff --git a/docker/test/tests/test_hub_collections.py b/docker/test/tests/test_hub_collections.py index 962f8ff8df4..0d1b3ee5e94 100644 --- a/docker/test/tests/test_hub_collections.py +++ b/docker/test/tests/test_hub_collections.py @@ -28,10 +28,8 @@ def test_install_two_collections(crowdsec, flavor): assert items[it1]['status'] == 'enabled' assert items[it2]['status'] == 'enabled' cs.wait_for_log([ - # f'*collections install "{it1}"*' - # f'*collections install "{it2}"*' - f'*Enabled collections: {it1}*', - f'*Enabled collections: {it2}*', + f'*enabling collections:{it1}*', + f'*enabling collections:{it2}*', ]) @@ -50,8 +48,7 @@ def test_disable_collection(crowdsec, flavor): items = {c['name'] for c in j['collections']} assert it not in items cs.wait_for_log([ - # f'*collections remove "{it}*", - f'*Removed symlink [[]{it}[]]*', + f'*disabling collections:{it}*', ]) @@ -72,7 +69,7 @@ def test_install_and_disable_collection(crowdsec, flavor): assert it not in items logs = cs.log_lines() # check that there was no attempt to install - assert not any(f'Enabled collections: {it}' in line for line in logs) + assert not any(f'enabling collections:{it}' in line for line in logs) # already done in bats, prividing here as example of a somewhat complex test @@ -91,7 +88,7 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor): # implicit check for tainted=False assert items[coll]['status'] == 'enabled' cs.wait_for_log([ - f'*Enabled collections: {coll}*', + f'*enabling collections:{coll}*', ]) scenario = 'crowdsecurity/http-crawl-non_statics' diff --git a/go.mod b/go.mod index 43f0ed4e6f2..e437bbd688a 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/creack/pty v1.1.21 // indirect github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 - github.com/crowdsecurity/go-cs-lib v0.0.15 + github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f github.com/crowdsecurity/grokky v0.2.2 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index 80f8a079bae..d092956d0a8 100644 --- a/go.sum +++ b/go.sum @@ -107,8 +107,10 @@ github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 h1:hyrYw3h github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607/go.mod h1:br36fEqurGYZQGit+iDYsIzW0FF6VufMbDzyyLxEuPA= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.15 h1:zNWqOPVLHgKUstlr6clom9d66S0eIIW66jQG3Y7FEvo= -github.com/crowdsecurity/go-cs-lib v0.0.15/go.mod h1:ePyQyJBxp1W/1bq4YpVAilnLSz7HkzmtI7TRhX187EU= +github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241203101722-e557f9809413 h1:VIedap4s3mXM4+tM2NMm7R3E/kn79ayLZaLHDqPYVCc= +github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241203101722-e557f9809413/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g= +github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f h1:Pd+O4UK78uQtTqbvYX+nHvqZ7TffD51uC4q0RE/podk= +github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g= github.com/crowdsecurity/grokky v0.2.2 h1:yALsI9zqpDArYzmSSxfBq2dhYuGUTKMJq8KOEIAsuo4= github.com/crowdsecurity/grokky v0.2.2/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= diff --git a/pkg/csconfig/cscli.go b/pkg/csconfig/cscli.go index 9393156c0ed..ad119dc9e13 100644 --- a/pkg/csconfig/cscli.go +++ b/pkg/csconfig/cscli.go @@ -10,6 +10,7 @@ type CscliCfg struct { Color string `yaml:"color,omitempty"` HubBranch string `yaml:"hub_branch"` HubURLTemplate string `yaml:"__hub_url_template__,omitempty"` + HubWithContent bool `yaml:"hub_with_content,omitempty"` SimulationConfig *SimulationConfig `yaml:"-"` DbConfig *DatabaseCfg `yaml:"-"` diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 683f1853b43..b41d1d16312 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -20,14 +20,14 @@ func (t *hubTransport) RoundTrip(req *http.Request) (*http.Response, error) { return t.RoundTripper.RoundTrip(req) } -// hubClient is the HTTP client used to communicate with the CrowdSec Hub. -var hubClient = &http.Client{ +// HubClient is the HTTP client used to communicate with the CrowdSec Hub. +var HubClient = &http.Client{ Timeout: 120 * time.Second, Transport: &hubTransport{http.DefaultTransport}, } -// safePath returns a joined path and ensures that it does not escape the base directory. -func safePath(dir, filePath string) (string, error) { +// SafePath returns a joined path and ensures that it does not escape the base directory. +func SafePath(dir, filePath string) (string, error) { absBaseDir, err := filepath.Abs(filepath.Clean(dir)) if err != nil { return "", err diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 17e7a0dc723..1b5dee34dd3 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -68,7 +68,7 @@ func testHub(t *testing.T, update bool) *Hub { if update { ctx := context.Background() - err := hub.Update(ctx) + err := hub.Update(ctx, false) require.NoError(t, err) } @@ -83,14 +83,14 @@ func envSetup(t *testing.T) *Hub { setResponseByPath() log.SetLevel(log.DebugLevel) - defaultTransport := hubClient.Transport + defaultTransport := HubClient.Transport t.Cleanup(func() { - hubClient.Transport = defaultTransport + HubClient.Transport = defaultTransport }) // Mock the http client - hubClient.Transport = newMockTransport() + HubClient.Transport = newMockTransport() hub := testHub(t, true) diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go deleted file mode 100644 index 90bc9e057f9..00000000000 --- a/pkg/cwhub/dataset.go +++ /dev/null @@ -1,72 +0,0 @@ -package cwhub - -import ( - "context" - "errors" - "fmt" - "io" - "time" - - "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" - - "github.com/crowdsecurity/go-cs-lib/downloader" - - "github.com/crowdsecurity/crowdsec/pkg/types" -) - -// The DataSet is a list of data sources required by an item (built from the data: section in the yaml). -type DataSet struct { - Data []types.DataSource `yaml:"data,omitempty"` -} - -// downloadDataSet downloads all the data files for an item. -func downloadDataSet(ctx context.Context, dataFolder string, force bool, reader io.Reader, logger *logrus.Logger) error { - dec := yaml.NewDecoder(reader) - - for { - data := &DataSet{} - - if err := dec.Decode(data); err != nil { - if errors.Is(err, io.EOF) { - break - } - - return fmt.Errorf("while reading file: %w", err) - } - - for _, dataS := range data.Data { - destPath, err := safePath(dataFolder, dataS.DestPath) - if err != nil { - return err - } - - d := downloader. - New(). - WithHTTPClient(hubClient). - ToFile(destPath). - CompareContent(). - WithLogger(logrus.WithField("url", dataS.SourceURL)) - - if !force { - d = d.WithLastModified(). - WithShelfLife(7 * 24 * time.Hour) - } - - downloaded, err := d.Download(ctx, dataS.SourceURL) - if err != nil { - return fmt.Errorf("while getting data: %w", err) - } - - if downloaded { - logger.Infof("Downloaded %s", destPath) - // a check on stdout is used while scripting to know if the hub has been upgraded - // and a configuration reload is required - // TODO: use a better way to communicate this - fmt.Printf("updated %s\n", destPath) - } - } - } - - return nil -} diff --git a/pkg/cwhub/doc.go b/pkg/cwhub/doc.go index f86b95c6454..a1ee9d37ee7 100644 --- a/pkg/cwhub/doc.go +++ b/pkg/cwhub/doc.go @@ -1,4 +1,5 @@ -// Package cwhub is responsible for installing and upgrading the local hub files for CrowdSec. +// Package cwhub is responsible for providing the state of the local hub to the security engine and cscli command. +// Installation, upgrade and removal of items or data files has been moved to pkg/hubops. // // # Definitions // @@ -84,20 +85,6 @@ // return fmt.Errorf("collection not found") // } // -// You can also install items if they have already been downloaded: -// -// // install a parser -// force := false -// downloadOnly := false -// err := parser.Install(force, downloadOnly) -// if err != nil { -// return fmt.Errorf("unable to install parser: %w", err) -// } -// -// As soon as you try to install an item that is not downloaded or is not up-to-date (meaning its computed hash -// does not correspond to the latest version available in the index), a download will be attempted and you'll -// get the error "remote hub configuration is not provided". -// // To provide the remote hub configuration, use the second parameter of NewHub(): // // remoteHub := cwhub.RemoteHubCfg{ diff --git a/pkg/cwhub/fetch.go b/pkg/cwhub/fetch.go new file mode 100644 index 00000000000..92198e63ef1 --- /dev/null +++ b/pkg/cwhub/fetch.go @@ -0,0 +1,108 @@ +package cwhub + +// Install, upgrade and remove items from the hub to the local configuration + +import ( + "context" + "crypto" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/go-cs-lib/downloader" + +) + + +// writeEmbeddedContentTo writes the embedded content to the specified path and checks the hash. +// If the content is base64 encoded, it will be decoded before writing. Check for item.Content +// before calling this method. +func (i *Item) writeEmbeddedContentTo(destPath, wantHash string) error { + if i.Content == "" { + return fmt.Errorf("no embedded content for %s", i.Name) + } + + content, err := base64.StdEncoding.DecodeString(i.Content) + if err != nil { + content = []byte(i.Content) + } + + dir := filepath.Dir(destPath) + + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("while creating %s: %w", dir, err) + } + + // check sha256 + hash := crypto.SHA256.New() + if _, err := hash.Write(content); err != nil { + return fmt.Errorf("while hashing %s: %w", i.Name, err) + } + + gotHash := hex.EncodeToString(hash.Sum(nil)) + if gotHash != wantHash { + return fmt.Errorf("hash mismatch: expected %s, got %s. The index file is invalid, please run 'cscli hub update' and try again", wantHash, gotHash) + } + + if err := os.WriteFile(destPath, content, 0o600); err != nil { + return fmt.Errorf("while writing %s: %w", destPath, err) + } + + return nil +} + +// writeRemoteContentTo downloads the content to the specified path and checks the hash. +func (i *Item) writeRemoteContentTo(ctx context.Context, destPath, wantHash string) (bool, string, error) { + url, err := i.hub.remote.urlTo(i.RemotePath) + if err != nil { + return false, "", fmt.Errorf("failed to build request: %w", err) + } + + d := downloader. + New(). + WithHTTPClient(HubClient). + ToFile(destPath). + WithETagFn(downloader.SHA256). + WithMakeDirs(true). + WithLogger(logrus.WithField("url", url)). + CompareContent(). + VerifyHash("sha256", wantHash) + + hasherr := downloader.HashMismatchError{} + + downloaded, err := d.Download(ctx, url) + + switch { + case errors.As(err, &hasherr): + i.hub.logger.Warnf("%s. The index file is outdated, please run 'cscli hub update' and try again", err.Error()) + case err != nil: + return false, "", err + } + + return downloaded, url, nil +} + +// FetchContentTo writes the last version of the item's YAML file to the specified path. +// Returns whether the file was downloaded, and the remote url for feedback purposes. +func (i *Item) FetchContentTo(ctx context.Context, destPath string) (bool, string, error) { + wantHash := i.latestHash() + if wantHash == "" { + return false, "", fmt.Errorf("%s: latest hash missing from index. The index file is invalid, please run 'cscli hub update' and try again", i.FQName()) + } + + // Use the embedded content if available + if i.Content != "" { + if err := i.writeEmbeddedContentTo(destPath, wantHash); err != nil { + return false, "", err + } + + return true, fmt.Sprintf("(embedded in %s)", i.hub.local.HubIndexFile), nil + } + + return i.writeRemoteContentTo(ctx, destPath, wantHash) +} diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index f74a794a512..55469fed711 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -61,7 +61,7 @@ func (h *Hub) Load() error { h.logger.Debugf("loading hub idx %s", h.local.HubIndexFile) if err := h.parseIndex(); err != nil { - return fmt.Errorf("failed to load hub index: %w", err) + return err } if err := h.localSync(); err != nil { @@ -82,21 +82,25 @@ func (h *Hub) parseIndex() error { return fmt.Errorf("failed to parse index: %w", err) } - h.logger.Debugf("%d item types in hub index", len(ItemTypes)) - // Iterate over the different types to complete the struct for _, itemType := range ItemTypes { - h.logger.Tracef("%s: %d items", itemType, len(h.GetItemMap(itemType))) - for name, item := range h.GetItemMap(itemType) { - item.hub = h - item.Name = name + if item == nil { + // likely defined as empty object or null in the index file + return fmt.Errorf("%s:%s has no index metadata", itemType, name) + } - // if the item has no (redundant) author, take it from the json key - if item.Author == "" && strings.Contains(name, "/") { - item.Author = strings.Split(name, "/")[0] + if item.RemotePath == "" { + return fmt.Errorf("%s:%s has no download path", itemType, name) } + if (itemType == PARSERS || itemType == POSTOVERFLOWS) && item.Stage == "" { + return fmt.Errorf("%s:%s has no stage", itemType, name) + } + + item.hub = h + item.Name = name + item.Type = itemType item.FileName = path.Base(item.RemotePath) @@ -152,23 +156,21 @@ func (h *Hub) ItemStats() []string { return ret } -// Update downloads the latest version of the index and writes it to disk if it changed. It cannot be called after Load() -// unless the hub is completely empty. -func (h *Hub) Update(ctx context.Context) error { +// Update downloads the latest version of the index and writes it to disk if it changed. +// It cannot be called after Load() unless the hub is completely empty. +func (h *Hub) Update(ctx context.Context, withContent bool) error { if len(h.pathIndex) > 0 { // if this happens, it's a bug. return errors.New("cannot update hub after items have been loaded") } - downloaded, err := h.remote.fetchIndex(ctx, h.local.HubIndexFile) + downloaded, err := h.remote.fetchIndex(ctx, h.local.HubIndexFile, withContent) if err != nil { return err } - if downloaded { - h.logger.Infof("Wrote index to %s", h.local.HubIndexFile) - } else { - h.logger.Info("hub index is up to date") + if !downloaded { + fmt.Println("Nothing to do, the hub index is up to date.") } return nil diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index 1c2c9ccceca..727b9a18fdf 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -24,7 +24,7 @@ func TestInitHubUpdate(t *testing.T) { ctx := context.Background() - err = hub.Update(ctx) + err = hub.Update(ctx, false) require.NoError(t, err) err = hub.Load() @@ -58,7 +58,7 @@ func TestUpdateIndex(t *testing.T) { ctx := context.Background() - err = hub.Update(ctx) + err = hub.Update(ctx, false) cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'") // bad domain @@ -70,7 +70,7 @@ func TestUpdateIndex(t *testing.T) { IndexPath: ".index.json", } - err = hub.Update(ctx) + err = hub.Update(ctx, false) require.NoError(t, err) // XXX: this is not failing // cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") @@ -86,6 +86,6 @@ func TestUpdateIndex(t *testing.T) { hub.local.HubIndexFile = "/does/not/exist/index.json" - err = hub.Update(ctx) + err = hub.Update(ctx, false) cstest.RequireErrorContains(t, err, "failed to create temporary download file for /does/not/exist/index.json:") } diff --git a/pkg/cwhub/item.go b/pkg/cwhub/item.go index 32d1acf94ff..8cdb88a18ed 100644 --- a/pkg/cwhub/item.go +++ b/pkg/cwhub/item.go @@ -2,11 +2,15 @@ package cwhub import ( "encoding/json" + "errors" "fmt" + "io/fs" + "os" "path/filepath" "slices" "github.com/Masterminds/semver/v3" + yaml "gopkg.in/yaml.v3" "github.com/crowdsecurity/crowdsec/pkg/emoji" ) @@ -98,52 +102,91 @@ func (s *ItemState) Emoji() string { } } +type Dependencies struct { + Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` + PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` + Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` + Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` + Contexts []string `json:"contexts,omitempty" yaml:"contexts,omitempty"` + AppsecConfigs []string `json:"appsec-configs,omitempty" yaml:"appsec-configs,omitempty"` + AppsecRules []string `json:"appsec-rules,omitempty" yaml:"appsec-rules,omitempty"` +} + +// a group of items of the same type +type itemgroup struct { + typeName string + itemNames []string +} + +func (d Dependencies) byType() []itemgroup { + return []itemgroup{ + {PARSERS, d.Parsers}, + {POSTOVERFLOWS, d.PostOverflows}, + {SCENARIOS, d.Scenarios}, + {CONTEXTS, d.Contexts}, + {APPSEC_CONFIGS, d.AppsecConfigs}, + {APPSEC_RULES, d.AppsecRules}, + {COLLECTIONS, d.Collections}, + } +} + +// SubItems iterates over the sub-items in the struct, excluding the ones that were not found in the hub. +func (d Dependencies) SubItems(hub *Hub) func(func(*Item) bool) { + return func(yield func(*Item) bool) { + for _, typeGroup := range d.byType() { + for _, name := range typeGroup.itemNames { + s := hub.GetItem(typeGroup.typeName, name) + if s == nil { + continue + } + if !yield(s) { + return + } + } + } + } +} + // Item is created from an index file and enriched with local info. type Item struct { hub *Hub // back pointer to the hub, to retrieve other items and call install/remove methods State ItemState `json:"-" yaml:"-"` // local state, not stored in the index - Type string `json:"type,omitempty" yaml:"type,omitempty"` // one of the ItemTypes - Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... - Name string `json:"name,omitempty" yaml:"name,omitempty"` // usually "author/name" - FileName string `json:"file_name,omitempty" yaml:"file_name,omitempty"` // eg. apache2-logs.yaml + Type string `json:"type,omitempty" yaml:"type,omitempty"` + Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... + Name string `json:"name,omitempty" yaml:"name,omitempty"` // usually "author/name" + FileName string `json:"file_name,omitempty" yaml:"file_name,omitempty"` // eg. apache2-logs.yaml Description string `json:"description,omitempty" yaml:"description,omitempty"` - Content string `json:"content,omitempty" yaml:"-"` - Author string `json:"author,omitempty" yaml:"author,omitempty"` - References []string `json:"references,omitempty" yaml:"references,omitempty"` + Content string `json:"content,omitempty" yaml:"-"` + References []string `json:"references,omitempty" yaml:"references,omitempty"` + // NOTE: RemotePath could be derived from the other fields RemotePath string `json:"path,omitempty" yaml:"path,omitempty"` // path relative to the base URL eg. /parsers/stage/author/file.yaml Version string `json:"version,omitempty" yaml:"version,omitempty"` // the last available version Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // all the known versions - // if it's a collection, it can have sub items - Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` - PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` - Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` - Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` - Contexts []string `json:"contexts,omitempty" yaml:"contexts,omitempty"` - AppsecConfigs []string `json:"appsec-configs,omitempty" yaml:"appsec-configs,omitempty"` - AppsecRules []string `json:"appsec-rules,omitempty" yaml:"appsec-rules,omitempty"` + // The index contains the dependencies of the "latest" version (collections only) + Dependencies } -// installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local +// InstallPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local // (eg. /etc/crowdsec/collections/xyz.yaml). // Raises an error if the path goes outside of the install dir. -func (i *Item) installPath() (string, error) { +func (i *Item) InstallPath() (string, error) { p := i.Type if i.Stage != "" { p = filepath.Join(p, i.Stage) } - return safePath(i.hub.local.InstallDir, filepath.Join(p, i.FileName)) + return SafePath(i.hub.local.InstallDir, filepath.Join(p, i.FileName)) } -// downloadPath returns the location of the actual config file in the hub +// DownloadPath returns the location of the actual config file in the hub // (eg. /etc/crowdsec/hub/collections/author/xyz.yaml). // Raises an error if the path goes outside of the hub dir. -func (i *Item) downloadPath() (string, error) { - ret, err := safePath(i.hub.local.HubDir, i.RemotePath) +func (i *Item) DownloadPath() (string, error) { + ret, err := SafePath(i.hub.local.HubDir, i.RemotePath) if err != nil { return "", err } @@ -203,76 +246,50 @@ func (i Item) MarshalYAML() (interface{}, error) { }, nil } -// SubItems returns a slice of sub-items, excluding the ones that were not found. -func (i *Item) SubItems() []*Item { - sub := make([]*Item, 0) - - for _, name := range i.Parsers { - s := i.hub.GetItem(PARSERS, name) - if s == nil { - continue - } - - sub = append(sub, s) - } - - for _, name := range i.PostOverflows { - s := i.hub.GetItem(POSTOVERFLOWS, name) - if s == nil { - continue - } +// LatestDependencies returns a slice of sub-items of the "latest" available version of the item, as opposed to the version that is actually installed. The information comes from the index. +func (i *Item) LatestDependencies() Dependencies { + return i.Dependencies +} - sub = append(sub, s) +// CurrentSubItems returns a slice of sub-items of the installed version, excluding the ones that were not found. +// The list comes from the content file if parseable, otherwise from the index (same as LatestDependencies). +func (i *Item) CurrentDependencies() Dependencies { + if !i.HasSubItems() { + return Dependencies{} } - for _, name := range i.Scenarios { - s := i.hub.GetItem(SCENARIOS, name) - if s == nil { - continue - } - - sub = append(sub, s) + if i.State.UpToDate { + return i.Dependencies } - for _, name := range i.Contexts { - s := i.hub.GetItem(CONTEXTS, name) - if s == nil { - continue - } - - sub = append(sub, s) + contentPath, err := i.InstallPath() + if err != nil { + i.hub.logger.Warningf("can't access dependencies for %s, using index", i.FQName()) + return i.Dependencies } - for _, name := range i.AppsecConfigs { - s := i.hub.GetItem(APPSEC_CONFIGS, name) - if s == nil { - continue - } - - sub = append(sub, s) + currentContent, err := os.ReadFile(contentPath) + if errors.Is(err, fs.ErrNotExist) { + return i.Dependencies } - - for _, name := range i.AppsecRules { - s := i.hub.GetItem(APPSEC_RULES, name) - if s == nil { - continue - } - - sub = append(sub, s) + if err != nil { + // a file might be corrupted, or in development + i.hub.logger.Warningf("can't read dependencies for %s, using index", i.FQName()) + return i.Dependencies } - for _, name := range i.Collections { - s := i.hub.GetItem(COLLECTIONS, name) - if s == nil { - continue - } + var d Dependencies - sub = append(sub, s) + // XXX: assume collection content never has multiple documents + if err := yaml.Unmarshal(currentContent, &d); err != nil { + i.hub.logger.Warningf("can't parse dependencies for %s, using index", i.FQName()) + return i.Dependencies } - - return sub + + return d } + func (i *Item) logMissingSubItems() { if !i.HasSubItems() { return @@ -337,7 +354,59 @@ func (i *Item) Ancestors() []*Item { return ret } -// descendants returns a list of all (direct or indirect) dependencies of the item. +// SafeToRemoveDeps returns a slice of dependencies that can be safely removed when this item is removed. +// The returned slice can contain items that are not installed, or not downloaded. +func (i *Item) SafeToRemoveDeps() ([]*Item, error) { + ret := make([]*Item, 0) + + // can return err for circular dependencies + descendants, err := i.descendants() + if err != nil { + return nil, err + } + + ancestors := i.Ancestors() + + for sub := range i.CurrentDependencies().SubItems(i.hub) { + safe := true + + // if the sub depends on a collection that is not a direct or indirect dependency + // of the current item, it is not removed + for _, subParent := range sub.Ancestors() { + if !subParent.State.Installed { + continue + } + + // the ancestor that would block the removal of the sub item is also an ancestor + // of the item we are removing, so we don't want false warnings + // (e.g. crowdsecurity/sshd-logs was not removed because it also belongs to crowdsecurity/linux, + // while we are removing crowdsecurity/sshd) + if slices.Contains(ancestors, subParent) { + continue + } + + // the sub-item belongs to the item we are removing, but we already knew that + if subParent == i { + continue + } + + if !slices.Contains(descendants, subParent) { + // not removing %s because it also belongs to %s", sub.FQName(), subParent.FQName()) + safe = false + break + } + } + + if safe { + ret = append(ret, sub) + } + } + + return ret, nil +} + + +// descendants returns a list of all (direct or indirect) dependencies of the item's current version. func (i *Item) descendants() ([]*Item, error) { var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error @@ -352,7 +421,7 @@ func (i *Item) descendants() ([]*Item, error) { visited[item] = true - for _, subItem := range item.SubItems() { + for subItem := range item.CurrentDependencies().SubItems(item.hub) { if subItem == i { return fmt.Errorf("circular dependency detected: %s depends on %s", item.Name, i.Name) } diff --git a/pkg/cwhub/iteminstall.go b/pkg/cwhub/iteminstall.go deleted file mode 100644 index 912897d0d7e..00000000000 --- a/pkg/cwhub/iteminstall.go +++ /dev/null @@ -1,73 +0,0 @@ -package cwhub - -import ( - "context" - "fmt" -) - -// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items. -func (i *Item) enable() error { - if i.State.Installed { - if i.State.Tainted { - return fmt.Errorf("%s is tainted, won't overwrite unless --force", i.Name) - } - - if i.State.IsLocal() { - return fmt.Errorf("%s is local, won't overwrite", i.Name) - } - - // if it's a collection, check sub-items even if the collection file itself is up-to-date - if i.State.UpToDate && !i.HasSubItems() { - i.hub.logger.Tracef("%s is installed and up-to-date, skip.", i.Name) - return nil - } - } - - for _, sub := range i.SubItems() { - if err := sub.enable(); err != nil { - return fmt.Errorf("while installing %s: %w", sub.Name, err) - } - } - - if err := i.createInstallLink(); err != nil { - return err - } - - i.hub.logger.Infof("Enabled %s: %s", i.Type, i.Name) - i.State.Installed = true - - return nil -} - -// Install installs the item from the hub, downloading it if needed. -func (i *Item) Install(ctx context.Context, force bool, downloadOnly bool) error { - if downloadOnly && i.State.Downloaded && i.State.UpToDate { - i.hub.logger.Infof("%s is already downloaded and up-to-date", i.Name) - - if !force { - return nil - } - } - - downloaded, err := i.downloadLatest(ctx, force, true) - if err != nil { - return err - } - - if downloadOnly && downloaded { - return nil - } - - if err := i.enable(); err != nil { - return fmt.Errorf("while enabling %s: %w", i.Name, err) - } - - // a check on stdout is used while scripting to know if the hub has been upgraded - // and a configuration reload is required - // TODO: use a better way to communicate this - fmt.Printf("installed %s\n", i.Name) - - i.hub.logger.Infof("Enabled %s", i.Name) - - return nil -} diff --git a/pkg/cwhub/iteminstall_test.go b/pkg/cwhub/iteminstall_test.go index 5bfc7e8148e..ba47f2f4b4a 100644 --- a/pkg/cwhub/iteminstall_test.go +++ b/pkg/cwhub/iteminstall_test.go @@ -1,5 +1,9 @@ package cwhub +// XXX: these tests are obsolete + +/* + import ( "context" "os" @@ -103,7 +107,7 @@ func TestInstallParser(t *testing.T) { - force update it - check its status - remove it - */ + * hub := envSetup(t) // map iteration is random by itself @@ -126,7 +130,7 @@ func TestInstallCollection(t *testing.T) { - force update it - check its status - remove it - */ + * hub := envSetup(t) // map iteration is random by itself @@ -139,3 +143,5 @@ func TestInstallCollection(t *testing.T) { break } } + +*/ diff --git a/pkg/cwhub/itemlink.go b/pkg/cwhub/itemlink.go deleted file mode 100644 index 8a78d6805b7..00000000000 --- a/pkg/cwhub/itemlink.go +++ /dev/null @@ -1,78 +0,0 @@ -package cwhub - -import ( - "fmt" - "os" - "path/filepath" -) - -// createInstallLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir. -func (i *Item) createInstallLink() error { - dest, err := i.installPath() - if err != nil { - return err - } - - destDir := filepath.Dir(dest) - if err = os.MkdirAll(destDir, os.ModePerm); err != nil { - return fmt.Errorf("while creating %s: %w", destDir, err) - } - - if _, err = os.Lstat(dest); !os.IsNotExist(err) { - i.hub.logger.Infof("%s already exists.", dest) - return nil - } - - src, err := i.downloadPath() - if err != nil { - return err - } - - if err = os.Symlink(src, dest); err != nil { - return fmt.Errorf("while creating symlink from %s to %s: %w", src, dest, err) - } - - return nil -} - -// removeInstallLink removes the symlink to the downloaded content. -func (i *Item) removeInstallLink() error { - syml, err := i.installPath() - if err != nil { - return err - } - - stat, err := os.Lstat(syml) - if err != nil { - return err - } - - // if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ... - if stat.Mode()&os.ModeSymlink == 0 { - i.hub.logger.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml) - return fmt.Errorf("%s isn't managed by hub", i.Name) - } - - hubpath, err := os.Readlink(syml) - if err != nil { - return fmt.Errorf("while reading symlink: %w", err) - } - - src, err := i.downloadPath() - if err != nil { - return err - } - - if hubpath != src { - i.hub.logger.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, src) - return fmt.Errorf("%s isn't managed by hub", i.Name) - } - - if err := os.Remove(syml); err != nil { - return fmt.Errorf("while removing symlink: %w", err) - } - - i.hub.logger.Infof("Removed symlink [%s]: %s", i.Name, syml) - - return nil -} diff --git a/pkg/cwhub/itemremove.go b/pkg/cwhub/itemremove.go deleted file mode 100644 index eca0c856237..00000000000 --- a/pkg/cwhub/itemremove.go +++ /dev/null @@ -1,138 +0,0 @@ -package cwhub - -import ( - "fmt" - "os" - "slices" -) - -// purge removes the actual config file that was downloaded. -func (i *Item) purge() (bool, error) { - if !i.State.Downloaded { - i.hub.logger.Debugf("removing %s: not downloaded -- no need to remove", i.Name) - return false, nil - } - - src, err := i.downloadPath() - if err != nil { - return false, err - } - - if err := os.Remove(src); err != nil { - if os.IsNotExist(err) { - i.hub.logger.Debugf("%s doesn't exist, no need to remove", src) - return false, nil - } - - return false, fmt.Errorf("while removing file: %w", err) - } - - i.State.Downloaded = false - i.hub.logger.Infof("Removed source file [%s]: %s", i.Name, src) - - return true, nil -} - -// disable removes the install link, and optionally the downloaded content. -func (i *Item) disable(purge bool, force bool) (bool, error) { - didRemove := true - - err := i.removeInstallLink() - if os.IsNotExist(err) { - if !purge && !force { - link, _ := i.installPath() - return false, fmt.Errorf("link %s does not exist (override with --force or --purge)", link) - } - - didRemove = false - } else if err != nil { - return false, err - } - - i.State.Installed = false - didPurge := false - - if purge { - if didPurge, err = i.purge(); err != nil { - return didRemove, err - } - } - - ret := didRemove || didPurge - - return ret, nil -} - -// Remove disables the item, optionally removing the downloaded content. -func (i *Item) Remove(purge bool, force bool) (bool, error) { - if i.State.IsLocal() { - i.hub.logger.Warningf("%s is a local item, please delete manually", i.Name) - return false, nil - } - - if i.State.Tainted && !force { - return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name) - } - - if !i.State.Installed && !purge { - i.hub.logger.Infof("removing %s: not installed -- no need to remove", i.Name) - return false, nil - } - - removed := false - - descendants, err := i.descendants() - if err != nil { - return false, err - } - - ancestors := i.Ancestors() - - for _, sub := range i.SubItems() { - if !sub.State.Installed { - continue - } - - // if the sub depends on a collection that is not a direct or indirect dependency - // of the current item, it is not removed - for _, subParent := range sub.Ancestors() { - if !purge && !subParent.State.Installed { - continue - } - - // the ancestor that would block the removal of the sub item is also an ancestor - // of the item we are removing, so we don't want false warnings - // (e.g. crowdsecurity/sshd-logs was not removed because it also belongs to crowdsecurity/linux, - // while we are removing crowdsecurity/sshd) - if slices.Contains(ancestors, subParent) { - continue - } - - // the sub-item belongs to the item we are removing, but we already knew that - if subParent == i { - continue - } - - if !slices.Contains(descendants, subParent) { - i.hub.logger.Infof("%s was not removed because it also belongs to %s", sub.Name, subParent.Name) - continue - } - } - - subRemoved, err := sub.Remove(purge, force) - if err != nil { - return false, fmt.Errorf("unable to disable %s: %w", i.Name, err) - } - - removed = removed || subRemoved - } - - didDisable, err := i.disable(purge, force) - if err != nil { - return false, fmt.Errorf("while removing %s: %w", i.Name, err) - } - - removed = removed || didDisable - - return removed, nil -} diff --git a/pkg/cwhub/itemupgrade.go b/pkg/cwhub/itemupgrade.go deleted file mode 100644 index 105e5ebec31..00000000000 --- a/pkg/cwhub/itemupgrade.go +++ /dev/null @@ -1,254 +0,0 @@ -package cwhub - -// Install, upgrade and remove items from the hub to the local configuration - -import ( - "context" - "crypto" - "encoding/base64" - "encoding/hex" - "errors" - "fmt" - "os" - "path/filepath" - - "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/go-cs-lib/downloader" - - "github.com/crowdsecurity/crowdsec/pkg/emoji" -) - -// Upgrade downloads and applies the last version of the item from the hub. -func (i *Item) Upgrade(ctx context.Context, force bool) (bool, error) { - if i.State.IsLocal() { - i.hub.logger.Infof("not upgrading %s: local item", i.Name) - return false, nil - } - - if !i.State.Downloaded { - return false, fmt.Errorf("can't upgrade %s: not installed", i.Name) - } - - if !i.State.Installed { - return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", i.Name) - } - - if i.State.UpToDate { - i.hub.logger.Infof("%s: up-to-date", i.Name) - - if err := i.DownloadDataIfNeeded(ctx, force); err != nil { - return false, fmt.Errorf("%s: download failed: %w", i.Name, err) - } - - if !force { - // no upgrade needed - return false, nil - } - } - - if _, err := i.downloadLatest(ctx, force, true); err != nil { - return false, fmt.Errorf("%s: download failed: %w", i.Name, err) - } - - if !i.State.UpToDate { - if i.State.Tainted { - i.hub.logger.Warningf("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name) - } - - return false, nil - } - - // a check on stdout is used while scripting to know if the hub has been upgraded - // and a configuration reload is required - // TODO: use a better way to communicate this - fmt.Printf("updated %s\n", i.Name) - i.hub.logger.Infof("%v %s: updated", emoji.Package, i.Name) - - return true, nil -} - -// downloadLatest downloads the latest version of the item to the hub directory. -func (i *Item) downloadLatest(ctx context.Context, overwrite bool, updateOnly bool) (bool, error) { - i.hub.logger.Debugf("Downloading %s %s", i.Type, i.Name) - - for _, sub := range i.SubItems() { - if !sub.State.Installed && updateOnly && sub.State.Downloaded { - i.hub.logger.Debugf("skipping upgrade of %s: not installed", i.Name) - continue - } - - i.hub.logger.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.State.Installed, updateOnly) - - // recurse as it's a collection - if sub.HasSubItems() { - i.hub.logger.Tracef("collection, recurse") - - if _, err := sub.downloadLatest(ctx, overwrite, updateOnly); err != nil { - return false, err - } - } - - downloaded := sub.State.Downloaded - - if _, err := sub.download(ctx, overwrite); err != nil { - return false, err - } - - // We need to enable an item when it has been added to a collection since latest release of the collection. - // We check if sub.Downloaded is false because maybe the item has been disabled by the user. - if !sub.State.Installed && !downloaded { - if err := sub.enable(); err != nil { - return false, fmt.Errorf("enabling '%s': %w", sub.Name, err) - } - } - } - - if !i.State.Installed && updateOnly && i.State.Downloaded && !overwrite { - i.hub.logger.Debugf("skipping upgrade of %s: not installed", i.Name) - return false, nil - } - - return i.download(ctx, overwrite) -} - -// FetchContentTo downloads the last version of the item's YAML file to the specified path. -func (i *Item) FetchContentTo(ctx context.Context, destPath string) (bool, string, error) { - wantHash := i.latestHash() - if wantHash == "" { - return false, "", errors.New("latest hash missing from index. The index file is invalid, please run 'cscli hub update' and try again") - } - - // Use the embedded content if available - if i.Content != "" { - // the content was historically base64 encoded - content, err := base64.StdEncoding.DecodeString(i.Content) - if err != nil { - content = []byte(i.Content) - } - - dir := filepath.Dir(destPath) - - if err := os.MkdirAll(dir, 0o755); err != nil { - return false, "", fmt.Errorf("while creating %s: %w", dir, err) - } - - // check sha256 - hash := crypto.SHA256.New() - if _, err := hash.Write(content); err != nil { - return false, "", fmt.Errorf("while hashing %s: %w", i.Name, err) - } - - gotHash := hex.EncodeToString(hash.Sum(nil)) - if gotHash != wantHash { - return false, "", fmt.Errorf("hash mismatch: expected %s, got %s. The index file is invalid, please run 'cscli hub update' and try again", wantHash, gotHash) - } - - if err := os.WriteFile(destPath, content, 0o600); err != nil { - return false, "", fmt.Errorf("while writing %s: %w", destPath, err) - } - - i.hub.logger.Debugf("Wrote %s content from .index.json to %s", i.Name, destPath) - - return true, fmt.Sprintf("(embedded in %s)", i.hub.local.HubIndexFile), nil - } - - url, err := i.hub.remote.urlTo(i.RemotePath) - if err != nil { - return false, "", fmt.Errorf("failed to build request: %w", err) - } - - d := downloader. - New(). - WithHTTPClient(hubClient). - ToFile(destPath). - WithETagFn(downloader.SHA256). - WithMakeDirs(true). - WithLogger(logrus.WithField("url", url)). - CompareContent(). - VerifyHash("sha256", wantHash) - - // TODO: recommend hub update if hash does not match - - downloaded, err := d.Download(ctx, url) - if err != nil { - return false, "", err - } - - return downloaded, url, nil -} - -// download downloads the item from the hub and writes it to the hub directory. -func (i *Item) download(ctx context.Context, overwrite bool) (bool, error) { - // ensure that target file is within target dir - finalPath, err := i.downloadPath() - if err != nil { - return false, err - } - - if i.State.IsLocal() { - i.hub.logger.Warningf("%s is local, can't download", i.Name) - return false, nil - } - - // if user didn't --force, don't overwrite local, tainted, up-to-date files - if !overwrite { - if i.State.Tainted { - i.hub.logger.Debugf("%s: tainted, not updated", i.Name) - return false, nil - } - - if i.State.UpToDate { - // We still have to check if data files are present - i.hub.logger.Debugf("%s: up-to-date, not updated", i.Name) - } - } - - downloaded, _, err := i.FetchContentTo(ctx, finalPath) - if err != nil { - return false, err - } - - if downloaded { - i.hub.logger.Infof("Downloaded %s", i.Name) - } - - i.State.Downloaded = true - i.State.Tainted = false - i.State.UpToDate = true - - // read content to get the list of data files - reader, err := os.Open(finalPath) - if err != nil { - return false, fmt.Errorf("while opening %s: %w", finalPath, err) - } - - defer reader.Close() - - if err = downloadDataSet(ctx, i.hub.local.InstallDataDir, overwrite, reader, i.hub.logger); err != nil { - return false, fmt.Errorf("while downloading data for %s: %w", i.FileName, err) - } - - return true, nil -} - -// DownloadDataIfNeeded downloads the data set for the item. -func (i *Item) DownloadDataIfNeeded(ctx context.Context, force bool) error { - itemFilePath, err := i.installPath() - if err != nil { - return err - } - - itemFile, err := os.Open(itemFilePath) - if err != nil { - return fmt.Errorf("while opening %s: %w", itemFilePath, err) - } - - defer itemFile.Close() - - if err = downloadDataSet(ctx, i.hub.local.InstallDataDir, force, itemFile, i.hub.logger); err != nil { - return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) - } - - return nil -} diff --git a/pkg/cwhub/itemupgrade_test.go b/pkg/cwhub/itemupgrade_test.go index 5f9e4d1944e..e523a222d69 100644 --- a/pkg/cwhub/itemupgrade_test.go +++ b/pkg/cwhub/itemupgrade_test.go @@ -1,5 +1,7 @@ package cwhub +/* + import ( "context" "testing" @@ -221,3 +223,5 @@ func pushUpdateToCollectionInHub() { responseByPath["/crowdsecurity/master/.index.json"] = fileToStringX("./testdata/index2.json") responseByPath["/crowdsecurity/master/collections/crowdsecurity/test_collection.yaml"] = fileToStringX("./testdata/collection_v2.yaml") } + +*/ diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go index 8d2dc2dbb94..c96471b390c 100644 --- a/pkg/cwhub/remote.go +++ b/pkg/cwhub/remote.go @@ -3,6 +3,7 @@ package cwhub import ( "context" "fmt" + "net/http" "net/url" "github.com/sirupsen/logrus" @@ -15,7 +16,6 @@ type RemoteHubCfg struct { Branch string URLTemplate string IndexPath string - EmbedItemContent bool } // urlTo builds the URL to download a file from the remote hub. @@ -32,7 +32,7 @@ func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) { return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil } -// addURLParam adds the "with_content=true" parameter to the URL if it's not already present. +// addURLParam adds a parameter with a value (ex. "with_content=true") to the URL if it's not already present. func addURLParam(rawURL string, param string, value string) (string, error) { parsedURL, err := url.Parse(rawURL) if err != nil { @@ -51,7 +51,7 @@ func addURLParam(rawURL string, param string, value string) (string, error) { } // fetchIndex downloads the index from the hub and returns the content. -func (r *RemoteHubCfg) fetchIndex(ctx context.Context, destPath string) (bool, error) { +func (r *RemoteHubCfg) fetchIndex(ctx context.Context, destPath string, withContent bool) (bool, error) { if r == nil { return false, ErrNilRemoteHub } @@ -61,7 +61,7 @@ func (r *RemoteHubCfg) fetchIndex(ctx context.Context, destPath string) (bool, e return false, fmt.Errorf("failed to build hub index request: %w", err) } - if r.EmbedItemContent { + if withContent { url, err = addURLParam(url, "with_content", "true") if err != nil { return false, fmt.Errorf("failed to add 'with_content' parameter to URL: %w", err) @@ -70,11 +70,14 @@ func (r *RemoteHubCfg) fetchIndex(ctx context.Context, destPath string) (bool, e downloaded, err := downloader. New(). - WithHTTPClient(hubClient). + WithHTTPClient(HubClient). ToFile(destPath). WithETagFn(downloader.SHA256). CompareContent(). WithLogger(logrus.WithField("url", url)). + BeforeRequest(func(_ *http.Request) { + fmt.Println("Downloading "+destPath) + }). Download(ctx, url) if err != nil { return false, err diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index c82822e64ef..d2b59df35d6 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -105,6 +105,9 @@ func (h *Hub) getItemFileInfo(path string, logger *logrus.Logger) (*itemFileInfo fname := subsHub[2] if ftype == PARSERS || ftype == POSTOVERFLOWS { + if len(subsHub) < 4 { + return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subsHub)) + } stage = subsHub[1] fauthor = subsHub[2] fname = subsHub[3] @@ -308,17 +311,12 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { // if we are walking hub dir, just mark present files as downloaded if info.inhub { - // wrong author - if info.fauthor != item.Author { - continue - } - // not the item we're looking for if !item.validPath(info.fauthor, info.fname) { continue } - src, err := item.downloadPath() + src, err := item.DownloadPath() if err != nil { return err } @@ -364,7 +362,7 @@ func (i *Item) checkSubItemVersions() []string { // ensure all the sub-items are installed, or tag the parent as tainted i.hub.logger.Tracef("checking submembers of %s installed:%t", i.Name, i.State.Installed) - for _, sub := range i.SubItems() { + for sub := range i.CurrentDependencies().SubItems(i.hub) { i.hub.logger.Tracef("check %s installed:%t", sub.Name, sub.State.Installed) if !i.State.Installed { diff --git a/pkg/emoji/emoji.go b/pkg/emoji/emoji.go index 51295a85411..9b939249bf0 100644 --- a/pkg/emoji/emoji.go +++ b/pkg/emoji/emoji.go @@ -11,4 +11,8 @@ const ( QuestionMark = "\u2753" // ❓ RedCircle = "\U0001f534" // 🔴 Warning = "\u26a0\ufe0f" // ⚠️ + InboxTray = "\U0001f4e5" // 📥 + DownArrow = "\u2b07" // ⬇️ + Wastebasket = "\U0001f5d1" // 🗑 + Sync = "\U0001F504" // 🔄 official name is Anticlockwise Downwards and Upwards Open Circle Arrows and I'm not even joking ) diff --git a/pkg/hubops/colorize.go b/pkg/hubops/colorize.go new file mode 100644 index 00000000000..3af2aecab93 --- /dev/null +++ b/pkg/hubops/colorize.go @@ -0,0 +1,38 @@ +package hubops + +import ( + "strings" + + "github.com/fatih/color" + + "github.com/crowdsecurity/crowdsec/pkg/emoji" +) + +// colorizeItemName splits the input string on "/" and colorizes the second part. +func colorizeItemName(fullname string) string { + parts := strings.SplitN(fullname, "/", 2) + if len(parts) == 2 { + bold := color.New(color.Bold) + author := parts[0] + name := parts[1] + return author + "/" + bold.Sprint(name) + } + return fullname +} + +func colorizeOpType(opType string) string { + switch opType { + case (&DownloadCommand{}).OperationType(): + return emoji.InboxTray + " " + color.BlueString(opType) + case (&EnableCommand{}).OperationType(): + return emoji.CheckMarkButton + " " + color.GreenString(opType) + case (&DisableCommand{}).OperationType(): + return emoji.CrossMark + " " + color.RedString(opType) + case (&PurgeCommand{}).OperationType(): + return emoji.Wastebasket + " " + color.RedString(opType) + case (&DataRefreshCommand{}).OperationType(): + return emoji.Sync + " " + opType + } + + return opType +} diff --git a/pkg/hubops/datarefresh.go b/pkg/hubops/datarefresh.go new file mode 100644 index 00000000000..985db8c1a11 --- /dev/null +++ b/pkg/hubops/datarefresh.go @@ -0,0 +1,75 @@ +package hubops + +import ( + "context" + "fmt" + "os" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +// XXX: TODO: temporary for hubtests, but will have to go. +// DownloadDataIfNeeded downloads the data set for the item. +func DownloadDataIfNeeded(ctx context.Context, hub *cwhub.Hub, item *cwhub.Item, force bool) (bool, error) { + itemFilePath, err := item.InstallPath() + if err != nil { + return false, err + } + + itemFile, err := os.Open(itemFilePath) + if err != nil { + return false, fmt.Errorf("while opening %s: %w", itemFilePath, err) + } + + defer itemFile.Close() + + needReload, err := downloadDataSet(ctx, hub.GetDataDir(), force, itemFile) + if err != nil { + return needReload, fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) + } + + return needReload, nil +} + +// DataRefreshCommand updates the data files associated with the installed hub items. +type DataRefreshCommand struct { + Force bool +} + +func NewDataRefreshCommand(force bool) *DataRefreshCommand { + return &DataRefreshCommand{Force: force} +} + +func (c *DataRefreshCommand) Prepare(plan *ActionPlan) (bool, error) { + // we can't prepare much at this point because we don't know which data files yet, + // and items needs to be downloaded/updated + // evertyhing will be done in Run() + return true, nil +} + +func (c *DataRefreshCommand) Run(ctx context.Context, plan *ActionPlan) error { + for _, itemType := range cwhub.ItemTypes { + for _, item := range plan.hub.GetInstalledByType(itemType, true) { + needReload, err := DownloadDataIfNeeded(ctx, plan.hub, item, c.Force) + if err != nil { + return err + } + + plan.ReloadNeeded = plan.ReloadNeeded || needReload + } + } + + return nil +} + +func (c *DataRefreshCommand) OperationType() string { + return "check & update data files" +} + +func (c *DataRefreshCommand) ItemType() string { + return "" +} + +func (c *DataRefreshCommand) Detail() string { + return "" +} diff --git a/pkg/hubops/disable.go b/pkg/hubops/disable.go new file mode 100644 index 00000000000..b6368e85036 --- /dev/null +++ b/pkg/hubops/disable.go @@ -0,0 +1,121 @@ +package hubops + +import ( + "context" + "fmt" + "os" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +// RemoveInstallLink removes the item's symlink between the installation directory and the local hub. +func RemoveInstallLink(i *cwhub.Item) error { + syml, err := i.InstallPath() + if err != nil { + return err + } + + stat, err := os.Lstat(syml) + if err != nil { + return err + } + + // if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ... + if stat.Mode()&os.ModeSymlink == 0 { + return fmt.Errorf("%s isn't managed by hub", i.Name) + } + + hubpath, err := os.Readlink(syml) + if err != nil { + return fmt.Errorf("while reading symlink: %w", err) + } + + src, err := i.DownloadPath() + if err != nil { + return err + } + + if hubpath != src { + return fmt.Errorf("%s isn't managed by hub", i.Name) + } + + if err := os.Remove(syml); err != nil { + return fmt.Errorf("while removing symlink: %w", err) + } + + return nil +} + +// DisableCommand uninstalls an item and its dependencies, ensuring that no +// sub-item is left in an inconsistent state. +type DisableCommand struct { + Item *cwhub.Item + Force bool +} + +func NewDisableCommand(item *cwhub.Item, force bool) *DisableCommand { + return &DisableCommand{Item: item, Force: force} +} + +func (c *DisableCommand) Prepare(plan *ActionPlan) (bool, error) { + i := c.Item + + if i.State.IsLocal() { + plan.Warning(i.FQName() + " is a local item, please delete manually") + return false, nil + } + + if i.State.Tainted && !c.Force { + return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name) + } + + if !i.State.Installed { + return false, nil + } + + subsToRemove, err := i.SafeToRemoveDeps() + if err != nil { + return false, err + } + + for _, sub := range subsToRemove { + if !sub.State.Installed { + continue + } + + if err := plan.AddCommand(NewDisableCommand(sub, c.Force)); err != nil { + return false, err + } + } + + return true, nil +} + +func (c *DisableCommand) Run(ctx context.Context, plan *ActionPlan) error { + i := c.Item + + fmt.Println("disabling " + colorizeItemName(i.FQName())) + + if err := RemoveInstallLink(i); err != nil { + return fmt.Errorf("while disabling %s: %w", i.FQName(), err) + } + + plan.ReloadNeeded = true + + i.State.Installed = false + i.State.Tainted = false + + return nil +} + +func (c *DisableCommand) OperationType() string { + return "disable" +} + +func (c *DisableCommand) ItemType() string { + return c.Item.Type +} + +func (c *DisableCommand) Detail() string { + return colorizeItemName(c.Item.Name) +} diff --git a/pkg/hubops/download.go b/pkg/hubops/download.go new file mode 100644 index 00000000000..4a722efdb77 --- /dev/null +++ b/pkg/hubops/download.go @@ -0,0 +1,212 @@ +package hubops + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + "github.com/fatih/color" + + "github.com/crowdsecurity/go-cs-lib/downloader" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/types" +) + + +// DownloadCommand handles the downloading of hub items. +// It ensures that items are fetched from the hub (or from the index file if it also has content) +// managing dependencies and verifying the integrity of downloaded content. +// This is used by "cscli install" and "cscli upgrade". +// Tainted items require the force parameter, local items are skipped. +type DownloadCommand struct { + Item *cwhub.Item + Force bool +} + +func NewDownloadCommand(item *cwhub.Item, force bool) *DownloadCommand { + return &DownloadCommand{Item: item, Force: force} +} + +func (c *DownloadCommand) Prepare(plan *ActionPlan) (bool, error) { + i := c.Item + + if i.State.IsLocal() { + plan.Info(i.FQName() + " - not downloading local item") + return false, nil + } + + // XXX: if it's tainted do we upgrade the dependencies anyway? + if i.State.Tainted && !c.Force { + plan.Warning(i.FQName() + " is tainted, use '--force' to overwrite") + return false, nil + } + + toDisable := make(map[*cwhub.Item]struct{}) + + var disableKeys []*cwhub.Item + + if i.State.Installed { + for sub := range i.CurrentDependencies().SubItems(plan.hub) { + disableKeys = append(disableKeys, sub) + toDisable[sub] = struct{}{} + } + } + + for sub := range i.LatestDependencies().SubItems(plan.hub) { + if err := plan.AddCommand(NewDownloadCommand(sub, c.Force)); err != nil { + return false, err + } + + if i.State.Installed { + // ensure the _new_ dependencies are installed too + if err := plan.AddCommand(NewEnableCommand(sub, c.Force)); err != nil { + return false, err + } + + for _, sub2 := range disableKeys { + if sub2 == sub { + delete(toDisable, sub) + } + } + } + } + + for sub := range toDisable { + if err := plan.AddCommand(NewDisableCommand(sub, c.Force)); err != nil { + return false, err + } + } + + if i.State.Downloaded && i.State.UpToDate { + return false, nil + } + + return true, nil +} + +// The DataSet is a list of data sources required by an item (built from the data: section in the yaml). +type DataSet struct { + Data []types.DataSource `yaml:"data,omitempty"` +} + +// downloadDataSet downloads all the data files for an item. +func downloadDataSet(ctx context.Context, dataFolder string, force bool, reader io.Reader) (bool, error) { + needReload := false + + dec := yaml.NewDecoder(reader) + + for { + data := &DataSet{} + + if err := dec.Decode(data); err != nil { + if errors.Is(err, io.EOF) { + break + } + + return needReload, fmt.Errorf("while reading file: %w", err) + } + + for _, dataS := range data.Data { + // XXX: check context cancellation + destPath, err := cwhub.SafePath(dataFolder, dataS.DestPath) + if err != nil { + return needReload, err + } + + d := downloader. + New(). + WithHTTPClient(cwhub.HubClient). + ToFile(destPath). + CompareContent(). + BeforeRequest(func(req *http.Request) { + fmt.Printf("downloading %s\n", req.URL) + }). + WithLogger(logrus.WithField("url", dataS.SourceURL)) + + if !force { + d = d.WithLastModified(). + WithShelfLife(7 * 24 * time.Hour) + } + + downloaded, err := d.Download(ctx, dataS.SourceURL) + if err != nil { + return needReload, fmt.Errorf("while getting data: %w", err) + } + + needReload = needReload || downloaded + } + } + + return needReload, nil +} + +func (c *DownloadCommand) Run(ctx context.Context, plan *ActionPlan) error { + i := c.Item + + fmt.Printf("downloading %s\n", colorizeItemName(i.FQName())) + + // ensure that target file is within target dir + finalPath, err := i.DownloadPath() + if err != nil { + return err + } + + downloaded, _, err := i.FetchContentTo(ctx, finalPath) + if err != nil { + return fmt.Errorf("%s: %w", i.FQName(), err) + } + + if downloaded { + plan.ReloadNeeded = true + } + + i.State.Downloaded = true + i.State.Tainted = false + i.State.UpToDate = true + + // read content to get the list of data files + reader, err := os.Open(finalPath) + if err != nil { + return fmt.Errorf("while opening %s: %w", finalPath, err) + } + + defer reader.Close() + + needReload, err := downloadDataSet(ctx, plan.hub.GetDataDir(), c.Force, reader) + if err != nil { + return fmt.Errorf("while downloading data for %s: %w", i.FileName, err) + } + + if needReload { + plan.ReloadNeeded = true + } + + return nil +} + +func (c *DownloadCommand) OperationType() string { + return "download" +} + +func (c *DownloadCommand) ItemType() string { + return c.Item.Type +} + +func (c *DownloadCommand) Detail() string { + i := c.Item + + version := color.YellowString(i.Version) + + if i.State.Downloaded { + version = c.Item.State.LocalVersion + " -> " + color.YellowString(i.Version) + } + + return colorizeItemName(c.Item.Name) + " (" + version + ")" +} diff --git a/pkg/hubops/enable.go b/pkg/hubops/enable.go new file mode 100644 index 00000000000..40de40c8662 --- /dev/null +++ b/pkg/hubops/enable.go @@ -0,0 +1,113 @@ +package hubops + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +// EnableCommand installs a hub item and its dependencies. +// In case this command is called during an upgrade, the sub-items list it taken from the +// latest version in the index, otherwise from the version that is currently installed. +type EnableCommand struct { + Item *cwhub.Item + Force bool + FromLatest bool +} + +func NewEnableCommand(item *cwhub.Item, force bool) *EnableCommand { + return &EnableCommand{Item: item, Force: force} +} + +func (c *EnableCommand) Prepare(plan *ActionPlan) (bool, error) { + var dependencies cwhub.Dependencies + + i := c.Item + + if c.FromLatest { + // we are upgrading + dependencies = i.LatestDependencies() + } else { + dependencies = i.CurrentDependencies() + } + + for sub := range dependencies.SubItems(plan.hub) { + if err := plan.AddCommand(NewEnableCommand(sub, c.Force)); err != nil { + return false, err + } + } + + if i.State.Installed { + return false, nil + } + + return true, nil +} + +// CreateInstallLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir. +func CreateInstallLink(i *cwhub.Item) error { + dest, err := i.InstallPath() + if err != nil { + return err + } + + destDir := filepath.Dir(dest) + if err = os.MkdirAll(destDir, os.ModePerm); err != nil { + return fmt.Errorf("while creating %s: %w", destDir, err) + } + + if _, err = os.Lstat(dest); err == nil { + // already exists + return nil + } else if !os.IsNotExist(err) { + return fmt.Errorf("failed to stat %s: %w", dest, err) + } + + src, err := i.DownloadPath() + if err != nil { + return err + } + + if err = os.Symlink(src, dest); err != nil { + return fmt.Errorf("while creating symlink from %s to %s: %w", src, dest, err) + } + + return nil +} + +func (c *EnableCommand) Run(ctx context.Context, plan *ActionPlan) error { + i := c.Item + + fmt.Println("enabling " + colorizeItemName(i.FQName())) + + if !i.State.Downloaded { + // XXX: this a warning? + return fmt.Errorf("can't enable %s: not downloaded", i.FQName()) + } + + if err := CreateInstallLink(i); err != nil { + return fmt.Errorf("while enabling %s: %w", i.FQName(), err) + } + + plan.ReloadNeeded = true + + i.State.Installed = true + i.State.Tainted = false + + return nil +} + +func (c *EnableCommand) OperationType() string { + return "enable" +} + +func (c *EnableCommand) ItemType() string { + return c.Item.Type +} + +func (c *EnableCommand) Detail() string { + return colorizeItemName(c.Item.Name) +} diff --git a/pkg/hubops/plan.go b/pkg/hubops/plan.go new file mode 100644 index 00000000000..1535bc41c64 --- /dev/null +++ b/pkg/hubops/plan.go @@ -0,0 +1,250 @@ +package hubops + +import ( + "context" + "fmt" + "os" + "slices" + "strings" + + "github.com/AlecAivazis/survey/v2" + isatty "github.com/mattn/go-isatty" + "github.com/fatih/color" + + "github.com/crowdsecurity/go-cs-lib/slicetools" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +// Command represents an operation that can be performed on a CrowdSec hub item. +// +// Each concrete implementation defines a Prepare() method to check for errors and preconditions, +// decide which sub-commands are required (like installing dependencies) and add them to the action plan. +type Command interface { + // Prepare sets up the command for execution within the given + // ActionPlan. It may add additional commands to the ActionPlan based + // on dependencies or prerequisites. Returns a boolean indicating + // whether the command execution should be skipped (it can be + // redundant, like installing something that is already installed) and + // an error if the preparation failed. + // NOTE: Returning an error will bubble up from the plan.AddCommand() method, + // but Prepare() might already have modified the plan's command slice. + Prepare(*ActionPlan) (bool, error) + + // Run executes the command within the provided context and ActionPlan. + // It performs the actual operation and returns an error if execution fails. + // NOTE: Returning an error will currently stop the execution of the action plan. + Run(ctx context.Context, plan *ActionPlan) error + + // OperationType returns a unique string representing the type of operation to perform + // (e.g., "download", "enable"). + OperationType() string + + // ItemType returns the type of item the operation is performed on + // (e.g., "collections"). Used in confirmation prompt and dry-run. + ItemType() string + + // Detail provides further details on the operation, + // such as the item's name and version. + Detail() string +} + +// UniqueKey generates a unique string key for a Command based on its operation type, item type, and detail. +// Is is used to avoid adding duplicate commands to the action plan. +func UniqueKey(c Command) string { + return fmt.Sprintf("%s:%s:%s", c.OperationType(), c.ItemType(), c.Detail()) +} + +// ActionPlan orchestrates the sequence of operations (Commands) to manage CrowdSec hub items. +type ActionPlan struct { + // hold the list of Commands to be executed as part of the action plan. + // If a command is skipped (i.e. calling Prepare() returned false), it won't be included in the slice. + commands []Command + + // Tracks unique commands + commandsTracker map[string]struct{} + + // A reference to the Hub instance, required for dependency lookup. + hub *cwhub.Hub + + // Indicates whether a reload of the CrowdSec service is required after executing the action plan. + ReloadNeeded bool +} + +func NewActionPlan(hub *cwhub.Hub) *ActionPlan { + return &ActionPlan{ + hub: hub, + commandsTracker: make(map[string]struct{}), + } +} + +func (p *ActionPlan) AddCommand(c Command) error { + ok, err := c.Prepare(p) + if err != nil { + return err + } + + if ok { + key := UniqueKey(c) + if _, exists := p.commandsTracker[key]; !exists { + p.commands = append(p.commands, c) + p.commandsTracker[key] = struct{}{} + } + } + + return nil +} + +func (p *ActionPlan) Info(msg string) { + fmt.Println(msg) +} + +func (p *ActionPlan) Warning(msg string) { + fmt.Printf("%s %s\n", color.YellowString("WARN"), msg) +} + +// Description returns a string representation of the action plan. +// If verbose is false, the operations are grouped by item type and operation type. +// If verbose is true, they are listed as they appear in the command slice. +func (p *ActionPlan) Description(verbose bool) string { + if verbose { + return p.verboseDescription() + } + + return p.compactDescription() +} + +func (p *ActionPlan) verboseDescription() string { + sb := strings.Builder{} + + // Here we display the commands in the order they will be executed. + for _, cmd := range p.commands { + sb.WriteString(colorizeOpType(cmd.OperationType()) + " " + cmd.ItemType() + ":" + cmd.Detail() + "\n") + } + + return sb.String() +} + +// describe the operations of a given type in a compact way. +func describe(opType string, desc map[string]map[string][]string, sb *strings.Builder) { + if _, ok := desc[opType]; !ok { + return + } + + sb.WriteString(colorizeOpType(opType) + "\n") + + // iterate cwhub.ItemTypes in reverse order, so we have collections first + for _, itemType := range slicetools.Backward(cwhub.ItemTypes) { + if desc[opType][itemType] == nil { + continue + } + + details := desc[opType][itemType] + // Sorting for user convenience, but it's not the same order the commands will be carried out. + slices.Sort(details) + + if itemType != "" { + sb.WriteString(" " + itemType + ": ") + } + + if len(details) != 0 { + sb.WriteString(strings.Join(details, ", ")) + sb.WriteString("\n") + } + } +} + +func (p *ActionPlan) compactDescription() string { + desc := make(map[string]map[string][]string) + + for _, cmd := range p.commands { + opType := cmd.OperationType() + itemType := cmd.ItemType() + detail := cmd.Detail() + + if _, ok := desc[opType]; !ok { + desc[opType] = make(map[string][]string) + } + + desc[opType][itemType] = append(desc[opType][itemType], detail) + } + + sb := strings.Builder{} + + // Enforce presentation order. + + describe("download", desc, &sb) + delete(desc, "download") + describe("enable", desc, &sb) + delete(desc, "enable") + describe("disable", desc, &sb) + delete(desc, "disable") + describe("remove", desc, &sb) + delete(desc, "remove") + + for optype := range desc { + describe(optype, desc, &sb) + } + + return sb.String() +} + +func (p *ActionPlan) Confirm(verbose bool) (bool, error) { + if !isatty.IsTerminal(os.Stdout.Fd()) && !isatty.IsCygwinTerminal(os.Stdout.Fd()) { + return true, nil + } + + fmt.Println("The following actions will be performed:\n" + p.Description(verbose)) + + var answer bool + + prompt := &survey.Confirm{ + Message: "Do you want to continue?", + Default: true, + } + + if err := survey.AskOne(prompt, &answer); err != nil { + return false, err + } + + fmt.Println() + + return answer, nil +} + +func (p *ActionPlan) Execute(ctx context.Context, confirm bool, dryRun bool, verbose bool) error { + var err error + + if len(p.commands) == 0 { + // XXX: show skipped commands, warnings? + fmt.Println("Nothing to do.") + return nil + } + + if dryRun { + fmt.Println("Action plan:\n" + p.Description(verbose)) + fmt.Println("Dry run, no action taken.") + + return nil + } + + if !confirm { + confirm, err = p.Confirm(verbose) + if err != nil { + return err + } + } + + if !confirm { + fmt.Println("Operation canceled.") + return nil + } + + for _, c := range p.commands { + if err := c.Run(ctx, p); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/hubops/purge.go b/pkg/hubops/purge.go new file mode 100644 index 00000000000..3b415b27428 --- /dev/null +++ b/pkg/hubops/purge.go @@ -0,0 +1,88 @@ +package hubops + +import ( + "context" + "fmt" + "os" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +// PurgeCommand removes the downloaded content of a hub item, effectively +// removing it from the local system. This command also removes the sub-items +// but not the associated data files. +type PurgeCommand struct { + Item *cwhub.Item + Force bool +} + +func NewPurgeCommand(item *cwhub.Item, force bool) *PurgeCommand { + return &PurgeCommand{Item: item, Force: force} +} + +func (c *PurgeCommand) Prepare(plan *ActionPlan) (bool, error) { + i := c.Item + + if i.State.IsLocal() { + // not downloaded, by definition + return false, nil + } + + if i.State.Tainted && !c.Force { + return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name) + } + + subsToRemove, err := i.SafeToRemoveDeps() + if err != nil { + return false, err + } + + for _, sub := range subsToRemove { + if err := plan.AddCommand(NewPurgeCommand(sub, c.Force)); err != nil { + return false, err + } + } + + if !i.State.Downloaded { + return false, nil + } + + return true, nil +} + +func (c *PurgeCommand) Run(ctx context.Context, plan *ActionPlan) error { + i := c.Item + + fmt.Println("purging " + colorizeItemName(i.FQName())) + + src, err := i.DownloadPath() + if err != nil { + return err + } + + if err := os.Remove(src); err != nil { + if os.IsNotExist(err) { + return nil + } + + return fmt.Errorf("while removing file: %w", err) + } + + i.State.Downloaded = false + i.State.Tainted = false + i.State.UpToDate = false + + return nil +} + +func (c *PurgeCommand) OperationType() string { + return "purge (delete source)" +} + +func (c *PurgeCommand) ItemType() string { + return c.Item.Type +} + +func (c *PurgeCommand) Detail() string { + return colorizeItemName(c.Item.Name) +} diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index bc9c8955d0d..d999d15ba6e 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -15,6 +15,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/hubops" "github.com/crowdsecurity/crowdsec/pkg/parser" ) @@ -224,7 +225,7 @@ func (t *HubTestItem) InstallHub() error { // install data for parsers if needed for _, item := range hub.GetInstalledByType(cwhub.PARSERS, true) { - if err := item.DownloadDataIfNeeded(ctx, true); err != nil { + if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", item.Name, err) } @@ -233,7 +234,7 @@ func (t *HubTestItem) InstallHub() error { // install data for scenarios if needed for _, item := range hub.GetInstalledByType(cwhub.SCENARIOS, true) { - if err := item.DownloadDataIfNeeded(ctx, true); err != nil { + if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", item.Name, err) } @@ -242,7 +243,7 @@ func (t *HubTestItem) InstallHub() error { // install data for postoverflows if needed for _, item := range hub.GetInstalledByType(cwhub.POSTOVERFLOWS, true) { - if err := item.DownloadDataIfNeeded(ctx, true); err != nil { + if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", item.Name, err) } diff --git a/pkg/setup/detect_test.go b/pkg/setup/detect_test.go index 553617032a4..475f3af0928 100644 --- a/pkg/setup/detect_test.go +++ b/pkg/setup/detect_test.go @@ -54,7 +54,7 @@ func TestSetupHelperProcess(t *testing.T) { } fmt.Fprint(os.Stdout, fakeSystemctlOutput) - os.Exit(0) //nolint:revive,deep-exit + os.Exit(0) //nolint:revive } func tempYAML(t *testing.T, content string) os.File { diff --git a/pkg/setup/install.go b/pkg/setup/install.go index d63a1ee1775..dcefe744a76 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -13,6 +13,7 @@ import ( "gopkg.in/yaml.v3" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/hubops" ) // AcquisDocument is created from a SetupItem. It represents a single YAML document, and can be part of a multi-document file. @@ -47,12 +48,14 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) { } // InstallHubItems installs the objects recommended in a setup file. -func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, dryRun bool) error { +func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dryRun, verbose bool) error { setupEnvelope, err := decodeSetup(input, false) if err != nil { return err } + plan := hubops.NewActionPlan(hub) + for _, setupItem := range setupEnvelope.Setup { forceAction := false downloadOnly := false @@ -68,70 +71,50 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, dryRun b return fmt.Errorf("collection %s not found", collection) } - if dryRun { - fmt.Println("dry-run: would install collection", collection) - - continue - } - - if err := item.Install(ctx, forceAction, downloadOnly); err != nil { - return fmt.Errorf("while installing collection %s: %w", item.Name, err) + plan.AddCommand(hubops.NewDownloadCommand(item, forceAction)) + if !downloadOnly { + plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) } } for _, parser := range setupItem.Install.Parsers { - if dryRun { - fmt.Println("dry-run: would install parser", parser) - - continue - } - item := hub.GetItem(cwhub.PARSERS, parser) if item == nil { return fmt.Errorf("parser %s not found", parser) } - if err := item.Install(ctx, forceAction, downloadOnly); err != nil { - return fmt.Errorf("while installing parser %s: %w", item.Name, err) + plan.AddCommand(hubops.NewDownloadCommand(item, forceAction)) + if !downloadOnly { + plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) } } for _, scenario := range setupItem.Install.Scenarios { - if dryRun { - fmt.Println("dry-run: would install scenario", scenario) - - continue - } - item := hub.GetItem(cwhub.SCENARIOS, scenario) if item == nil { return fmt.Errorf("scenario %s not found", scenario) } - if err := item.Install(ctx, forceAction, downloadOnly); err != nil { - return fmt.Errorf("while installing scenario %s: %w", item.Name, err) + plan.AddCommand(hubops.NewDownloadCommand(item, forceAction)) + if !downloadOnly { + plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) } } for _, postoverflow := range setupItem.Install.PostOverflows { - if dryRun { - fmt.Println("dry-run: would install postoverflow", postoverflow) - - continue - } - item := hub.GetItem(cwhub.POSTOVERFLOWS, postoverflow) if item == nil { return fmt.Errorf("postoverflow %s not found", postoverflow) } - if err := item.Install(ctx, forceAction, downloadOnly); err != nil { - return fmt.Errorf("while installing postoverflow %s: %w", item.Name, err) + plan.AddCommand(hubops.NewDownloadCommand(item, forceAction)) + if !downloadOnly { + plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) } } } - return nil + return plan.Execute(ctx, yes, dryRun, verbose) } // marshalAcquisDocuments creates the monolithic file, or itemized files (if a directory is provided) with the acquisition documents. diff --git a/test/bats/07_setup.bats b/test/bats/07_setup.bats index f832ac572d2..72a8b64a57a 100644 --- a/test/bats/07_setup.bats +++ b/test/bats/07_setup.bats @@ -511,8 +511,9 @@ update-notifier-motd.timer enabled enabled rune -0 jq -e '.installed == false' <(output) # we install it - rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/apache2"]}}]}' - assert_output 'dry-run: would install collection crowdsecurity/apache2' + rune -0 cscli setup install-hub /dev/stdin --dry-run --output raw <<< '{"setup":[{"install":{"collections":["crowdsecurity/apache2"]}}]}' + assert_line --regexp 'download collections:crowdsecurity/apache2' + assert_line --regexp 'enable collections:crowdsecurity/apache2' # still not installed rune -0 cscli collections inspect crowdsecurity/apache2 -o json @@ -520,8 +521,8 @@ update-notifier-motd.timer enabled enabled # same with dependencies rune -0 cscli collections remove --all - rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/linux"]}}]}' - assert_output 'dry-run: would install collection crowdsecurity/linux' + rune -0 cscli setup install-hub /dev/stdin --dry-run --output raw <<< '{"setup":[{"install":{"collections":["crowdsecurity/linux"]}}]}' + assert_line --regexp 'enable collections:crowdsecurity/linux' } @test "cscli setup install-hub (dry run: install multiple collections)" { @@ -530,8 +531,8 @@ update-notifier-motd.timer enabled enabled rune -0 jq -e '.installed == false' <(output) # we install it - rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/apache2"]}}]}' - assert_output 'dry-run: would install collection crowdsecurity/apache2' + rune -0 cscli setup install-hub /dev/stdin --dry-run --output raw <<< '{"setup":[{"install":{"collections":["crowdsecurity/apache2"]}}]}' + assert_line --regexp 'enable collections:crowdsecurity/apache2' # still not installed rune -0 cscli collections inspect crowdsecurity/apache2 -o json @@ -539,15 +540,15 @@ update-notifier-motd.timer enabled enabled } @test "cscli setup install-hub (dry run: install multiple collections, parsers, scenarios, postoverflows)" { - rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/aws-console","crowdsecurity/caddy"],"parsers":["crowdsecurity/asterisk-logs"],"scenarios":["crowdsecurity/smb-fs"],"postoverflows":["crowdsecurity/cdn-whitelist","crowdsecurity/rdns"]}}]}' - assert_line 'dry-run: would install collection crowdsecurity/aws-console' - assert_line 'dry-run: would install collection crowdsecurity/caddy' - assert_line 'dry-run: would install parser crowdsecurity/asterisk-logs' - assert_line 'dry-run: would install scenario crowdsecurity/smb-fs' - assert_line 'dry-run: would install postoverflow crowdsecurity/cdn-whitelist' - assert_line 'dry-run: would install postoverflow crowdsecurity/rdns' - - rune -1 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/foo"]}}]}' + rune -0 cscli setup install-hub /dev/stdin --dry-run --output raw <<< '{"setup":[{"install":{"collections":["crowdsecurity/aws-console","crowdsecurity/caddy"],"parsers":["crowdsecurity/asterisk-logs"],"scenarios":["crowdsecurity/smb-bf"],"postoverflows":["crowdsecurity/cdn-whitelist","crowdsecurity/rdns"]}}]}' + assert_line --regexp 'enable collections:crowdsecurity/aws-console' + assert_line --regexp 'enable collections:crowdsecurity/caddy' + assert_line --regexp 'enable parsers:crowdsecurity/asterisk-logs' + assert_line --regexp 'enable scenarios:crowdsecurity/smb-bf' + assert_line --regexp 'enable postoverflows:crowdsecurity/cdn-whitelist' + assert_line --regexp 'enable postoverflows:crowdsecurity/rdns' + + rune -1 cscli setup install-hub /dev/stdin --dry-run --output raw <<< '{"setup":[{"install":{"collections":["crowdsecurity/foo"]}}]}' assert_stderr --partial 'collection crowdsecurity/foo not found' } diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index b8fa1e9efca..03723ecc82b 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -20,7 +20,6 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_strip_index } teardown() { @@ -76,7 +75,7 @@ teardown() { assert_stderr --partial "invalid hub item appsec-rules:crowdsecurity/vpatch-laravel-debug-mode: latest version missing from index" rune -1 cscli appsec-rules install crowdsecurity/vpatch-laravel-debug-mode --force - assert_stderr --partial "error while installing 'crowdsecurity/vpatch-laravel-debug-mode': latest hash missing from index. The index file is invalid, please run 'cscli hub update' and try again" + assert_stderr --partial "appsec-rules:crowdsecurity/vpatch-laravel-debug-mode: latest hash missing from index. The index file is invalid, please run 'cscli hub update' and try again" } @test "missing reference in hub index" { @@ -108,47 +107,28 @@ teardown() { @test "cscli hub update" { rm -f "$INDEX_PATH" rune -0 cscli hub update - assert_stderr --partial "Wrote index to $INDEX_PATH" + assert_output "Downloading $INDEX_PATH" rune -0 cscli hub update - assert_stderr --partial "hub index is up to date" + assert_output "Nothing to do, the hub index is up to date." } -@test "cscli hub upgrade" { +@test "cscli hub upgrade (up to date)" { rune -0 cscli hub upgrade - assert_stderr --partial "Upgrading parsers" - assert_stderr --partial "Upgraded 0 parsers" - assert_stderr --partial "Upgrading postoverflows" - assert_stderr --partial "Upgraded 0 postoverflows" - assert_stderr --partial "Upgrading scenarios" - assert_stderr --partial "Upgraded 0 scenarios" - assert_stderr --partial "Upgrading contexts" - assert_stderr --partial "Upgraded 0 contexts" - assert_stderr --partial "Upgrading collections" - assert_stderr --partial "Upgraded 0 collections" - assert_stderr --partial "Upgrading appsec-configs" - assert_stderr --partial "Upgraded 0 appsec-configs" - assert_stderr --partial "Upgrading appsec-rules" - assert_stderr --partial "Upgraded 0 appsec-rules" - assert_stderr --partial "Upgrading collections" - assert_stderr --partial "Upgraded 0 collections" + refute_output rune -0 cscli parsers install crowdsecurity/syslog-logs - rune -0 cscli hub upgrade - assert_stderr --partial "crowdsecurity/syslog-logs: up-to-date" - rune -0 cscli hub upgrade --force - assert_stderr --partial "crowdsecurity/syslog-logs: up-to-date" - assert_stderr --partial "crowdsecurity/syslog-logs: updated" - assert_stderr --partial "Upgraded 1 parsers" - # this is used by the cron script to know if the hub was updated - assert_output --partial "updated crowdsecurity/syslog-logs" + refute_output + skip "todo: data files are re-downloaded with --force" } @test "cscli hub upgrade (with local items)" { mkdir -p "$CONFIG_DIR/collections" touch "$CONFIG_DIR/collections/foo.yaml" rune -0 cscli hub upgrade - assert_stderr --partial "not upgrading foo.yaml: local item" + assert_output - <<-EOT + collections:foo.yaml - not downloading local item + EOT } @test "cscli hub types" { diff --git a/test/bats/20_hub_collections.bats b/test/bats/20_hub_collections.bats deleted file mode 100644 index 6822339ae40..00000000000 --- a/test/bats/20_hub_collections.bats +++ /dev/null @@ -1,381 +0,0 @@ -#!/usr/bin/env bats -# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: - -set -u - -setup_file() { - load "../lib/setup_file.sh" - ./instance-data load - HUB_DIR=$(config_get '.config_paths.hub_dir') - export HUB_DIR - INDEX_PATH=$(config_get '.config_paths.index_path') - export INDEX_PATH - CONFIG_DIR=$(config_get '.config_paths.config_dir') - export CONFIG_DIR -} - -teardown_file() { - load "../lib/teardown_file.sh" -} - -setup() { - load "../lib/setup.sh" - load "../lib/bats-file/load.bash" - ./instance-data load - hub_strip_index -} - -teardown() { - ./instance-crowdsec stop -} - -#---------- - -@test "cscli collections list" { - hub_purge_all - - # no items - rune -0 cscli collections list - assert_output --partial "COLLECTIONS" - rune -0 cscli collections list -o json - assert_json '{collections:[]}' - rune -0 cscli collections list -o raw - assert_output 'name,status,version,description' - - # some items - rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb - - rune -0 cscli collections list - assert_output --partial crowdsecurity/sshd - assert_output --partial crowdsecurity/smb - rune -0 grep -c enabled <(output) - assert_output "2" - - rune -0 cscli collections list -o json - assert_output --partial crowdsecurity/sshd - assert_output --partial crowdsecurity/smb - rune -0 jq '.collections | length' <(output) - assert_output "2" - - rune -0 cscli collections list -o raw - assert_output --partial crowdsecurity/sshd - assert_output --partial crowdsecurity/smb - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "2" -} - -@test "cscli collections list -a" { - expected=$(jq <"$INDEX_PATH" -r '.collections | length') - - rune -0 cscli collections list -a - rune -0 grep -c disabled <(output) - assert_output "$expected" - - rune -0 cscli collections list -o json -a - rune -0 jq '.collections | length' <(output) - assert_output "$expected" - - rune -0 cscli collections list -o raw -a - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "$expected" - - # the list should be the same in all formats, and sorted (not case sensitive) - - list_raw=$(cscli collections list -o raw -a | tail -n +2 | cut -d, -f1) - list_human=$(cscli collections list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) - list_json=$(cscli collections list -o json -a | jq -r '.collections[].name') - - rune -0 sort -f <<<"$list_raw" - assert_output "$list_raw" - - assert_equal "$list_raw" "$list_json" - assert_equal "$list_raw" "$list_human" -} - -@test "cscli collections list [collection]..." { - # non-existent - rune -1 cscli collections install foo/bar - assert_stderr --partial "can't find 'foo/bar' in collections" - - # not installed - rune -0 cscli collections list crowdsecurity/smb - assert_output --regexp 'crowdsecurity/smb.*disabled' - - # install two items - rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb - - # list an installed item - rune -0 cscli collections list crowdsecurity/sshd - assert_output --regexp "crowdsecurity/sshd" - refute_output --partial "crowdsecurity/smb" - - # list multiple installed and non installed items - rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb crowdsecurity/nginx - assert_output --partial "crowdsecurity/sshd" - assert_output --partial "crowdsecurity/smb" - assert_output --partial "crowdsecurity/nginx" - - rune -0 cscli collections list crowdsecurity/sshd -o json - rune -0 jq '.collections | length' <(output) - assert_output "1" - rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb crowdsecurity/nginx -o json - rune -0 jq '.collections | length' <(output) - assert_output "3" - - rune -0 cscli collections list crowdsecurity/sshd -o raw - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "1" - rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb -o raw - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "2" -} - -@test "cscli collections install" { - rune -1 cscli collections install - assert_stderr --partial 'requires at least 1 arg(s), only received 0' - - # not in hub - rune -1 cscli collections install crowdsecurity/blahblah - assert_stderr --partial "can't find 'crowdsecurity/blahblah' in collections" - - # simple install - rune -0 cscli collections install crowdsecurity/sshd - rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics - assert_output --partial 'crowdsecurity/sshd' - assert_output --partial 'installed: true' - - # autocorrect - rune -1 cscli collections install crowdsecurity/ssshd - assert_stderr --partial "can't find 'crowdsecurity/ssshd' in collections, did you mean 'crowdsecurity/sshd'?" - - # install multiple - rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb - rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics - assert_output --partial 'crowdsecurity/sshd' - assert_output --partial 'installed: true' - rune -0 cscli collections inspect crowdsecurity/smb --no-metrics - assert_output --partial 'crowdsecurity/smb' - assert_output --partial 'installed: true' -} - -@test "cscli collections install (file location and download-only)" { - rune -0 cscli collections install crowdsecurity/linux --download-only - rune -0 cscli collections inspect crowdsecurity/linux --no-metrics - assert_output --partial 'crowdsecurity/linux' - assert_output --partial 'installed: false' - assert_file_exists "$HUB_DIR/collections/crowdsecurity/linux.yaml" - assert_file_not_exists "$CONFIG_DIR/collections/linux.yaml" - - rune -0 cscli collections install crowdsecurity/linux - rune -0 cscli collections inspect crowdsecurity/linux --no-metrics - assert_output --partial 'installed: true' - assert_file_exists "$CONFIG_DIR/collections/linux.yaml" -} - -@test "cscli collections install --force (tainted)" { - rune -0 cscli collections install crowdsecurity/sshd - echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" - - rune -1 cscli collections install crowdsecurity/sshd - assert_stderr --partial "error while installing 'crowdsecurity/sshd': while enabling crowdsecurity/sshd: crowdsecurity/sshd is tainted, won't overwrite unless --force" - - rune -0 cscli collections install crowdsecurity/sshd --force - assert_stderr --partial "Enabled crowdsecurity/sshd" -} - -@test "cscli collections install --ignore (skip on errors)" { - rune -1 cscli collections install foo/bar crowdsecurity/sshd - assert_stderr --partial "can't find 'foo/bar' in collections" - refute_stderr --partial "Enabled collections: crowdsecurity/sshd" - - rune -0 cscli collections install foo/bar crowdsecurity/sshd --ignore - assert_stderr --partial "can't find 'foo/bar' in collections" - assert_stderr --partial "Enabled collections: crowdsecurity/sshd" -} - -@test "cscli collections inspect" { - rune -1 cscli collections inspect - assert_stderr --partial 'requires at least 1 arg(s), only received 0' - # required for metrics - ./instance-crowdsec start - - rune -1 cscli collections inspect blahblah/blahblah - assert_stderr --partial "can't find 'blahblah/blahblah' in collections" - - # one item - rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics - assert_line 'type: collections' - assert_line 'name: crowdsecurity/sshd' - assert_line 'author: crowdsecurity' - assert_line 'path: collections/crowdsecurity/sshd.yaml' - assert_line 'installed: false' - refute_line --partial 'Current metrics:' - - # one item, with metrics - rune -0 cscli collections inspect crowdsecurity/sshd - assert_line --partial 'Current metrics:' - - # one item, json - rune -0 cscli collections inspect crowdsecurity/sshd -o json - rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output) - assert_json '["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",false]' - - # one item, raw - rune -0 cscli collections inspect crowdsecurity/sshd -o raw - assert_line 'type: collections' - assert_line 'name: crowdsecurity/sshd' - assert_line 'author: crowdsecurity' - assert_line 'path: collections/crowdsecurity/sshd.yaml' - assert_line 'installed: false' - refute_line --partial 'Current metrics:' - - # multiple items - rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb --no-metrics - assert_output --partial 'crowdsecurity/sshd' - assert_output --partial 'crowdsecurity/smb' - rune -1 grep -c 'Current metrics:' <(output) - assert_output "0" - - # multiple items, with metrics - rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb - rune -0 grep -c 'Current metrics:' <(output) - assert_output "2" - - # multiple items, json - rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o json - rune -0 jq -sc '[.[] | [.type, .name, .author, .path, .installed]]' <(output) - assert_json '[["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",false],["collections","crowdsecurity/smb","crowdsecurity","collections/crowdsecurity/smb.yaml",false]]' - - # multiple items, raw - rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o raw - assert_output --partial 'crowdsecurity/sshd' - assert_output --partial 'crowdsecurity/smb' - rune -1 grep -c 'Current metrics:' <(output) - assert_output "0" -} - -@test "cscli collections remove" { - rune -1 cscli collections remove - assert_stderr --partial "specify at least one collection to remove or '--all'" - rune -1 cscli collections remove blahblah/blahblah - assert_stderr --partial "can't find 'blahblah/blahblah' in collections" - - rune -0 cscli collections install crowdsecurity/sshd --download-only - rune -0 cscli collections remove crowdsecurity/sshd - assert_stderr --partial 'removing crowdsecurity/sshd: not installed -- no need to remove' - - rune -0 cscli collections install crowdsecurity/sshd - rune -0 cscli collections remove crowdsecurity/sshd - assert_stderr --partial 'Removed crowdsecurity/sshd' - - rune -0 cscli collections remove crowdsecurity/sshd --purge - assert_stderr --partial 'Removed source file [crowdsecurity/sshd]' - - rune -0 cscli collections remove crowdsecurity/sshd - assert_stderr --partial 'removing crowdsecurity/sshd: not installed -- no need to remove' - - rune -0 cscli collections remove crowdsecurity/sshd --purge --debug - assert_stderr --partial 'removing crowdsecurity/sshd: not downloaded -- no need to remove' - refute_stderr --partial 'Removed source file [crowdsecurity/sshd]' - - # install, then remove, check files - rune -0 cscli collections install crowdsecurity/sshd - assert_file_exists "$CONFIG_DIR/collections/sshd.yaml" - rune -0 cscli collections remove crowdsecurity/sshd - assert_file_not_exists "$CONFIG_DIR/collections/sshd.yaml" - - # delete is an alias for remove - rune -0 cscli collections install crowdsecurity/sshd - assert_file_exists "$CONFIG_DIR/collections/sshd.yaml" - rune -0 cscli collections delete crowdsecurity/sshd - assert_file_not_exists "$CONFIG_DIR/collections/sshd.yaml" - - # purge - assert_file_exists "$HUB_DIR/collections/crowdsecurity/sshd.yaml" - rune -0 cscli collections remove crowdsecurity/sshd --purge - assert_file_not_exists "$HUB_DIR/collections/crowdsecurity/sshd.yaml" - - rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb - - # --all - rune -0 cscli collections list -o raw - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "2" - - rune -0 cscli collections remove --all - - rune -0 cscli collections list -o raw - rune -1 grep -vc 'name,status,version,description' <(output) - assert_output "0" -} - -@test "cscli collections remove --force" { - # remove a collections that belongs to a collection - rune -0 cscli collections install crowdsecurity/linux - rune -0 cscli collections remove crowdsecurity/sshd - assert_stderr --partial "crowdsecurity/sshd belongs to collections: [crowdsecurity/linux]" - assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this collection" -} - -@test "cscli collections upgrade" { - rune -1 cscli collections upgrade - assert_stderr --partial "specify at least one collection to upgrade or '--all'" - rune -1 cscli collections upgrade blahblah/blahblah - assert_stderr --partial "can't find 'blahblah/blahblah' in collections" - rune -0 cscli collections remove crowdsecurity/exim --purge - rune -1 cscli collections upgrade crowdsecurity/exim - assert_stderr --partial "can't upgrade crowdsecurity/exim: not installed" - rune -0 cscli collections install crowdsecurity/exim --download-only - rune -1 cscli collections upgrade crowdsecurity/exim - assert_stderr --partial "can't upgrade crowdsecurity/exim: downloaded but not installed" - - # hash of the string "v0.0" - sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" - - # add version 0.0 to all collections - new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.collections |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') - echo "$new_hub" >"$INDEX_PATH" - - rune -0 cscli collections install crowdsecurity/sshd - - echo "v0.0" > "$CONFIG_DIR/collections/sshd.yaml" - rune -0 cscli collections inspect crowdsecurity/sshd -o json - rune -0 jq -e '.local_version=="0.0"' <(output) - - # upgrade - rune -0 cscli collections upgrade crowdsecurity/sshd - rune -0 cscli collections inspect crowdsecurity/sshd -o json - rune -0 jq -e '.local_version==.version' <(output) - - # taint - echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" - # XXX: should return error - rune -0 cscli collections upgrade crowdsecurity/sshd - assert_stderr --partial "crowdsecurity/sshd is tainted, --force to overwrite" - rune -0 cscli collections inspect crowdsecurity/sshd -o json - rune -0 jq -e '.local_version=="?"' <(output) - - # force upgrade with taint - rune -0 cscli collections upgrade crowdsecurity/sshd --force - rune -0 cscli collections inspect crowdsecurity/sshd -o json - rune -0 jq -e '.local_version==.version' <(output) - - # multiple items - rune -0 cscli collections install crowdsecurity/smb - echo "v0.0" >"$CONFIG_DIR/collections/sshd.yaml" - echo "v0.0" >"$CONFIG_DIR/collections/smb.yaml" - rune -0 cscli collections list -o json - rune -0 jq -e '[.collections[].local_version]==["0.0","0.0"]' <(output) - rune -0 cscli collections upgrade crowdsecurity/sshd crowdsecurity/smb - rune -0 cscli collections list -o json - rune -0 jq -e 'any(.collections[].local_version; .=="0.0") | not' <(output) - - # upgrade all - echo "v0.0" >"$CONFIG_DIR/collections/sshd.yaml" - echo "v0.0" >"$CONFIG_DIR/collections/smb.yaml" - rune -0 cscli collections list -o json - rune -0 jq -e '[.collections[].local_version]==["0.0","0.0"]' <(output) - rune -0 cscli collections upgrade --all - rune -0 cscli collections list -o json - rune -0 jq -e 'any(.collections[].local_version; .=="0.0") | not' <(output) -} diff --git a/test/bats/20_hub_collections_dep.bats b/test/bats/20_hub_collections_dep.bats index 673b812dc0d..94a984709a8 100644 --- a/test/bats/20_hub_collections_dep.bats +++ b/test/bats/20_hub_collections_dep.bats @@ -20,7 +20,6 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_strip_index } teardown() { @@ -84,18 +83,32 @@ teardown() { assert_stderr --partial "crowdsecurity/smb is tainted, use '--force' to remove" } +@test "cscli collections inspect (dependencies)" { + rune -0 cscli collections install crowdsecurity/smb + + # The inspect command must show the dependencies of the local or older version. + echo "{'collections': ['crowdsecurity/sshd']}" >"$CONFIG_DIR/collections/smb.yaml" + + rune -0 cscli collections inspect crowdsecurity/smb --no-metrics -o json + rune -0 jq -e '.collections' <(output) + assert_json '["crowdsecurity/sshd"]' +} + @test "cscli collections (dependencies II: the revenge)" { rune -0 cscli collections install crowdsecurity/wireguard baudneo/gotify rune -0 cscli collections remove crowdsecurity/wireguard - assert_stderr --partial "crowdsecurity/syslog-logs was not removed because it also belongs to baudneo/gotify" + assert_output --regexp 'disabling collections:crowdsecurity/wireguard' + refute_output --regexp 'disabling parsers:crowdsecurity/syslog-logs' rune -0 cscli collections inspect crowdsecurity/wireguard -o json rune -0 jq -e '.installed==false' <(output) + rune -0 cscli parsers inspect crowdsecurity/syslog-logs -o json + rune -0 jq -e '.installed==true' <(output) } @test "cscli collections (dependencies III: origins)" { # it is perfectly fine to remove an item belonging to a collection that we are removing anyway - # inject a dependency: sshd requires the syslog-logs parsers, but linux does too + # inject a direct dependency: sshd requires the syslog-logs parsers, but linux does too hub_dep=$(jq <"$INDEX_PATH" '. * {collections:{"crowdsecurity/sshd":{parsers:["crowdsecurity/syslog-logs"]}}}') echo "$hub_dep" >"$INDEX_PATH" @@ -108,11 +121,8 @@ teardown() { # removing linux should remove syslog-logs even though sshd depends on it rune -0 cscli collections remove crowdsecurity/linux - refute_stderr --partial "crowdsecurity/syslog-logs was not removed" - # we must also consider indirect dependencies - refute_stderr --partial "crowdsecurity/ssh-bf was not removed" - rune -0 cscli parsers list -o json - rune -0 jq -e '.parsers | length == 0' <(output) + rune -0 cscli hub list -o json + rune -0 jq -e 'add | length == 0' <(output) } @test "cscli collections (dependencies IV: looper)" { diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats index d29a7d2c14c..2f1c952848b 100644 --- a/test/bats/20_hub_items.bats +++ b/test/bats/20_hub_items.bats @@ -22,7 +22,6 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_strip_index } teardown() { @@ -82,7 +81,7 @@ teardown() { rune -0 cscli collections install crowdsecurity/sshd rune -1 cscli collections inspect crowdsecurity/sshd --no-metrics # XXX: we are on the verbose side here... - assert_stderr --regexp "Error: failed to read Hub index: failed to sync hub items: failed to scan .*: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version. Run 'sudo cscli hub update' to download the index again" + assert_stderr "Error: failed to read hub index: failed to sync hub items: failed to scan $CONFIG_DIR: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version. Run 'sudo cscli hub update' to download the index again" } @test "removing or purging an item already removed by hand" { @@ -91,19 +90,21 @@ teardown() { rune -0 jq -r '.local_path' <(output) rune -0 rm "$(output)" - rune -0 cscli parsers remove crowdsecurity/syslog-logs --debug - assert_stderr --partial "removing crowdsecurity/syslog-logs: not installed -- no need to remove" + rune -0 cscli parsers remove crowdsecurity/syslog-logs + assert_output "Nothing to do." rune -0 cscli parsers inspect crowdsecurity/syslog-logs -o json rune -0 jq -r '.path' <(output) rune -0 rm "$HUB_DIR/$(output)" - rune -0 cscli parsers remove crowdsecurity/syslog-logs --purge --debug - assert_stderr --partial "removing crowdsecurity/syslog-logs: not downloaded -- no need to remove" + rune -0 cscli parsers remove crowdsecurity/syslog-logs --purge + assert_output "Nothing to do." - rune -0 cscli parsers remove crowdsecurity/linux --all --error --purge --force - rune -0 cscli collections remove crowdsecurity/linux --all --error --purge --force - refute_output + rune -0 cscli parsers remove --all --error --purge --force + assert_output "Nothing to do." + refute_stderr + rune -0 cscli collections remove --all --error --purge --force + assert_output "Nothing to do." refute_stderr } @@ -121,7 +122,7 @@ teardown() { # and not from hub update rune -0 cscli hub update - assert_stderr --partial "collection crowdsecurity/sshd is tainted" + assert_stderr --partial "collection crowdsecurity/sshd is tainted by local changes" refute_stderr --partial "collection foobar.yaml is tainted" } @@ -150,25 +151,42 @@ teardown() { @test "a local item cannot be downloaded by cscli" { rune -0 mkdir -p "$CONFIG_DIR/collections" rune -0 touch "$CONFIG_DIR/collections/foobar.yaml" - rune -1 cscli collections install foobar.yaml - assert_stderr --partial "foobar.yaml is local, can't download" - rune -1 cscli collections install foobar.yaml --force - assert_stderr --partial "foobar.yaml is local, can't download" + rune -0 cscli collections install foobar.yaml + assert_output --partial "Nothing to do." + rune -0 cscli collections install foobar.yaml --force + assert_output --partial "Nothing to do." + rune -0 cscli collections install --download-only foobar.yaml + assert_output --partial "Nothing to do." } @test "a local item cannot be removed by cscli" { - rune -0 mkdir -p "$CONFIG_DIR/collections" - rune -0 touch "$CONFIG_DIR/collections/foobar.yaml" - rune -0 cscli collections remove foobar.yaml - assert_stderr --partial "foobar.yaml is a local item, please delete manually" - rune -0 cscli collections remove foobar.yaml --purge - assert_stderr --partial "foobar.yaml is a local item, please delete manually" - rune -0 cscli collections remove foobar.yaml --force - assert_stderr --partial "foobar.yaml is a local item, please delete manually" - rune -0 cscli collections remove --all - assert_stderr --partial "foobar.yaml is a local item, please delete manually" - rune -0 cscli collections remove --all --purge - assert_stderr --partial "foobar.yaml is a local item, please delete manually" + rune -0 mkdir -p "$CONFIG_DIR/scenarios" + rune -0 touch "$CONFIG_DIR/scenarios/foobar.yaml" + rune -0 cscli scenarios remove foobar.yaml + assert_output - <<-EOT + WARN scenarios:foobar.yaml is a local item, please delete manually + Nothing to do. + EOT + rune -0 cscli scenarios remove foobar.yaml --purge + assert_output - <<-EOT + WARN scenarios:foobar.yaml is a local item, please delete manually + Nothing to do. + EOT + rune -0 cscli scenarios remove foobar.yaml --force + assert_output - <<-EOT + WARN scenarios:foobar.yaml is a local item, please delete manually + Nothing to do. + EOT + + rune -0 cscli scenarios install crowdsecurity/ssh-bf + + rune -0 cscli scenarios remove --all + assert_line "WARN scenarios:foobar.yaml is a local item, please delete manually" + assert_line "disabling scenarios:crowdsecurity/ssh-bf" + + rune -0 cscli scenarios remove --all --purge + assert_line "WARN scenarios:foobar.yaml is a local item, please delete manually" + assert_line "purging scenarios:crowdsecurity/ssh-bf" } @test "a dangling link is reported with a warning" { diff --git a/test/bats/20_hub_parsers.bats b/test/bats/20_hub_parsers.bats deleted file mode 100644 index 791b1a2177f..00000000000 --- a/test/bats/20_hub_parsers.bats +++ /dev/null @@ -1,383 +0,0 @@ -#!/usr/bin/env bats -# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: - -set -u - -setup_file() { - load "../lib/setup_file.sh" - ./instance-data load - HUB_DIR=$(config_get '.config_paths.hub_dir') - export HUB_DIR - INDEX_PATH=$(config_get '.config_paths.index_path') - export INDEX_PATH - CONFIG_DIR=$(config_get '.config_paths.config_dir') - export CONFIG_DIR -} - -teardown_file() { - load "../lib/teardown_file.sh" -} - -setup() { - load "../lib/setup.sh" - load "../lib/bats-file/load.bash" - ./instance-data load - hub_strip_index -} - -teardown() { - ./instance-crowdsec stop -} - -#---------- - -@test "cscli parsers list" { - hub_purge_all - - # no items - rune -0 cscli parsers list - assert_output --partial "PARSERS" - rune -0 cscli parsers list -o json - assert_json '{parsers:[]}' - rune -0 cscli parsers list -o raw - assert_output 'name,status,version,description' - - # some items - rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth - - rune -0 cscli parsers list - assert_output --partial crowdsecurity/whitelists - assert_output --partial crowdsecurity/windows-auth - rune -0 grep -c enabled <(output) - assert_output "2" - - rune -0 cscli parsers list -o json - assert_output --partial crowdsecurity/whitelists - assert_output --partial crowdsecurity/windows-auth - rune -0 jq '.parsers | length' <(output) - assert_output "2" - - rune -0 cscli parsers list -o raw - assert_output --partial crowdsecurity/whitelists - assert_output --partial crowdsecurity/windows-auth - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "2" -} - -@test "cscli parsers list -a" { - expected=$(jq <"$INDEX_PATH" -r '.parsers | length') - - rune -0 cscli parsers list -a - rune -0 grep -c disabled <(output) - assert_output "$expected" - - rune -0 cscli parsers list -o json -a - rune -0 jq '.parsers | length' <(output) - assert_output "$expected" - - rune -0 cscli parsers list -o raw -a - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "$expected" - - # the list should be the same in all formats, and sorted (not case sensitive) - - list_raw=$(cscli parsers list -o raw -a | tail -n +2 | cut -d, -f1) - list_human=$(cscli parsers list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) - list_json=$(cscli parsers list -o json -a | jq -r '.parsers[].name') - - rune -0 sort -f <<<"$list_raw" - assert_output "$list_raw" - - assert_equal "$list_raw" "$list_json" - assert_equal "$list_raw" "$list_human" -} - -@test "cscli parsers list [parser]..." { - # non-existent - rune -1 cscli parsers install foo/bar - assert_stderr --partial "can't find 'foo/bar' in parsers" - - # not installed - rune -0 cscli parsers list crowdsecurity/whitelists - assert_output --regexp 'crowdsecurity/whitelists.*disabled' - - # install two items - rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth - - # list an installed item - rune -0 cscli parsers list crowdsecurity/whitelists - assert_output --regexp "crowdsecurity/whitelists.*enabled" - refute_output --partial "crowdsecurity/windows-auth" - - # list multiple installed and non installed items - rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs - assert_output --partial "crowdsecurity/whitelists" - assert_output --partial "crowdsecurity/windows-auth" - assert_output --partial "crowdsecurity/traefik-logs" - - rune -0 cscli parsers list crowdsecurity/whitelists -o json - rune -0 jq '.parsers | length' <(output) - assert_output "1" - rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o json - rune -0 jq '.parsers | length' <(output) - assert_output "3" - - rune -0 cscli parsers list crowdsecurity/whitelists -o raw - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "1" - rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o raw - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "3" -} - -@test "cscli parsers install" { - rune -1 cscli parsers install - assert_stderr --partial 'requires at least 1 arg(s), only received 0' - - # not in hub - rune -1 cscli parsers install crowdsecurity/blahblah - assert_stderr --partial "can't find 'crowdsecurity/blahblah' in parsers" - - # simple install - rune -0 cscli parsers install crowdsecurity/whitelists - rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics - assert_output --partial 'crowdsecurity/whitelists' - assert_output --partial 'installed: true' - - # autocorrect - rune -1 cscli parsers install crowdsecurity/sshd-logz - assert_stderr --partial "can't find 'crowdsecurity/sshd-logz' in parsers, did you mean 'crowdsecurity/sshd-logs'?" - - # install multiple - rune -0 cscli parsers install crowdsecurity/pgsql-logs crowdsecurity/postfix-logs - rune -0 cscli parsers inspect crowdsecurity/pgsql-logs --no-metrics - assert_output --partial 'crowdsecurity/pgsql-logs' - assert_output --partial 'installed: true' - rune -0 cscli parsers inspect crowdsecurity/postfix-logs --no-metrics - assert_output --partial 'crowdsecurity/postfix-logs' - assert_output --partial 'installed: true' -} - -@test "cscli parsers install (file location and download-only)" { - rune -0 cscli parsers install crowdsecurity/whitelists --download-only - rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics - assert_output --partial 'crowdsecurity/whitelists' - assert_output --partial 'installed: false' - assert_file_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" - assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" - - rune -0 cscli parsers install crowdsecurity/whitelists - rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics - assert_output --partial 'installed: true' - assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" -} - -@test "cscli parsers install --force (tainted)" { - rune -0 cscli parsers install crowdsecurity/whitelists - echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" - - rune -1 cscli parsers install crowdsecurity/whitelists - assert_stderr --partial "error while installing 'crowdsecurity/whitelists': while enabling crowdsecurity/whitelists: crowdsecurity/whitelists is tainted, won't overwrite unless --force" - - rune -0 cscli parsers install crowdsecurity/whitelists --force - assert_stderr --partial "Enabled crowdsecurity/whitelists" -} - -@test "cscli parsers install --ignore (skip on errors)" { - rune -1 cscli parsers install foo/bar crowdsecurity/whitelists - assert_stderr --partial "can't find 'foo/bar' in parsers" - refute_stderr --partial "Enabled parsers: crowdsecurity/whitelists" - - rune -0 cscli parsers install foo/bar crowdsecurity/whitelists --ignore - assert_stderr --partial "can't find 'foo/bar' in parsers" - assert_stderr --partial "Enabled parsers: crowdsecurity/whitelists" -} - -@test "cscli parsers inspect" { - rune -1 cscli parsers inspect - assert_stderr --partial 'requires at least 1 arg(s), only received 0' - # required for metrics - ./instance-crowdsec start - - rune -1 cscli parsers inspect blahblah/blahblah - assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" - - # one item - rune -0 cscli parsers inspect crowdsecurity/sshd-logs --no-metrics - assert_line 'type: parsers' - assert_line 'stage: s01-parse' - assert_line 'name: crowdsecurity/sshd-logs' - assert_line 'author: crowdsecurity' - assert_line 'path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml' - assert_line 'installed: false' - refute_line --partial 'Current metrics:' - - # one item, with metrics - rune -0 cscli parsers inspect crowdsecurity/sshd-logs - assert_line --partial 'Current metrics:' - - # one item, json - rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o json - rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output) - assert_json '["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",false]' - - # one item, raw - rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o raw - assert_line 'type: parsers' - assert_line 'name: crowdsecurity/sshd-logs' - assert_line 'stage: s01-parse' - assert_line 'author: crowdsecurity' - assert_line 'path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml' - assert_line 'installed: false' - refute_line --partial 'Current metrics:' - - # multiple items - rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists --no-metrics - assert_output --partial 'crowdsecurity/sshd-logs' - assert_output --partial 'crowdsecurity/whitelists' - rune -1 grep -c 'Current metrics:' <(output) - assert_output "0" - - # multiple items, with metrics - rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists - rune -0 grep -c 'Current metrics:' <(output) - assert_output "2" - - # multiple items, json - rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o json - rune -0 jq -sc '[.[] | [.type, .stage, .name, .author, .path, .installed]]' <(output) - assert_json '[["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",false],["parsers","s02-enrich","crowdsecurity/whitelists","crowdsecurity","parsers/s02-enrich/crowdsecurity/whitelists.yaml",false]]' - - # multiple items, raw - rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o raw - assert_output --partial 'crowdsecurity/sshd-logs' - assert_output --partial 'crowdsecurity/whitelists' - rune -1 grep -c 'Current metrics:' <(output) - assert_output "0" -} - -@test "cscli parsers remove" { - rune -1 cscli parsers remove - assert_stderr --partial "specify at least one parser to remove or '--all'" - rune -1 cscli parsers remove blahblah/blahblah - assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" - - rune -0 cscli parsers install crowdsecurity/whitelists --download-only - rune -0 cscli parsers remove crowdsecurity/whitelists - assert_stderr --partial "removing crowdsecurity/whitelists: not installed -- no need to remove" - - rune -0 cscli parsers install crowdsecurity/whitelists - rune -0 cscli parsers remove crowdsecurity/whitelists - assert_stderr --partial "Removed crowdsecurity/whitelists" - - rune -0 cscli parsers remove crowdsecurity/whitelists --purge - assert_stderr --partial 'Removed source file [crowdsecurity/whitelists]' - - rune -0 cscli parsers remove crowdsecurity/whitelists - assert_stderr --partial "removing crowdsecurity/whitelists: not installed -- no need to remove" - - rune -0 cscli parsers remove crowdsecurity/whitelists --purge --debug - assert_stderr --partial 'removing crowdsecurity/whitelists: not downloaded -- no need to remove' - refute_stderr --partial 'Removed source file [crowdsecurity/whitelists]' - - # install, then remove, check files - rune -0 cscli parsers install crowdsecurity/whitelists - assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" - rune -0 cscli parsers remove crowdsecurity/whitelists - assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" - - # delete is an alias for remove - rune -0 cscli parsers install crowdsecurity/whitelists - assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" - rune -0 cscli parsers delete crowdsecurity/whitelists - assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" - - # purge - assert_file_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" - rune -0 cscli parsers remove crowdsecurity/whitelists --purge - assert_file_not_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" - - rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth - - # --all - rune -0 cscli parsers list -o raw - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "2" - - rune -0 cscli parsers remove --all - - rune -0 cscli parsers list -o raw - rune -1 grep -vc 'name,status,version,description' <(output) - assert_output "0" -} - -@test "cscli parsers remove --force" { - # remove a parser that belongs to a collection - rune -0 cscli collections install crowdsecurity/sshd - rune -0 cscli parsers remove crowdsecurity/sshd-logs - assert_stderr --partial "crowdsecurity/sshd-logs belongs to collections: [crowdsecurity/sshd]" - assert_stderr --partial "Run 'sudo cscli parsers remove crowdsecurity/sshd-logs --force' if you want to force remove this parser" -} - -@test "cscli parsers upgrade" { - rune -1 cscli parsers upgrade - assert_stderr --partial "specify at least one parser to upgrade or '--all'" - rune -1 cscli parsers upgrade blahblah/blahblah - assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" - rune -0 cscli parsers remove crowdsecurity/pam-logs --purge - rune -1 cscli parsers upgrade crowdsecurity/pam-logs - assert_stderr --partial "can't upgrade crowdsecurity/pam-logs: not installed" - rune -0 cscli parsers install crowdsecurity/pam-logs --download-only - rune -1 cscli parsers upgrade crowdsecurity/pam-logs - assert_stderr --partial "can't upgrade crowdsecurity/pam-logs: downloaded but not installed" - - # hash of the string "v0.0" - sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" - - # add version 0.0 to all parsers - new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.parsers |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') - echo "$new_hub" >"$INDEX_PATH" - - rune -0 cscli parsers install crowdsecurity/whitelists - - echo "v0.0" > "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" - rune -0 cscli parsers inspect crowdsecurity/whitelists -o json - rune -0 jq -e '.local_version=="0.0"' <(output) - - # upgrade - rune -0 cscli parsers upgrade crowdsecurity/whitelists - rune -0 cscli parsers inspect crowdsecurity/whitelists -o json - rune -0 jq -e '.local_version==.version' <(output) - - # taint - echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" - # XXX: should return error - rune -0 cscli parsers upgrade crowdsecurity/whitelists - assert_stderr --partial "crowdsecurity/whitelists is tainted, --force to overwrite" - rune -0 cscli parsers inspect crowdsecurity/whitelists -o json - rune -0 jq -e '.local_version=="?"' <(output) - - # force upgrade with taint - rune -0 cscli parsers upgrade crowdsecurity/whitelists --force - rune -0 cscli parsers inspect crowdsecurity/whitelists -o json - rune -0 jq -e '.local_version==.version' <(output) - - # multiple items - rune -0 cscli parsers install crowdsecurity/windows-auth - echo "v0.0" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" - echo "v0.0" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml" - rune -0 cscli parsers list -o json - rune -0 jq -e '[.parsers[].local_version]==["0.0","0.0"]' <(output) - rune -0 cscli parsers upgrade crowdsecurity/whitelists crowdsecurity/windows-auth - rune -0 cscli parsers list -o json - rune -0 jq -e 'any(.parsers[].local_version; .=="0.0") | not' <(output) - - # upgrade all - echo "v0.0" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" - echo "v0.0" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml" - rune -0 cscli parsers list -o json - rune -0 jq -e '[.parsers[].local_version]==["0.0","0.0"]' <(output) - rune -0 cscli parsers upgrade --all - rune -0 cscli parsers list -o json - rune -0 jq -e 'any(.parsers[].local_version; .=="0.0") | not' <(output) -} diff --git a/test/bats/20_hub_postoverflows.bats b/test/bats/20_hub_postoverflows.bats deleted file mode 100644 index 37337b08caa..00000000000 --- a/test/bats/20_hub_postoverflows.bats +++ /dev/null @@ -1,383 +0,0 @@ -#!/usr/bin/env bats -# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: - -set -u - -setup_file() { - load "../lib/setup_file.sh" - ./instance-data load - HUB_DIR=$(config_get '.config_paths.hub_dir') - export HUB_DIR - INDEX_PATH=$(config_get '.config_paths.index_path') - export INDEX_PATH - CONFIG_DIR=$(config_get '.config_paths.config_dir') - export CONFIG_DIR -} - -teardown_file() { - load "../lib/teardown_file.sh" -} - -setup() { - load "../lib/setup.sh" - load "../lib/bats-file/load.bash" - ./instance-data load - hub_strip_index -} - -teardown() { - ./instance-crowdsec stop -} - -#---------- - -@test "cscli postoverflows list" { - hub_purge_all - - # no items - rune -0 cscli postoverflows list - assert_output --partial "POSTOVERFLOWS" - rune -0 cscli postoverflows list -o json - assert_json '{postoverflows:[]}' - rune -0 cscli postoverflows list -o raw - assert_output 'name,status,version,description' - - # some items - rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist - - rune -0 cscli postoverflows list - assert_output --partial crowdsecurity/rdns - assert_output --partial crowdsecurity/cdn-whitelist - rune -0 grep -c enabled <(output) - assert_output "2" - - rune -0 cscli postoverflows list -o json - assert_output --partial crowdsecurity/rdns - assert_output --partial crowdsecurity/cdn-whitelist - rune -0 jq '.postoverflows | length' <(output) - assert_output "2" - - rune -0 cscli postoverflows list -o raw - assert_output --partial crowdsecurity/rdns - assert_output --partial crowdsecurity/cdn-whitelist - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "2" -} - -@test "cscli postoverflows list -a" { - expected=$(jq <"$INDEX_PATH" -r '.postoverflows | length') - - rune -0 cscli postoverflows list -a - rune -0 grep -c disabled <(output) - assert_output "$expected" - - rune -0 cscli postoverflows list -o json -a - rune -0 jq '.postoverflows | length' <(output) - assert_output "$expected" - - rune -0 cscli postoverflows list -o raw -a - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "$expected" - - # the list should be the same in all formats, and sorted (not case sensitive) - - list_raw=$(cscli postoverflows list -o raw -a | tail -n +2 | cut -d, -f1) - list_human=$(cscli postoverflows list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) - list_json=$(cscli postoverflows list -o json -a | jq -r '.postoverflows[].name') - - rune -0 sort -f <<<"$list_raw" - assert_output "$list_raw" - - assert_equal "$list_raw" "$list_json" - assert_equal "$list_raw" "$list_human" -} - -@test "cscli postoverflows list [postoverflow]..." { - # non-existent - rune -1 cscli postoverflows install foo/bar - assert_stderr --partial "can't find 'foo/bar' in postoverflows" - - # not installed - rune -0 cscli postoverflows list crowdsecurity/rdns - assert_output --regexp 'crowdsecurity/rdns.*disabled' - - # install two items - rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist - - # list an installed item - rune -0 cscli postoverflows list crowdsecurity/rdns - assert_output --regexp "crowdsecurity/rdns.*enabled" - refute_output --partial "crowdsecurity/cdn-whitelist" - - # list multiple installed and non installed items - rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist crowdsecurity/ipv6_to_range - assert_output --partial "crowdsecurity/rdns" - assert_output --partial "crowdsecurity/cdn-whitelist" - assert_output --partial "crowdsecurity/ipv6_to_range" - - rune -0 cscli postoverflows list crowdsecurity/rdns -o json - rune -0 jq '.postoverflows | length' <(output) - assert_output "1" - rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist crowdsecurity/ipv6_to_range -o json - rune -0 jq '.postoverflows | length' <(output) - assert_output "3" - - rune -0 cscli postoverflows list crowdsecurity/rdns -o raw - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "1" - rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist crowdsecurity/ipv6_to_range -o raw - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "3" -} - -@test "cscli postoverflows install" { - rune -1 cscli postoverflows install - assert_stderr --partial 'requires at least 1 arg(s), only received 0' - - # not in hub - rune -1 cscli postoverflows install crowdsecurity/blahblah - assert_stderr --partial "can't find 'crowdsecurity/blahblah' in postoverflows" - - # simple install - rune -0 cscli postoverflows install crowdsecurity/rdns - rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics - assert_output --partial 'crowdsecurity/rdns' - assert_output --partial 'installed: true' - - # autocorrect - rune -1 cscli postoverflows install crowdsecurity/rdnf - assert_stderr --partial "can't find 'crowdsecurity/rdnf' in postoverflows, did you mean 'crowdsecurity/rdns'?" - - # install multiple - rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist - rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics - assert_output --partial 'crowdsecurity/rdns' - assert_output --partial 'installed: true' - rune -0 cscli postoverflows inspect crowdsecurity/cdn-whitelist --no-metrics - assert_output --partial 'crowdsecurity/cdn-whitelist' - assert_output --partial 'installed: true' -} - -@test "cscli postoverflows install (file location and download-only)" { - rune -0 cscli postoverflows install crowdsecurity/rdns --download-only - rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics - assert_output --partial 'crowdsecurity/rdns' - assert_output --partial 'installed: false' - assert_file_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml" - assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" - - rune -0 cscli postoverflows install crowdsecurity/rdns - rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics - assert_output --partial 'installed: true' - assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" -} - -@test "cscli postoverflows install --force (tainted)" { - rune -0 cscli postoverflows install crowdsecurity/rdns - echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" - - rune -1 cscli postoverflows install crowdsecurity/rdns - assert_stderr --partial "error while installing 'crowdsecurity/rdns': while enabling crowdsecurity/rdns: crowdsecurity/rdns is tainted, won't overwrite unless --force" - - rune -0 cscli postoverflows install crowdsecurity/rdns --force - assert_stderr --partial "Enabled crowdsecurity/rdns" -} - -@test "cscli postoverflow install --ignore (skip on errors)" { - rune -1 cscli postoverflows install foo/bar crowdsecurity/rdns - assert_stderr --partial "can't find 'foo/bar' in postoverflows" - refute_stderr --partial "Enabled postoverflows: crowdsecurity/rdns" - - rune -0 cscli postoverflows install foo/bar crowdsecurity/rdns --ignore - assert_stderr --partial "can't find 'foo/bar' in postoverflows" - assert_stderr --partial "Enabled postoverflows: crowdsecurity/rdns" -} - -@test "cscli postoverflows inspect" { - rune -1 cscli postoverflows inspect - assert_stderr --partial 'requires at least 1 arg(s), only received 0' - # required for metrics - ./instance-crowdsec start - - rune -1 cscli postoverflows inspect blahblah/blahblah - assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" - - # one item - rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics - assert_line 'type: postoverflows' - assert_line 'stage: s00-enrich' - assert_line 'name: crowdsecurity/rdns' - assert_line 'author: crowdsecurity' - assert_line 'path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml' - assert_line 'installed: false' - refute_line --partial 'Current metrics:' - - # one item, with metrics - rune -0 cscli postoverflows inspect crowdsecurity/rdns - assert_line --partial 'Current metrics:' - - # one item, json - rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json - rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output) - assert_json '["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",false]' - - # one item, raw - rune -0 cscli postoverflows inspect crowdsecurity/rdns -o raw - assert_line 'type: postoverflows' - assert_line 'name: crowdsecurity/rdns' - assert_line 'stage: s00-enrich' - assert_line 'author: crowdsecurity' - assert_line 'path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml' - assert_line 'installed: false' - refute_line --partial 'Current metrics:' - - # multiple items - rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist --no-metrics - assert_output --partial 'crowdsecurity/rdns' - assert_output --partial 'crowdsecurity/cdn-whitelist' - rune -1 grep -c 'Current metrics:' <(output) - assert_output "0" - - # multiple items, with metrics - rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist - rune -0 grep -c 'Current metrics:' <(output) - assert_output "2" - - # multiple items, json - rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist -o json - rune -0 jq -sc '[.[] | [.type, .stage, .name, .author, .path, .installed]]' <(output) - assert_json '[["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",false],["postoverflows","s01-whitelist","crowdsecurity/cdn-whitelist","crowdsecurity","postoverflows/s01-whitelist/crowdsecurity/cdn-whitelist.yaml",false]]' - - # multiple items, raw - rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist -o raw - assert_output --partial 'crowdsecurity/rdns' - assert_output --partial 'crowdsecurity/cdn-whitelist' - run -1 grep -c 'Current metrics:' <(output) - assert_output "0" -} - -@test "cscli postoverflows remove" { - rune -1 cscli postoverflows remove - assert_stderr --partial "specify at least one postoverflow to remove or '--all'" - rune -1 cscli postoverflows remove blahblah/blahblah - assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" - - rune -0 cscli postoverflows install crowdsecurity/rdns --download-only - rune -0 cscli postoverflows remove crowdsecurity/rdns - assert_stderr --partial "removing crowdsecurity/rdns: not installed -- no need to remove" - - rune -0 cscli postoverflows install crowdsecurity/rdns - rune -0 cscli postoverflows remove crowdsecurity/rdns - assert_stderr --partial 'Removed crowdsecurity/rdns' - - rune -0 cscli postoverflows remove crowdsecurity/rdns --purge - assert_stderr --partial 'Removed source file [crowdsecurity/rdns]' - - rune -0 cscli postoverflows remove crowdsecurity/rdns - assert_stderr --partial 'removing crowdsecurity/rdns: not installed -- no need to remove' - - rune -0 cscli postoverflows remove crowdsecurity/rdns --purge --debug - assert_stderr --partial 'removing crowdsecurity/rdns: not downloaded -- no need to remove' - refute_stderr --partial 'Removed source file [crowdsecurity/rdns]' - - # install, then remove, check files - rune -0 cscli postoverflows install crowdsecurity/rdns - assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" - rune -0 cscli postoverflows remove crowdsecurity/rdns - assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" - - # delete is an alias for remove - rune -0 cscli postoverflows install crowdsecurity/rdns - assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" - rune -0 cscli postoverflows delete crowdsecurity/rdns - assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" - - # purge - assert_file_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml" - rune -0 cscli postoverflows remove crowdsecurity/rdns --purge - assert_file_not_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml" - - rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist - - # --all - rune -0 cscli postoverflows list -o raw - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "2" - - rune -0 cscli postoverflows remove --all - - rune -0 cscli postoverflows list -o raw - rune -1 grep -vc 'name,status,version,description' <(output) - assert_output "0" -} - -@test "cscli postoverflows remove --force" { - # remove a postoverflow that belongs to a collection - rune -0 cscli collections install crowdsecurity/auditd - rune -0 cscli postoverflows remove crowdsecurity/auditd-whitelisted-process - assert_stderr --partial "crowdsecurity/auditd-whitelisted-process belongs to collections: [crowdsecurity/auditd]" - assert_stderr --partial "Run 'sudo cscli postoverflows remove crowdsecurity/auditd-whitelisted-process --force' if you want to force remove this postoverflow" -} - -@test "cscli postoverflows upgrade" { - rune -1 cscli postoverflows upgrade - assert_stderr --partial "specify at least one postoverflow to upgrade or '--all'" - rune -1 cscli postoverflows upgrade blahblah/blahblah - assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" - rune -0 cscli postoverflows remove crowdsecurity/discord-crawler-whitelist --purge - rune -1 cscli postoverflows upgrade crowdsecurity/discord-crawler-whitelist - assert_stderr --partial "can't upgrade crowdsecurity/discord-crawler-whitelist: not installed" - rune -0 cscli postoverflows install crowdsecurity/discord-crawler-whitelist --download-only - rune -1 cscli postoverflows upgrade crowdsecurity/discord-crawler-whitelist - assert_stderr --partial "can't upgrade crowdsecurity/discord-crawler-whitelist: downloaded but not installed" - - # hash of the string "v0.0" - sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" - - # add version 0.0 to all postoverflows - new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.postoverflows |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') - echo "$new_hub" >"$INDEX_PATH" - - rune -0 cscli postoverflows install crowdsecurity/rdns - - echo "v0.0" > "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" - rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json - rune -0 jq -e '.local_version=="0.0"' <(output) - - # upgrade - rune -0 cscli postoverflows upgrade crowdsecurity/rdns - rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json - rune -0 jq -e '.local_version==.version' <(output) - - # taint - echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" - # XXX: should return error - rune -0 cscli postoverflows upgrade crowdsecurity/rdns - assert_stderr --partial "crowdsecurity/rdns is tainted, --force to overwrite" - rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json - rune -0 jq -e '.local_version=="?"' <(output) - - # force upgrade with taint - rune -0 cscli postoverflows upgrade crowdsecurity/rdns --force - rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json - rune -0 jq -e '.local_version==.version' <(output) - - # multiple items - rune -0 cscli postoverflows install crowdsecurity/cdn-whitelist - echo "v0.0" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" - echo "v0.0" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml" - rune -0 cscli postoverflows list -o json - rune -0 jq -e '[.postoverflows[].local_version]==["0.0","0.0"]' <(output) - rune -0 cscli postoverflows upgrade crowdsecurity/rdns crowdsecurity/cdn-whitelist - rune -0 cscli postoverflows list -o json - rune -0 jq -e 'any(.postoverflows[].local_version; .=="0.0") | not' <(output) - - # upgrade all - echo "v0.0" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" - echo "v0.0" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml" - rune -0 cscli postoverflows list -o json - rune -0 jq -e '[.postoverflows[].local_version]==["0.0","0.0"]' <(output) - rune -0 cscli postoverflows upgrade --all - rune -0 cscli postoverflows list -o json - rune -0 jq -e 'any(.postoverflows[].local_version; .=="0.0") | not' <(output) -} diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats deleted file mode 100644 index b5f3a642233..00000000000 --- a/test/bats/20_hub_scenarios.bats +++ /dev/null @@ -1,383 +0,0 @@ -#!/usr/bin/env bats -# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: - -set -u - -setup_file() { - load "../lib/setup_file.sh" - ./instance-data load - HUB_DIR=$(config_get '.config_paths.hub_dir') - export HUB_DIR - INDEX_PATH=$(config_get '.config_paths.index_path') - export INDEX_PATH - CONFIG_DIR=$(config_get '.config_paths.config_dir') - export CONFIG_DIR -} - -teardown_file() { - load "../lib/teardown_file.sh" -} - -setup() { - load "../lib/setup.sh" - load "../lib/bats-file/load.bash" - ./instance-data load - hub_strip_index -} - -teardown() { - ./instance-crowdsec stop -} - -#---------- - -@test "cscli scenarios list" { - hub_purge_all - - # no items - rune -0 cscli scenarios list - assert_output --partial "SCENARIOS" - rune -0 cscli scenarios list -o json - assert_json '{scenarios:[]}' - rune -0 cscli scenarios list -o raw - assert_output 'name,status,version,description' - - # some items - rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf - - rune -0 cscli scenarios list - assert_output --partial crowdsecurity/ssh-bf - assert_output --partial crowdsecurity/telnet-bf - rune -0 grep -c enabled <(output) - assert_output "2" - - rune -0 cscli scenarios list -o json - assert_output --partial crowdsecurity/ssh-bf - assert_output --partial crowdsecurity/telnet-bf - rune -0 jq '.scenarios | length' <(output) - assert_output "2" - - rune -0 cscli scenarios list -o raw - assert_output --partial crowdsecurity/ssh-bf - assert_output --partial crowdsecurity/telnet-bf - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "2" -} - -@test "cscli scenarios list -a" { - expected=$(jq <"$INDEX_PATH" -r '.scenarios | length') - - rune -0 cscli scenarios list -a - rune -0 grep -c disabled <(output) - assert_output "$expected" - - rune -0 cscli scenarios list -o json -a - rune -0 jq '.scenarios | length' <(output) - assert_output "$expected" - - rune -0 cscli scenarios list -o raw -a - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "$expected" - - # the list should be the same in all formats, and sorted (not case sensitive) - - list_raw=$(cscli scenarios list -o raw -a | tail -n +2 | cut -d, -f1) - list_human=$(cscli scenarios list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) - list_json=$(cscli scenarios list -o json -a | jq -r '.scenarios[].name') - - # use python to sort because it handles "_" like go - rune -0 python3 -c 'import sys; print("".join(sorted(sys.stdin.readlines(), key=str.casefold)), end="")' <<<"$list_raw" - assert_output "$list_raw" - - assert_equal "$list_raw" "$list_json" - assert_equal "$list_raw" "$list_human" -} - -@test "cscli scenarios list [scenario]..." { - # non-existent - rune -1 cscli scenario install foo/bar - assert_stderr --partial "can't find 'foo/bar' in scenarios" - - # not installed - rune -0 cscli scenarios list crowdsecurity/ssh-bf - assert_output --regexp 'crowdsecurity/ssh-bf.*disabled' - - # install two items - rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf - - # list an installed item - rune -0 cscli scenarios list crowdsecurity/ssh-bf - assert_output --regexp "crowdsecurity/ssh-bf.*enabled" - refute_output --partial "crowdsecurity/telnet-bf" - - # list multiple installed and non installed items - rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf crowdsecurity/aws-bf - assert_output --partial "crowdsecurity/ssh-bf" - assert_output --partial "crowdsecurity/telnet-bf" - assert_output --partial "crowdsecurity/aws-bf" - - rune -0 cscli scenarios list crowdsecurity/ssh-bf -o json - rune -0 jq '.scenarios | length' <(output) - assert_output "1" - rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o json - rune -0 jq '.scenarios | length' <(output) - assert_output "3" - - rune -0 cscli scenarios list crowdsecurity/ssh-bf -o raw - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "1" - rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o raw - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "3" -} - -@test "cscli scenarios install" { - rune -1 cscli scenarios install - assert_stderr --partial 'requires at least 1 arg(s), only received 0' - - # not in hub - rune -1 cscli scenarios install crowdsecurity/blahblah - assert_stderr --partial "can't find 'crowdsecurity/blahblah' in scenarios" - - # simple install - rune -0 cscli scenarios install crowdsecurity/ssh-bf - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics - assert_output --partial 'crowdsecurity/ssh-bf' - assert_output --partial 'installed: true' - - # autocorrect - rune -1 cscli scenarios install crowdsecurity/ssh-tf - assert_stderr --partial "can't find 'crowdsecurity/ssh-tf' in scenarios, did you mean 'crowdsecurity/ssh-bf'?" - - # install multiple - rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics - assert_output --partial 'crowdsecurity/ssh-bf' - assert_output --partial 'installed: true' - rune -0 cscli scenarios inspect crowdsecurity/telnet-bf --no-metrics - assert_output --partial 'crowdsecurity/telnet-bf' - assert_output --partial 'installed: true' -} - -@test "cscli scenarios install (file location and download-only)" { - # simple install - rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics - assert_output --partial 'crowdsecurity/ssh-bf' - assert_output --partial 'installed: false' - assert_file_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml" - assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" - - rune -0 cscli scenarios install crowdsecurity/ssh-bf - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics - assert_output --partial 'installed: true' - assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" -} - -@test "cscli scenarios install --force (tainted)" { - rune -0 cscli scenarios install crowdsecurity/ssh-bf - echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" - - rune -1 cscli scenarios install crowdsecurity/ssh-bf - assert_stderr --partial "error while installing 'crowdsecurity/ssh-bf': while enabling crowdsecurity/ssh-bf: crowdsecurity/ssh-bf is tainted, won't overwrite unless --force" - - rune -0 cscli scenarios install crowdsecurity/ssh-bf --force - assert_stderr --partial "Enabled crowdsecurity/ssh-bf" -} - -@test "cscli scenarios install --ignore (skip on errors)" { - rune -1 cscli scenarios install foo/bar crowdsecurity/ssh-bf - assert_stderr --partial "can't find 'foo/bar' in scenarios" - refute_stderr --partial "Enabled scenarios: crowdsecurity/ssh-bf" - - rune -0 cscli scenarios install foo/bar crowdsecurity/ssh-bf --ignore - assert_stderr --partial "can't find 'foo/bar' in scenarios" - assert_stderr --partial "Enabled scenarios: crowdsecurity/ssh-bf" -} - -@test "cscli scenarios inspect" { - rune -1 cscli scenarios inspect - assert_stderr --partial 'requires at least 1 arg(s), only received 0' - # required for metrics - ./instance-crowdsec start - - rune -1 cscli scenarios inspect blahblah/blahblah - assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" - - # one item - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics - assert_line 'type: scenarios' - assert_line 'name: crowdsecurity/ssh-bf' - assert_line 'author: crowdsecurity' - assert_line 'path: scenarios/crowdsecurity/ssh-bf.yaml' - assert_line 'installed: false' - refute_line --partial 'Current metrics:' - - # one item, with metrics - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf - assert_line --partial 'Current metrics:' - - # one item, json - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json - rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output) - assert_json '["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",false]' - - # one item, raw - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o raw - assert_line 'type: scenarios' - assert_line 'name: crowdsecurity/ssh-bf' - assert_line 'author: crowdsecurity' - assert_line 'path: scenarios/crowdsecurity/ssh-bf.yaml' - assert_line 'installed: false' - refute_line --partial 'Current metrics:' - - # multiple items - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf --no-metrics - assert_output --partial 'crowdsecurity/ssh-bf' - assert_output --partial 'crowdsecurity/telnet-bf' - rune -1 grep -c 'Current metrics:' <(output) - assert_output "0" - - # multiple items, with metrics - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf - rune -0 grep -c 'Current metrics:' <(output) - assert_output "2" - - # multiple items, json - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o json - rune -0 jq -sc '[.[] | [.type, .name, .author, .path, .installed]]' <(output) - assert_json '[["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",false],["scenarios","crowdsecurity/telnet-bf","crowdsecurity","scenarios/crowdsecurity/telnet-bf.yaml",false]]' - - # multiple items, raw - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o raw - assert_output --partial 'crowdsecurity/ssh-bf' - assert_output --partial 'crowdsecurity/telnet-bf' - run -1 grep -c 'Current metrics:' <(output) - assert_output "0" -} - -@test "cscli scenarios remove" { - rune -1 cscli scenarios remove - assert_stderr --partial "specify at least one scenario to remove or '--all'" - rune -1 cscli scenarios remove blahblah/blahblah - assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" - - rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only - rune -0 cscli scenarios remove crowdsecurity/ssh-bf - assert_stderr --partial "removing crowdsecurity/ssh-bf: not installed -- no need to remove" - - rune -0 cscli scenarios install crowdsecurity/ssh-bf - rune -0 cscli scenarios remove crowdsecurity/ssh-bf - assert_stderr --partial "Removed crowdsecurity/ssh-bf" - - rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge - assert_stderr --partial 'Removed source file [crowdsecurity/ssh-bf]' - - rune -0 cscli scenarios remove crowdsecurity/ssh-bf - assert_stderr --partial "removing crowdsecurity/ssh-bf: not installed -- no need to remove" - - rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge --debug - assert_stderr --partial 'removing crowdsecurity/ssh-bf: not downloaded -- no need to remove' - refute_stderr --partial 'Removed source file [crowdsecurity/ssh-bf]' - - # install, then remove, check files - rune -0 cscli scenarios install crowdsecurity/ssh-bf - assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" - rune -0 cscli scenarios remove crowdsecurity/ssh-bf - assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" - - # delete is an alias for remove - rune -0 cscli scenarios install crowdsecurity/ssh-bf - assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" - rune -0 cscli scenarios delete crowdsecurity/ssh-bf - assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" - - # purge - assert_file_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml" - rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge - assert_file_not_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml" - - rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf - - # --all - rune -0 cscli scenarios list -o raw - rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "2" - - rune -0 cscli scenarios remove --all - - rune -0 cscli scenarios list -o raw - rune -1 grep -vc 'name,status,version,description' <(output) - assert_output "0" -} - -@test "cscli scenarios remove --force" { - # remove a scenario that belongs to a collection - rune -0 cscli collections install crowdsecurity/sshd - rune -0 cscli scenarios remove crowdsecurity/ssh-bf - assert_stderr --partial "crowdsecurity/ssh-bf belongs to collections: [crowdsecurity/sshd]" - assert_stderr --partial "Run 'sudo cscli scenarios remove crowdsecurity/ssh-bf --force' if you want to force remove this scenario" -} - -@test "cscli scenarios upgrade" { - rune -1 cscli scenarios upgrade - assert_stderr --partial "specify at least one scenario to upgrade or '--all'" - rune -1 cscli scenarios upgrade blahblah/blahblah - assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" - rune -0 cscli scenarios remove crowdsecurity/vsftpd-bf --purge - rune -1 cscli scenarios upgrade crowdsecurity/vsftpd-bf - assert_stderr --partial "can't upgrade crowdsecurity/vsftpd-bf: not installed" - rune -0 cscli scenarios install crowdsecurity/vsftpd-bf --download-only - rune -1 cscli scenarios upgrade crowdsecurity/vsftpd-bf - assert_stderr --partial "can't upgrade crowdsecurity/vsftpd-bf: downloaded but not installed" - - # hash of the string "v0.0" - sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" - - # add version 0.0 to all scenarios - new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.scenarios |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') - echo "$new_hub" >"$INDEX_PATH" - - rune -0 cscli scenarios install crowdsecurity/ssh-bf - - echo "v0.0" > "$CONFIG_DIR/scenarios/ssh-bf.yaml" - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json - rune -0 jq -e '.local_version=="0.0"' <(output) - - # upgrade - rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json - rune -0 jq -e '.local_version==.version' <(output) - - # taint - echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" - # XXX: should return error - rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf - assert_stderr --partial "crowdsecurity/ssh-bf is tainted, --force to overwrite" - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json - rune -0 jq -e '.local_version=="?"' <(output) - - # force upgrade with taint - rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf --force - rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json - rune -0 jq -e '.local_version==.version' <(output) - - # multiple items - rune -0 cscli scenarios install crowdsecurity/telnet-bf - echo "v0.0" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" - echo "v0.0" >"$CONFIG_DIR/scenarios/telnet-bf.yaml" - rune -0 cscli scenarios list -o json - rune -0 jq -e '[.scenarios[].local_version]==["0.0","0.0"]' <(output) - rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/telnet-bf - rune -0 cscli scenarios list -o json - rune -0 jq -e 'any(.scenarios[].local_version; .=="0.0") | not' <(output) - - # upgrade all - echo "v0.0" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" - echo "v0.0" >"$CONFIG_DIR/scenarios/telnet-bf.yaml" - rune -0 cscli scenarios list -o json - rune -0 jq -e '[.scenarios[].local_version]==["0.0","0.0"]' <(output) - rune -0 cscli scenarios upgrade --all - rune -0 cscli scenarios list -o json - rune -0 jq -e 'any(.scenarios[].local_version; .=="0.0") | not' <(output) -} diff --git a/test/bats/cscli-hubtype-inspect.bats b/test/bats/cscli-hubtype-inspect.bats new file mode 100644 index 00000000000..9c96aadb3ad --- /dev/null +++ b/test/bats/cscli-hubtype-inspect.bats @@ -0,0 +1,93 @@ +#!/usr/bin/env bats + +# Generic tests for the command "cscli inspect". +# +# Behavior that is specific to a hubtype should be tested in a separate file. + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli parsers inspect" { + rune -1 cscli parsers inspect + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + # required for metrics + ./instance-crowdsec start + + rune -1 cscli parsers inspect blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" + + # one item + rune -0 cscli parsers inspect crowdsecurity/sshd-logs --no-metrics + assert_line 'type: parsers' + assert_line 'name: crowdsecurity/sshd-logs' + assert_line 'path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # one item, with metrics + rune -0 cscli parsers inspect crowdsecurity/sshd-logs + assert_line --partial 'Current metrics:' + + # one item, json + rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o json + rune -0 jq -c '[.type, .name, .path, .installed]' <(output) + assert_json '["parsers","crowdsecurity/sshd-logs","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",false]' + + # one item, raw + rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o raw + assert_line 'type: parsers' + assert_line 'name: crowdsecurity/sshd-logs' + assert_line 'path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # multiple items + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists --no-metrics + assert_output --partial 'crowdsecurity/sshd-logs' + assert_output --partial 'crowdsecurity/whitelists' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" + + # multiple items, with metrics + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists + rune -0 grep -c 'Current metrics:' <(output) + assert_output "2" + + # multiple items, json + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o json + rune -0 jq -sc '[.[] | [.type, .name, .path, .installed]]' <(output) + assert_json '[["parsers","crowdsecurity/sshd-logs","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",false],["parsers","crowdsecurity/whitelists","parsers/s02-enrich/crowdsecurity/whitelists.yaml",false]]' + + # multiple items, raw + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o raw + assert_output --partial 'crowdsecurity/sshd-logs' + assert_output --partial 'crowdsecurity/whitelists' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" +} diff --git a/test/bats/cscli-hubtype-install.bats b/test/bats/cscli-hubtype-install.bats new file mode 100644 index 00000000000..2304e5a72cc --- /dev/null +++ b/test/bats/cscli-hubtype-install.bats @@ -0,0 +1,269 @@ +#!/usr/bin/env bats + +# Generic tests for the command "cscli install". +# +# Behavior that is specific to a hubtype should be tested in a separate file. + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR +# INDEX_PATH=$(config_get '.config_paths.index_path') +# export INDEX_PATH + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + # make sure the hub is empty + hub_purge_all +} + +teardown() { + # most tests don't need the service, but we ensure it's stopped + ./instance-crowdsec stop +} + +#---------- + +@test "cscli install (no argument)" { + rune -1 cscli parsers install + refute_output + assert_stderr --partial 'requires at least 1 arg(s), only received 0' +} + +@test "cscli install (aliased)" { + rune -1 cscli parser install + refute_output + assert_stderr --partial 'requires at least 1 arg(s), only received 0' +} + +@test "install an item (non-existent)" { + rune -1 cscli parsers install foo/bar + assert_stderr --partial "can't find 'foo/bar' in parsers" +} + +@test "install an item (dry run)" { + rune -0 cscli parsers install crowdsecurity/whitelists --dry-run + assert_output - --regexp <<-EOT + Action plan: + 📥 download + parsers: crowdsecurity/whitelists \([0-9]+.[0-9]+\) + ✅ enable + parsers: crowdsecurity/whitelists + + Dry run, no action taken. + EOT + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics -o json + rune -0 jq -e '.installed==false' <(output) + assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" +} + +@test "install an item (dry-run, de-duplicate commands)" { + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/whitelists --dry-run --output raw + assert_output - --regexp <<-EOT + Action plan: + 📥 download parsers:crowdsecurity/whitelists \([0-9]+.[0-9]+\) + ✅ enable parsers:crowdsecurity/whitelists + + Dry run, no action taken. + EOT + refute_stderr +} + +@test "install an item" { + rune -0 cscli parsers install crowdsecurity/whitelists + assert_output - <<-EOT + downloading parsers:crowdsecurity/whitelists + enabling parsers:crowdsecurity/whitelists + + $RELOAD_MESSAGE + EOT + refute_stderr + + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics -o json + rune -0 jq -e '.installed==true' <(output) + assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" +} + +@test "install an item (autocorrect)" { + rune -1 cscli parsers install crowdsecurity/whatelists + assert_stderr --partial "can't find 'crowdsecurity/whatelists' in parsers, did you mean 'crowdsecurity/whitelists'?" + refute_output +} + +@test "install an item (download only)" { + assert_file_not_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" + rune -0 cscli parsers install crowdsecurity/whitelists --download-only + assert_output - <<-EOT + downloading parsers:crowdsecurity/whitelists + + $RELOAD_MESSAGE + EOT + refute_stderr + + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics -o json + rune -0 jq -e '.installed==false' <(output) + assert_file_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" +} + +@test "install an item (already installed)" { + rune -0 cscli parsers install crowdsecurity/whitelists + rune -0 cscli parsers install crowdsecurity/whitelists --dry-run + assert_output "Nothing to do." + refute_stderr + rune -0 cscli parsers install crowdsecurity/whitelists + assert_output "Nothing to do." + refute_stderr +} + +@test "install an item (force is no-op if not tainted)" { + rune -0 cscli parsers install crowdsecurity/whitelists + rune -0 cscli parsers install crowdsecurity/whitelists + assert_output "Nothing to do." + refute_stderr + rune -0 cscli parsers install crowdsecurity/whitelists --force + assert_output "Nothing to do." + refute_stderr +} + +@test "install an item (tainted, requires --force)" { + rune -0 cscli parsers install crowdsecurity/whitelists + echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + rune -0 cscli parsers install crowdsecurity/whitelists --dry-run + assert_output - --stderr <<-EOT + WARN parsers:crowdsecurity/whitelists is tainted, use '--force' to overwrite + Nothing to do. + EOT + refute_stderr + + # XXX should this fail with status 1 instead? + rune -0 cscli parsers install crowdsecurity/whitelists + assert_output - <<-EOT + WARN parsers:crowdsecurity/whitelists is tainted, use '--force' to overwrite + Nothing to do. + EOT + refute_stderr + + rune -0 cscli parsers install crowdsecurity/whitelists --force + assert_output - <<-EOT + downloading parsers:crowdsecurity/whitelists + + $RELOAD_MESSAGE + EOT + refute_stderr + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics -o json + rune -0 jq -e '.installed==true' <(output) +} + +@test "install multiple items" { + rune -0 cscli parsers install crowdsecurity/pgsql-logs crowdsecurity/postfix-logs + rune -0 cscli parsers inspect crowdsecurity/pgsql-logs --no-metrics -o json + rune -0 jq -e '.installed==true' <(output) + rune -0 cscli parsers inspect crowdsecurity/postfix-logs --no-metrics -o json + rune -0 jq -e '.installed==true' <(output) +} + +@test "install multiple items (some already installed)" { + rune -0 cscli parsers install crowdsecurity/pgsql-logs + rune -0 cscli parsers install crowdsecurity/pgsql-logs crowdsecurity/postfix-logs --dry-run + assert_output - --regexp <<-EOT + Action plan: + 📥 download + parsers: crowdsecurity/postfix-logs \([0-9]+.[0-9]+\) + ✅ enable + parsers: crowdsecurity/postfix-logs + + Dry run, no action taken. + EOT + refute_stderr +} + +@test "install one or multiple items (ignore errors)" { + rune -0 cscli parsers install foo/bar --ignore + assert_stderr --partial "can't find 'foo/bar' in parsers" + assert_output "Nothing to do." + + rune -0 cscli parsers install crowdsecurity/whitelists + echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + # XXX: this is not testing '--ignore' anymore; TODO find a better error to ignore + # and maybe re-evaluate the --ignore flag + rune -0 cscli parsers install crowdsecurity/whitelists --ignore + assert_output - <<-EOT + WARN parsers:crowdsecurity/whitelists is tainted, use '--force' to overwrite + Nothing to do. + EOT + refute_stderr + + # error on one item, should still install the others + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/pgsql-logs --ignore + refute_stderr + assert_output - <<-EOT + WARN parsers:crowdsecurity/whitelists is tainted, use '--force' to overwrite + downloading parsers:crowdsecurity/pgsql-logs + enabling parsers:crowdsecurity/pgsql-logs + + $RELOAD_MESSAGE + EOT + rune -0 cscli parsers inspect crowdsecurity/pgsql-logs --no-metrics -o json + rune -0 jq -e '.installed==true' <(output) +} + +@test "override part of a collection with local items" { + # A collection will use a local item to fulfil a dependency provided it has + # the correct name field. + + mkdir -p "$CONFIG_DIR/parsers/s01-parse" + echo "name: crowdsecurity/sshd-logs" > "$CONFIG_DIR/parsers/s01-parse/sshd-logs.yaml" + rune -0 cscli parsers list -o json + rune -0 jq -c '.parsers[] | [.name,.status]' <(output) + assert_json '["crowdsecurity/sshd-logs","enabled,local"]' + + # attempt to install from hub + rune -0 cscli parsers install crowdsecurity/sshd-logs + assert_line 'parsers:crowdsecurity/sshd-logs - not downloading local item' + rune -0 cscli parsers list -o json + rune -0 jq -c '.parsers[] | [.name,.status]' <(output) + assert_json '["crowdsecurity/sshd-logs","enabled,local"]' + + # attempt to install from a collection + rune -0 cscli collections install crowdsecurity/sshd + assert_line 'parsers:crowdsecurity/sshd-logs - not downloading local item' + + # verify it installed the rest of the collection + assert_line 'enabling contexts:crowdsecurity/bf_base' + assert_line 'enabling collections:crowdsecurity/sshd' + + # remove them + rune -0 cscli collections delete crowdsecurity/sshd --force --purge + rune -0 rm "$CONFIG_DIR/parsers/s01-parse/sshd-logs.yaml" + + # do the same with a different file name + echo "name: crowdsecurity/sshd-logs" > "$CONFIG_DIR/parsers/s01-parse/something.yaml" + rune -0 cscli parsers list -o json + rune -0 jq -c '.parsers[] | [.name,.status]' <(output) + assert_json '["crowdsecurity/sshd-logs","enabled,local"]' + + # attempt to install from hub + rune -0 cscli parsers install crowdsecurity/sshd-logs + assert_line 'parsers:crowdsecurity/sshd-logs - not downloading local item' + + # attempt to install from a collection + rune -0 cscli collections install crowdsecurity/sshd + assert_line 'parsers:crowdsecurity/sshd-logs - not downloading local item' + + # verify it installed the rest of the collection + assert_line 'enabling contexts:crowdsecurity/bf_base' + assert_line 'enabling collections:crowdsecurity/sshd' +} diff --git a/test/bats/cscli-hubtype-list.bats b/test/bats/cscli-hubtype-list.bats new file mode 100644 index 00000000000..14113650c74 --- /dev/null +++ b/test/bats/cscli-hubtype-list.bats @@ -0,0 +1,130 @@ +#!/usr/bin/env bats + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli parsers list" { + hub_purge_all + + # no items + rune -0 cscli parsers list + assert_output --partial "PARSERS" + rune -0 cscli parsers list -o json + assert_json '{parsers:[]}' + rune -0 cscli parsers list -o raw + assert_output 'name,status,version,description' + + # some items + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth + + rune -0 cscli parsers list + assert_output --partial crowdsecurity/whitelists + assert_output --partial crowdsecurity/windows-auth + rune -0 grep -c enabled <(output) + assert_output "2" + + rune -0 cscli parsers list -o json + assert_output --partial crowdsecurity/whitelists + assert_output --partial crowdsecurity/windows-auth + rune -0 jq '.parsers | length' <(output) + assert_output "2" + + rune -0 cscli parsers list -o raw + assert_output --partial crowdsecurity/whitelists + assert_output --partial crowdsecurity/windows-auth + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli parsers list -a" { + expected=$(jq <"$INDEX_PATH" -r '.parsers | length') + + rune -0 cscli parsers list -a + rune -0 grep -c disabled <(output) + assert_output "$expected" + + rune -0 cscli parsers list -o json -a + rune -0 jq '.parsers | length' <(output) + assert_output "$expected" + + rune -0 cscli parsers list -o raw -a + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "$expected" + + # the list should be the same in all formats, and sorted (not case sensitive) + + list_raw=$(cscli parsers list -o raw -a | tail -n +2 | cut -d, -f1) + list_human=$(cscli parsers list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) + list_json=$(cscli parsers list -o json -a | jq -r '.parsers[].name') + + # use python to sort because it handles "_" like go + rune -0 python3 -c 'import sys; print("".join(sorted(sys.stdin.readlines(), key=str.casefold)), end="")' <<<"$list_raw" + assert_output "$list_raw" + + assert_equal "$list_raw" "$list_json" + assert_equal "$list_raw" "$list_human" +} + +@test "cscli parsers list [parser]..." { + # non-existent + rune -1 cscli parsers install foo/bar + assert_stderr --partial "can't find 'foo/bar' in parsers" + + # not installed + rune -0 cscli parsers list crowdsecurity/whitelists + assert_output --regexp 'crowdsecurity/whitelists.*disabled' + + # install two items + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth + + # list an installed item + rune -0 cscli parsers list crowdsecurity/whitelists + assert_output --regexp "crowdsecurity/whitelists.*enabled" + refute_output --partial "crowdsecurity/windows-auth" + + # list multiple installed and non installed items + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs + assert_output --partial "crowdsecurity/whitelists" + assert_output --partial "crowdsecurity/windows-auth" + assert_output --partial "crowdsecurity/traefik-logs" + + rune -0 cscli parsers list crowdsecurity/whitelists -o json + rune -0 jq '.parsers | length' <(output) + assert_output "1" + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o json + rune -0 jq '.parsers | length' <(output) + assert_output "3" + + rune -0 cscli parsers list crowdsecurity/whitelists -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "1" + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "3" +} diff --git a/test/bats/cscli-hubtype-remove.bats b/test/bats/cscli-hubtype-remove.bats new file mode 100644 index 00000000000..32db8efe788 --- /dev/null +++ b/test/bats/cscli-hubtype-remove.bats @@ -0,0 +1,245 @@ +#!/usr/bin/env bats + +# Generic tests for the command "cscli remove". +# +# Behavior that is specific to a hubtype should be tested in a separate file. + + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR +# INDEX_PATH=$(config_get '.config_paths.index_path') +# export INDEX_PATH + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + # make sure the hub is empty + hub_purge_all +} + +teardown() { + # most tests don't need the service, but we ensure it's stopped + ./instance-crowdsec stop +} + +#---------- + +@test "cscli remove (no argument)" { + rune -1 cscli parsers remove + refute_output + assert_stderr --partial "specify at least one parser to remove or '--all'" +} + +@test "cscli remove (aliased)" { + rune -1 cscli parser remove + refute_output + assert_stderr --partial "specify at least one parser to remove or '--all'" +} + +@test "cscli delete (alias of remove)" { + rune -1 cscli parsers delete + refute_output + assert_stderr --partial "specify at least one parser to remove or '--all'" +} + +@test "remove an item (non-existent)" { + rune -1 cscli parsers remove foo/bar + refute_output + assert_stderr --partial "can't find 'foo/bar' in parsers" +} + +@test "remove an item (not downloaded)" { + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics -o json + rune -0 jq -e '.downloaded==false' <(output) + + rune -0 cscli parsers remove crowdsecurity/whitelists --dry-run + assert_output "Nothing to do." + refute_stderr + rune -0 cscli parsers remove crowdsecurity/whitelists + assert_output "Nothing to do." + refute_stderr + rune -0 cscli parsers remove crowdsecurity/whitelists --force + assert_output "Nothing to do." + refute_stderr + rune -0 cscli parsers remove crowdsecurity/whitelists --purge + assert_output "Nothing to do." + refute_stderr +} + +@test "remove an item (not installed)" { + rune -0 cscli parsers install crowdsecurity/whitelists --download-only + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics -o json + rune -0 jq -e '.installed==false' <(output) + + rune -0 cscli parsers remove crowdsecurity/whitelists --dry-run + assert_output "Nothing to do." + refute_stderr + rune -0 cscli parsers remove crowdsecurity/whitelists + assert_output "Nothing to do." + refute_stderr + rune -0 cscli parsers remove crowdsecurity/whitelists --force + assert_output "Nothing to do." + refute_stderr + rune -0 cscli parsers remove crowdsecurity/whitelists --purge + assert_output --partial "purging parsers:crowdsecurity/whitelists" +} + +@test "remove an item (dry run)" { + rune -0 cscli parsers install crowdsecurity/whitelists + assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + rune -0 cscli parsers remove crowdsecurity/whitelists --dry-run + assert_output - --regexp <<-EOT + Action plan: + ❌ disable + parsers: crowdsecurity/whitelists + + Dry run, no action taken. + EOT + refute_stderr + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics -o json + rune -0 jq -e '.installed==true' <(output) + assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" +} + +@test "remove an item" { + rune -0 cscli parsers install crowdsecurity/whitelists + assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + rune -0 cscli parsers remove crowdsecurity/whitelists + assert_output - <<-EOT + disabling parsers:crowdsecurity/whitelists + + $RELOAD_MESSAGE + EOT + refute_stderr + + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics -o json + rune -0 jq -e '.installed==false' <(output) + assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + assert_file_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" +} + +@test "remove an item (purge)" { + rune -0 cscli parsers install crowdsecurity/whitelists + assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + rune -0 cscli parsers remove crowdsecurity/whitelists --purge + assert_output - <<-EOT + disabling parsers:crowdsecurity/whitelists + purging parsers:crowdsecurity/whitelists + + $RELOAD_MESSAGE + EOT + refute_stderr + + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics -o json + rune -0 jq -e '.downloaded==false' <(output) + assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + assert_file_not_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" +} + +@test "remove multiple items" { + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth + rune -0 cscli parsers remove crowdsecurity/whitelists crowdsecurity/windows-auth --dry-run + assert_output - --regexp <<-EOT + Action plan: + ❌ disable + parsers: crowdsecurity/whitelists, crowdsecurity/windows-auth + + Dry run, no action taken. + EOT + refute_stderr + + rune -0 cscli parsers remove crowdsecurity/whitelists crowdsecurity/windows-auth + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics -o json + rune -0 jq -e '.installed==false' <(output) + rune -0 cscli parsers inspect crowdsecurity/windows-auth --no-metrics -o json + rune -0 jq -e '.installed==false' <(output) +} + +@test "remove all items of a same type" { + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth + + rune -1 cscli parsers remove crowdsecurity/whitelists --all + assert_stderr "Error: can't specify items and '--all' at the same time" + + rune -0 cscli parsers remove --all --dry-run + assert_output - --regexp <<-EOT + Action plan: + ❌ disable + parsers: crowdsecurity/whitelists, crowdsecurity/windows-auth + + Dry run, no action taken. + EOT + refute_stderr + + rune -0 cscli parsers remove --all + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics -o json + rune -0 jq -e '.installed==false' <(output) + rune -0 cscli parsers inspect crowdsecurity/windows-auth --no-metrics -o json + rune -0 jq -e '.installed==false' <(output) +} + +@test "remove an item (tainted, requires --force)" { + rune -0 cscli parsers install crowdsecurity/whitelists + echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + rune -1 cscli parsers remove crowdsecurity/whitelists --dry-run + assert_stderr --partial "crowdsecurity/whitelists is tainted, use '--force' to remove" + refute_output + + rune -1 cscli parsers remove crowdsecurity/whitelists + assert_stderr --partial "crowdsecurity/whitelists is tainted, use '--force' to remove" + refute_output + + rune -0 cscli parsers remove crowdsecurity/whitelists --force + assert_output - <<-EOT + disabling parsers:crowdsecurity/whitelists + + $RELOAD_MESSAGE + EOT + refute_stderr + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics -o json + rune -0 jq -e '.installed==false' <(output) + assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" +} + +@test "remove an item that belongs to a collection (requires --force)" { + rune -0 cscli collections install crowdsecurity/sshd + # XXX: should exit with 1? + rune -0 cscli parsers remove crowdsecurity/sshd-logs + assert_output "Nothing to do." + assert_stderr --partial "crowdsecurity/sshd-logs belongs to collections: [crowdsecurity/sshd]" + assert_stderr --partial "Run 'sudo cscli parsers remove crowdsecurity/sshd-logs --force' if you want to force remove this parser" + assert_file_exists "$CONFIG_DIR/parsers/s01-parse/sshd-logs.yaml" + + rune -0 cscli parsers remove crowdsecurity/sshd-logs --force + assert_output - <<-EOT + disabling parsers:crowdsecurity/sshd-logs + + $RELOAD_MESSAGE + EOT + refute_stderr + assert_file_not_exists "$CONFIG_DIR/parsers/s01-parse/sshd-logs.yaml" +} + +@test "remove an item (autocomplete)" { + rune -0 cscli parsers install crowdsecurity/whitelists + rune -0 cscli __complete parsers remove crowd + assert_stderr --partial '[Debug] parsers: [crowdsecurity/whitelists]' + assert_output --partial 'crowdsecurity/whitelists' +} diff --git a/test/bats/cscli-hubtype-upgrade.bats b/test/bats/cscli-hubtype-upgrade.bats new file mode 100644 index 00000000000..4244e611cf6 --- /dev/null +++ b/test/bats/cscli-hubtype-upgrade.bats @@ -0,0 +1,253 @@ +#!/usr/bin/env bats + +# Generic tests for the upgrade of hub items and data files. +# +# Commands under test: +# cscli upgrade +# +# This file should test behavior that can be applied to all types. + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + # make sure the hub is empty + hub_purge_all +} + +teardown() { + # most tests don't need the service, but we ensure it's stopped + ./instance-crowdsec stop +} + +hub_inject_v0() { + # add a version 0.0 to all parsers + + # hash of the string "v0.0" + sha256_0_0="daa1832414a685d69269e0ae15024b908f4602db45f9900e9c6e7f204af207c0" + + new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.parsers |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') + echo "$new_hub" >"$INDEX_PATH" +} + +install_v0() { + local hubtype=$1 + shift + local item_name=$1 + shift + + cscli "$hubtype" install "$item_name" + printf "%s" "v0.0" > "$(jq -r '.local_path' <(cscli "$hubtype" inspect "$item_name" --no-metrics -o json))" +} + +#---------- + +@test "cscli upgrade (no argument)" { + rune -1 cscli parsers upgrade + refute_output + assert_stderr --partial "specify at least one parser to upgrade or '--all'" +} + +@test "cscli upgrade (aliased)" { + rune -1 cscli parser upgrade + refute_output + assert_stderr --partial "specify at least one parser to upgrade or '--all'" +} + +@test "upgrade an item (non-existent)" { + rune -1 cscli parsers upgrade foo/bar + assert_stderr --partial "can't find 'foo/bar' in parsers" +} + +@test "upgrade an item (non installed)" { + rune -0 cscli parsers upgrade crowdsecurity/whitelists + assert_output - <<-EOT + downloading parsers:crowdsecurity/whitelists + + $RELOAD_MESSAGE + EOT + refute_stderr + + rune -0 cscli parsers install crowdsecurity/whitelists --download-only + rune -0 cscli parsers upgrade crowdsecurity/whitelists + assert_output 'Nothing to do.' + refute_stderr +} + +@test "upgrade an item (up-to-date)" { + rune -0 cscli parsers install crowdsecurity/whitelists + rune -0 cscli parsers upgrade crowdsecurity/whitelists --dry-run + assert_output 'Nothing to do.' + rune -0 cscli parsers upgrade crowdsecurity/whitelists + assert_output 'Nothing to do.' +} + +@test "upgrade an item (dry run)" { + hub_inject_v0 + install_v0 parsers crowdsecurity/whitelists + latest=$(get_latest_version parsers crowdsecurity/whitelists) + + rune -0 cscli parsers upgrade crowdsecurity/whitelists --dry-run + assert_output - <<-EOT + Action plan: + 📥 download + parsers: crowdsecurity/whitelists (0.0 -> $latest) + + Dry run, no action taken. + EOT + refute_stderr +} + +get_latest_version() { + local hubtype=$1 + shift + local item_name=$1 + shift + + cscli "$hubtype" inspect "$item_name" -o json | jq -r '.version' +} + +@test "upgrade an item" { + hub_inject_v0 + install_v0 parsers crowdsecurity/whitelists + + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + + rune -0 cscli parsers upgrade crowdsecurity/whitelists + assert_output - <<-EOT + downloading parsers:crowdsecurity/whitelists + + $RELOAD_MESSAGE + EOT + refute_stderr + + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + + # the version is now the latest + rune -0 jq -e '.local_version==.version' <(output) +} + +@test "upgrade an item (tainted, requires --force)" { + rune -0 cscli parsers install crowdsecurity/whitelists + echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version=="?"' <(output) + + rune -0 cscli parsers upgrade crowdsecurity/whitelists --dry-run + assert_output - <<-EOT + WARN parsers:crowdsecurity/whitelists is tainted, use '--force' to overwrite + Nothing to do. + EOT + refute_stderr + + rune -0 cscli parsers upgrade crowdsecurity/whitelists + assert_output - <<-EOT + WARN parsers:crowdsecurity/whitelists is tainted, use '--force' to overwrite + Nothing to do. + EOT + refute_stderr + + rune -0 cscli parsers upgrade crowdsecurity/whitelists --force + assert_output - <<-EOT + downloading parsers:crowdsecurity/whitelists + + $RELOAD_MESSAGE + EOT + refute_stderr + + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version==.version' <(output) +} + +@test "upgrade multiple items" { + hub_inject_v0 + + install_v0 parsers crowdsecurity/whitelists + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + latest_whitelists=$(get_latest_version parsers crowdsecurity/whitelists) + + install_v0 parsers crowdsecurity/sshd-logs + rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + latest_sshd=$(get_latest_version parsers crowdsecurity/sshd-logs) + + rune -0 cscli parsers upgrade crowdsecurity/whitelists crowdsecurity/sshd-logs --dry-run + assert_output - <<-EOT + Action plan: + 📥 download + parsers: crowdsecurity/sshd-logs (0.0 -> $latest_sshd), crowdsecurity/whitelists (0.0 -> $latest_whitelists) + + Dry run, no action taken. + EOT + refute_stderr + + rune -0 cscli parsers upgrade crowdsecurity/whitelists crowdsecurity/sshd-logs + assert_output - <<-EOT + downloading parsers:crowdsecurity/whitelists + downloading parsers:crowdsecurity/sshd-logs + + $RELOAD_MESSAGE + EOT + refute_stderr + + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version==.version' <(output) + + rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o json + rune -0 jq -e '.local_version==.version' <(output) +} + +@test "upgrade all items of the same type" { + hub_inject_v0 + + install_v0 parsers crowdsecurity/whitelists + install_v0 parsers crowdsecurity/sshd-logs + install_v0 parsers crowdsecurity/windows-auth + + rune -0 cscli parsers upgrade --all + assert_output - <<-EOT + downloading parsers:crowdsecurity/sshd-logs + downloading parsers:crowdsecurity/whitelists + downloading parsers:crowdsecurity/windows-auth + + $RELOAD_MESSAGE + EOT + refute_stderr + + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version==.version' <(output) + + rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o json + rune -0 jq -e '.local_version==.version' <(output) + + rune -0 cscli parsers inspect crowdsecurity/windows-auth -o json + rune -0 jq -e '.local_version==.version' <(output) +} + +@test "upgrade an item (autocomplete)" { + rune -0 cscli parsers install crowdsecurity/whitelists + rune -0 cscli __complete parsers upgrade crowd + assert_stderr --partial '[Debug] parsers: [crowdsecurity/whitelists]' + assert_output --partial 'crowdsecurity/whitelists' +} + diff --git a/test/bats/cscli-parsers.bats b/test/bats/cscli-parsers.bats new file mode 100644 index 00000000000..6ff138e9fd8 --- /dev/null +++ b/test/bats/cscli-parsers.bats @@ -0,0 +1,44 @@ +#!/usr/bin/env bats + +# Tests for the "cscli parsers" behavior that is not covered by cscli-hubtype-*.bats + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli parsers inspect (includes the stage attribute)" { + rune -0 cscli parsers inspect crowdsecurity/sshd-logs --no-metrics -o human + assert_line 'stage: s01-parse' + + rune -0 cscli parsers inspect crowdsecurity/sshd-logs --no-metrics -o raw + assert_line 'stage: s01-parse' + + rune -0 cscli parsers inspect crowdsecurity/sshd-logs --no-metrics -o json + rune -0 jq -r '.stage' <(output) + assert_output 's01-parse' +} diff --git a/test/bats/cscli-postoverflows.bats b/test/bats/cscli-postoverflows.bats new file mode 100644 index 00000000000..979ee81defb --- /dev/null +++ b/test/bats/cscli-postoverflows.bats @@ -0,0 +1,44 @@ +#!/usr/bin/env bats + +# Tests for the "cscli postoverflows" behavior that is not covered by cscli-hubtype-*.bats + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli postoverflows inspect (includes the stage attribute)" { + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics -o human + assert_line 'stage: s00-enrich' + + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics -o raw + assert_line 'stage: s00-enrich' + + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics -o json + rune -0 jq -r '.stage' <(output) + assert_output 's00-enrich' +} diff --git a/test/bats/hub-index.bats b/test/bats/hub-index.bats new file mode 100644 index 00000000000..76759991e4a --- /dev/null +++ b/test/bats/hub-index.bats @@ -0,0 +1,357 @@ +#!/usr/bin/env bats + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "malformed index - null item" { + yq -o json >"$INDEX_PATH" <<-'EOF' + parsers: + author/pars1: + EOF + + rune -1 cscli hub list + assert_stderr --partial "failed to read hub index: parsers:author/pars1 has no index metadata." +} + +@test "malformed index - no download path" { + yq -o json >"$INDEX_PATH" <<-'EOF' + parsers: + author/pars1: + version: "0.0" + versions: + 0.0: + digest: daa1832414a685d69269e0ae15024b908f4602db45f9900e9c6e7f204af207c0 + EOF + + rune -1 cscli hub list + assert_stderr --partial "failed to read hub index: parsers:author/pars1 has no download path." +} + +@test "malformed parser - no stage" { + # Installing a parser requires a stage directory + yq -o json >"$INDEX_PATH" <<-'EOF' + parsers: + author/pars1: + path: parsers/s01-parse/author/pars1.yaml + version: "0.0" + versions: + 0.0: + digest: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + content: "{}" + EOF + + rune -1 cscli hub list -o raw + assert_stderr --partial "failed to read hub index: parsers:author/pars1 has no stage." +} + +@test "malformed parser - short path" { + # Installing a parser requires a stage directory + yq -o json >"$INDEX_PATH" <<-'EOF' + parsers: + author/pars1: + path: parsers/s01-parse/pars1.yaml + stage: s01-parse + version: "0.0" + versions: + 0.0: + digest: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + content: "{}" + EOF + + rune -0 cscli hub list -o raw + rune -0 cscli parsers install author/pars1 + rune -0 cscli hub list + # XXX here the item is installed but won't work, we only have a warning + assert_stderr --partial 'Ignoring file' + assert_stderr --partial 'path is too short' +} + +@test "malformed item - not yaml" { + # Installing an item requires reading the list of data files + yq -o json >"$INDEX_PATH" <<-'EOF' + parsers: + author/pars1: + path: parsers/s01-parse/pars1.yaml + stage: s01-parse + version: "0.0" + versions: + 0.0: + digest: daa1832414a685d69269e0ae15024b908f4602db45f9900e9c6e7f204af207c0 + content: "v0.0" + EOF + + rune -0 cscli hub list -o raw + rune -1 cscli parsers install author/pars1 + assert_stderr --partial 'unmarshal errors' +} + +@test "malformed item - hash mismatch" { + yq -o json >"$INDEX_PATH" <<-'EOF' + parsers: + author/pars1: + path: parsers/s01-parse/pars1.yaml + stage: s01-parse + version: "0.0" + versions: + 0.0: + digest: "0000000000000000000000000000000000000000000000000000000000000000" + content: "v0.0" + EOF + + rune -0 cscli hub list -o raw + rune -1 cscli parsers install author/pars1 + assert_stderr --partial 'parsers:author/pars1: hash mismatch: expected 0000000000000000000000000000000000000000000000000000000000000000, got daa1832414a685d69269e0ae15024b908f4602db45f9900e9c6e7f204af207c0.' +} + +@test "install minimal item" { + yq -o json >"$INDEX_PATH" <<-'EOF' + parsers: + author/pars1: + path: parsers/s01-parse/pars1.yaml + stage: s01-parse + version: "0.0" + versions: + 0.0: + digest: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + content: "{}" + EOF + + rune -0 cscli hub list -o raw + rune -0 cscli parsers install author/pars1 + assert_line "downloading parsers:author/pars1" + assert_line "enabling parsers:author/pars1" + rune -0 cscli hub list +} + +@test "replace an item in a collection update" { + # A new version of coll1 will uninstall pars1 and install pars2. + yq -o json >"$INDEX_PATH" <<-'EOF' + collections: + author/coll1: + path: collections/author/coll1.yaml + version: "0.0" + versions: + 0.0: + digest: 801e11865f8fdf82a348e70fe3f568af190715c40a176e058da2ad21ff5e20be + content: "{'parsers': ['author/pars1']}" + parsers: + - author/pars1 + parsers: + author/pars1: + path: parsers/s01-parse/author/pars1.yaml + stage: s01-parse + version: "0.0" + versions: + 0.0: + digest: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + content: "{}" + author/pars2: + path: parsers/s01-parse/author/pars2.yaml + stage: s01-parse + version: "0.0" + versions: + 0.0: + digest: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + content: "{}" + EOF + + rune -0 cscli hub list + rune -0 cscli collections install author/coll1 + + yq -o json >"$INDEX_PATH" <<-'EOF' + collections: + author/coll1: + path: collections/author/coll1.yaml + version: "0.1" + versions: + 0.0: + digest: 801e11865f8fdf82a348e70fe3f568af190715c40a176e058da2ad21ff5e20be + 0.1: + digest: f3c535c2d01abec5aadbb5ce03c357a478d91b116410c9fee288e073cd34c0dd + content: "{'parsers': ['author/pars2']}" + parsers: + - author/pars2 + parsers: + author/pars1: + path: parsers/s01-parse/author/pars1.yaml + stage: s01-parse + version: "0.0" + versions: + 0.0: + digest: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + content: "{}" + author/pars2: + path: parsers/s01-parse/author/pars2.yaml + stage: s01-parse + version: "0.0" + versions: + 0.0: + digest: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + content: "{}" + EOF + + rune -0 cscli hub list -o raw + rune -0 cscli collections upgrade author/coll1 + assert_output - <<-EOT + downloading parsers:author/pars2 + enabling parsers:author/pars2 + disabling parsers:author/pars1 + downloading collections:author/coll1 + + $RELOAD_MESSAGE + EOT + + rune -0 cscli hub list -o raw + assert_output - <<-EOT + name,status,version,description,type + author/pars2,enabled,0.0,,parsers + author/coll1,enabled,0.1,,collections + EOT +} + +@test "replace an outdated item only if it's not used elsewhere" { + # XXX + skip "not implemented" + # A new version of coll1 will uninstall pars1 and install pars2. + # Pars3 will not be uninstalled because it's still required by coll2. + yq -o json >"$INDEX_PATH" <<-'EOF' + collections: + author/coll1: + path: collections/author/coll1.yaml + version: "0.0" + versions: + 0.0: + digest: 0c397c7b3e19d730578932fdc260c53f39bd2488fad87207ab6b7e4dc315b067 + content: "{'parsers': ['author/pars1', 'author/pars3']}" + parsers: + - author/pars1 + - author/pars3 + author/coll2: + path: collections/author/coll2.yaml + version: "0.0" + versions: + 0.0: + digest: 96df483ff697d4d214792b135a3ba5ddaca0ebfd856e7da89215926394ac4001 + content: "{'parsers': ['author/pars3']}" + parsers: + - author/pars3 + parsers: + author/pars1: + path: parsers/s01-parse/author/pars1.yaml + stage: s01-parse + version: "0.0" + versions: + 0.0: + digest: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + content: "{}" + author/pars2: + path: parsers/s01-parse/author/pars2.yaml + stage: s01-parse + version: "0.0" + versions: + 0.0: + digest: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + content: "{}" + author/pars3: + path: parsers/s01-parse/author/pars3.yaml + stage: s01-parse + version: "0.0" + versions: + 0.0: + digest: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + content: "{}" + EOF + + rune -0 cscli hub list + rune -0 cscli collections install author/coll1 author/coll2 + + yq -o json >"$INDEX_PATH" <<-'EOF' + collections: + author/coll1: + path: collections/author/coll1.yaml + version: "0.1" + versions: + 0.0: + digest: 0c397c7b3e19d730578932fdc260c53f39bd2488fad87207ab6b7e4dc315b067 + 0.1: + digest: f3c535c2d01abec5aadbb5ce03c357a478d91b116410c9fee288e073cd34c0dd + content: "{'parsers': ['author/pars2']}" + parsers: + - author/pars2 + author/coll2: + path: collections/author/coll2.yaml + version: "0.0" + versions: + 0.0: + digest: 96df483ff697d4d214792b135a3ba5ddaca0ebfd856e7da89215926394ac4001 + content: "{'parsers': ['author/pars3']}" + parsers: + - author/pars3 + parsers: + author/pars1: + path: parsers/s01-parse/author/pars1.yaml + stage: s01-parse + version: "0.0" + versions: + 0.0: + digest: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + content: "{}" + author/pars2: + path: parsers/s01-parse/author/pars2.yaml + stage: s01-parse + version: "0.0" + versions: + 0.0: + digest: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + content: "{}" + author/pars3: + path: parsers/s01-parse/author/pars3.yaml + stage: s01-parse + version: "0.0" + versions: + 0.0: + digest: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + content: "{}" + EOF + + rune -0 cscli hub list -o raw + rune -0 cscli collections upgrade author/coll1 + assert_output - <<-EOT + downloading parsers:author/pars2 + enabling parsers:author/pars2 + disabling parsers:author/pars1 + downloading collections:author/coll1 + + $RELOAD_MESSAGE + EOT + + rune -0 cscli hub list -o raw + assert_output - <<-EOT + name,status,version,description,type + author/pars2,enabled,0.0,,parsers + author/pars3,enabled,0.0,,parsers + author/coll1,enabled,0.1,,collections + EOT +} diff --git a/test/bin/remove-all-hub-items b/test/bin/remove-all-hub-items index 981602b775a..b5d611782ff 100755 --- a/test/bin/remove-all-hub-items +++ b/test/bin/remove-all-hub-items @@ -14,7 +14,7 @@ echo "Pre-downloading Hub content..." types=$("$CSCLI" hub types -o raw) for itemtype in $types; do - "$CSCLI" "$itemtype" remove --all --force + "$CSCLI" "$itemtype" remove --all --force --purge --yes done echo " done." diff --git a/test/lib/config/config-local b/test/lib/config/config-local index 3e3c806b616..4f3ec7cc2ae 100755 --- a/test/lib/config/config-local +++ b/test/lib/config/config-local @@ -117,7 +117,7 @@ make_init_data() { "$CSCLI" --warning hub update --with-content # preload some content and data files - "$CSCLI" collections install crowdsecurity/linux --download-only + "$CSCLI" collections install crowdsecurity/linux --download-only --yes # sub-items did not respect --download-only ./bin/remove-all-hub-items diff --git a/test/lib/setup_file.sh b/test/lib/setup_file.sh index 39a084596e2..902edc5de82 100755 --- a/test/lib/setup_file.sh +++ b/test/lib/setup_file.sh @@ -260,16 +260,6 @@ hub_purge_all() { } export -f hub_purge_all -# remove unused data from the index, to make sure we don't rely on it in any way -hub_strip_index() { - local INDEX - INDEX=$(config_get .config_paths.index_path) - local hub_min - hub_min=$(jq <"$INDEX" 'del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') - echo "$hub_min" >"$INDEX" -} -export -f hub_strip_index - # remove color and style sequences from stdin plaintext() { sed -E 's/\x1B\[[0-9;]*[JKmsu]//g' @@ -340,3 +330,17 @@ lp-get-token() { echo "$resp" | yq -r '.token' } export -f lp-get-token + +case $(uname) in + "Linux") + # shellcheck disable=SC2089 + RELOAD_MESSAGE="Run 'sudo systemctl reload crowdsec' for the new configuration to be effective." + ;; + *) + # shellcheck disable=SC2089 + RELOAD_MESSAGE="Run 'sudo service crowdsec reload' for the new configuration to be effective." + ;; +esac + +# shellcheck disable=SC2090 +export RELOAD_MESSAGE