cscli: improved hub management (#3352)

cmd/crowdsec-cli/clihub/hub.go

@@ -5,15 +5,18 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"os"
 
 	"github.com/fatih/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v3"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/reload"
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/hubops"
 )
 
 type configGetter = func() *csconfig.Config
@@ -55,11 +58,11 @@ func (cli *cliHub) List(out io.Writer, hub *cwhub.Hub, all bool) error {
 	cfg := cli.cfg()
 
 	for _, v := range hub.Warnings {
-		log.Info(v)
+		fmt.Fprintln(os.Stderr, v)
 	}
 
 	for _, line := range hub.ItemStats() {
-		log.Info(line)
+		fmt.Fprintln(os.Stderr, line)
 	}
 
 	items := make(map[string][]*cwhub.Item)
@@ -100,23 +103,22 @@ func (cli *cliHub) newListCmd() *cobra.Command {
 	}
 
 	flags := cmd.Flags()
-	flags.BoolVarP(&all, "all", "a", false, "List disabled items as well")
+	flags.BoolVarP(&all, "all", "a", false, "List all available items, including those not installed")
 
 	return cmd
 }
 
 func (cli *cliHub) update(ctx context.Context, withContent bool) error {
 	local := cli.cfg().Hub
 	remote := require.RemoteHub(ctx, cli.cfg())
-	remote.EmbedItemContent = withContent
 
 	// don't use require.Hub because if there is no index file, it would fail
 	hub, err := cwhub.NewHub(local, remote, log.StandardLogger())
 	if err != nil {
 		return err
 	}
 
-	if err := hub.Update(ctx); err != nil {
+	if err := hub.Update(ctx, withContent); err != nil {
 		return fmt.Errorf("failed to update hub: %w", err)
 	}
 
@@ -125,7 +127,7 @@ func (cli *cliHub) update(ctx context.Context, withContent bool) error {
 	}
 
 	for _, v := range hub.Warnings {
-		log.Info(v)
+		fmt.Fprintln(os.Stderr, v)
 	}
 
 	return nil
@@ -140,10 +142,18 @@ func (cli *cliHub) newUpdateCmd() *cobra.Command {
 		Long: `
 Fetches the .index.json file from the hub, containing the list of available configs.
 `,
+		Example: `# Download the last version of the index file.
+cscli hub update
+
+# Download a 4x bigger version with all item contents (effectively pre-caching item downloads, but not data files).
+cscli hub update --with-content`,
 		Args:              cobra.NoArgs,
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			return cli.update(cmd.Context(), withContent)
+			if cmd.Flags().Changed("with-content") {
+				return cli.update(cmd.Context(), withContent)
+			}
+			return cli.update(cmd.Context(), cli.cfg().Cscli.HubWithContent)
 		},
 	}
 
@@ -153,52 +163,63 @@ Fetches the .index.json file from the hub, containing the list of available conf
 	return cmd
 }
 
-func (cli *cliHub) upgrade(ctx context.Context, force bool) error {
-	hub, err := require.Hub(cli.cfg(), require.RemoteHub(ctx, cli.cfg()), log.StandardLogger())
+func (cli *cliHub) upgrade(ctx context.Context, yes bool, dryRun bool, force bool) error {
+	cfg := cli.cfg()
+
+	hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger())
 	if err != nil {
 		return err
 	}
 
+	plan := hubops.NewActionPlan(hub)
+
 	for _, itemType := range cwhub.ItemTypes {
-		updated := 0
+		for _, item := range hub.GetInstalledByType(itemType, true) {
+			plan.AddCommand(hubops.NewDownloadCommand(item, force))
+		}
+	}
 
-		log.Infof("Upgrading %s", itemType)
+	plan.AddCommand(hubops.NewDataRefreshCommand(force))
 
-		for _, item := range hub.GetInstalledByType(itemType, true) {
-			didUpdate, err := item.Upgrade(ctx, force)
-			if err != nil {
-				return err
-			}
+	verbose := (cfg.Cscli.Output == "raw")
 
-			if didUpdate {
-				updated++
-			}
-		}
+	if err := plan.Execute(ctx, yes, dryRun, verbose); err != nil {
+		return err
+	}
 
-		log.Infof("Upgraded %d %s", updated, itemType)
+	if plan.ReloadNeeded {
+		fmt.Println("\n" + reload.Message)
 	}
 
 	return nil
 }
 
 func (cli *cliHub) newUpgradeCmd() *cobra.Command {
-	var force bool
+	var (
+		yes bool
+		dryRun bool
+		force bool
+	)
 
 	cmd := &cobra.Command{
 		Use:   "upgrade",
 		Short: "Upgrade all configurations to their latest version",
 		Long: `
 Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available.
 `,
+		// TODO: Example
 		Args:              cobra.NoArgs,
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			return cli.upgrade(cmd.Context(), force)
+			return cli.upgrade(cmd.Context(), yes, dryRun, force)
 		},
 	}
 
 	flags := cmd.Flags()
-	flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated files")
+	flags.BoolVar(&yes, "yes", false, "Confirm execution without prompt")
+	flags.BoolVar(&dryRun, "dry-run", false, "Don't install or remove anything; print the execution plan")
+	flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated items; always update data files")
+	cmd.MarkFlagsMutuallyExclusive("yes", "dry-run")
 
 	return cmd
 }

cmd/crowdsec-cli/clihub/items.go

@@ -5,13 +5,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"os"
-	"path/filepath"
 	"slices"
 	"strings"
 
-	"gopkg.in/yaml.v3"
-
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
@@ -145,42 +141,3 @@ func ListItems(out io.Writer, wantColor string, itemTypes []string, items map[st
 
 	return nil
 }
-
-func InspectItem(item *cwhub.Item, wantMetrics bool, output string, prometheusURL string, wantColor string) error {
-	switch output {
-	case "human", "raw":
-		enc := yaml.NewEncoder(os.Stdout)
-		enc.SetIndent(2)
-
-		if err := enc.Encode(item); err != nil {
-			return fmt.Errorf("unable to encode item: %w", err)
-		}
-	case "json":
-		b, err := json.MarshalIndent(*item, "", "  ")
-		if err != nil {
-			return fmt.Errorf("unable to serialize item: %w", err)
-		}
-
-		fmt.Print(string(b))
-	}
-
-	if output != "human" {
-		return nil
-	}
-
-	if item.State.Tainted {
-		fmt.Println()
-		fmt.Printf(`This item is tainted. Use "%s %s inspect --diff %s" to see why.`, filepath.Base(os.Args[0]), item.Type, item.Name)
-		fmt.Println()
-	}
-
-	if wantMetrics {
-		fmt.Printf("\nCurrent metrics: \n")
-
-		if err := showMetrics(prometheusURL, item, wantColor); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

cmd/crowdsec-cli/clihub/utils_table.go

@@ -3,7 +3,6 @@ package clihub
 import (
 	"fmt"
 	"io"
-	"strconv"
 
 	"github.com/jedib0t/go-pretty/v6/table"
 
@@ -24,62 +23,3 @@ func listHubItemTable(out io.Writer, wantColor string, title string, items []*cw
 	io.WriteString(out, title+"\n")
 	io.WriteString(out, t.Render()+"\n")
 }
-
-func appsecMetricsTable(out io.Writer, wantColor string, itemName string, metrics map[string]int) {
-	t := cstable.NewLight(out, wantColor).Writer
-	t.AppendHeader(table.Row{"Inband Hits", "Outband Hits"})
-
-	t.AppendRow(table.Row{
-		strconv.Itoa(metrics["inband_hits"]),
-		strconv.Itoa(metrics["outband_hits"]),
-	})
-
-	io.WriteString(out, fmt.Sprintf("\n - (AppSec Rule) %s:\n", itemName))
-	io.WriteString(out, t.Render()+"\n")
-}
-
-func scenarioMetricsTable(out io.Writer, wantColor string, itemName string, metrics map[string]int) {
-	if metrics["instantiation"] == 0 {
-		return
-	}
-
-	t := cstable.New(out, wantColor).Writer
-	t.AppendHeader(table.Row{"Current Count", "Overflows", "Instantiated", "Poured", "Expired"})
-
-	t.AppendRow(table.Row{
-		strconv.Itoa(metrics["curr_count"]),
-		strconv.Itoa(metrics["overflow"]),
-		strconv.Itoa(metrics["instantiation"]),
-		strconv.Itoa(metrics["pour"]),
-		strconv.Itoa(metrics["underflow"]),
-	})
-
-	io.WriteString(out, fmt.Sprintf("\n - (Scenario) %s:\n", itemName))
-	io.WriteString(out, t.Render()+"\n")
-}
-
-func parserMetricsTable(out io.Writer, wantColor string, itemName string, metrics map[string]map[string]int) {
-	t := cstable.New(out, wantColor).Writer
-	t.AppendHeader(table.Row{"Parsers", "Hits", "Parsed", "Unparsed"})
-
-	// don't show table if no hits
-	showTable := false
-
-	for source, stats := range metrics {
-		if stats["hits"] > 0 {
-			t.AppendRow(table.Row{
-				source,
-				strconv.Itoa(stats["hits"]),
-				strconv.Itoa(stats["parsed"]),
-				strconv.Itoa(stats["unparsed"]),
-			})
-
-			showTable = true
-		}
-	}
-
-	if showTable {
-		io.WriteString(out, fmt.Sprintf("\n - (Parser) %s:\n", itemName))
-		io.WriteString(out, t.Render()+"\n")
-	}
-}

cmd/crowdsec-cli/cliitem/inspect.go

@@ -0,0 +1,57 @@
+package cliitem
+
+import (
+	"fmt"
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func inspectItem(hub *cwhub.Hub, item *cwhub.Item, wantMetrics bool, output string, prometheusURL string, wantColor string) error {
+	// This is dirty...
+	// We want to show current dependencies (from content), not latest (from index).
+	// The item is modifed but after this function the whole hub should be thrown away.
+	// A cleaner way would be to copy the struct first.
+	item.Dependencies = item.CurrentDependencies()
+
+	switch output {
+	case "human", "raw":
+		enc := yaml.NewEncoder(os.Stdout)
+		enc.SetIndent(2)
+
+		if err := enc.Encode(item); err != nil {
+			return fmt.Errorf("unable to encode item: %w", err)
+		}
+	case "json":
+		b, err := json.MarshalIndent(*item, "", "  ")
+		if err != nil {
+			return fmt.Errorf("unable to serialize item: %w", err)
+		}
+
+		fmt.Print(string(b))
+	}
+
+	if output != "human" {
+		return nil
+	}
+
+	if item.State.Tainted {
+		fmt.Println()
+		fmt.Printf(`This item is tainted. Use "%s %s inspect --diff %s" to see why.`, filepath.Base(os.Args[0]), item.Type, item.Name)
+		fmt.Println()
+	}
+
+	if wantMetrics {
+		fmt.Printf("\nCurrent metrics: \n")
+
+		if err := showMetrics(prometheusURL, hub, item, wantColor); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}