Refact cwhub

cmd/crowdsec-cli/config_restore.go

@@ -21,31 +21,6 @@ type OldAPICfg struct {
 	Password  string `json:"password"`
 }
 
-// it's a rip of the cli version, but in silent-mode
-// XXX: redundant, should call InstallItem
-func silentInstallItem(hub *cwhub.Hub, name, obtype string) (string, error) {
-	var item = hub.GetItem(obtype, name)
-	if item == nil {
-		return "", fmt.Errorf("error retrieving item")
-	}
-	err := hub.DownloadLatest(item, false, false)
-	if err != nil {
-		return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
-	}
-	if err = hub.AddItem(*item); err != nil {
-		return "", err
-	}
-
-	err = hub.EnableItem(item)
-	if err != nil {
-		return "", fmt.Errorf("error while enabling %s : %v", item.Name, err)
-	}
-	if err := hub.AddItem(*item); err != nil {
-		return "", err
-	}
-	return fmt.Sprintf("Enabled %s", item.Name), nil
-}
-
 func restoreHub(dirPath string) error {
 	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
 	if err != nil {
@@ -70,13 +45,9 @@ func restoreHub(dirPath string) error {
 			return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
 		}
 		for _, toinstall := range upstreamList {
-			label, err := silentInstallItem(hub, toinstall, itype)
+			err := hub.InstallItem(toinstall, itype, false, false)
 			if err != nil {
 				log.Errorf("Error while installing %s : %s", toinstall, err)
-			} else if label != "" {
-				log.Infof("Installed %s : %s", toinstall, label)
-			} else {
-				log.Printf("Installed %s : ok", toinstall)
 			}
 		}
 

cmd/crowdsec-cli/hub.go

@@ -59,7 +59,7 @@ func runHubList(cmd *cobra.Command, args []string) error {
 		log.Info(v)
 	}
 
-	for line := range hub.ItemStats() {
+	for _, line := range hub.ItemStats() {
 		log.Info(line)
 	}
 

pkg/cwhub/dataset.go

@@ -21,24 +21,19 @@ type DataSet struct {
 func downloadFile(url string, destPath string) error {
 	log.Debugf("downloading %s in %s", url, destPath)
 
-	req, err := http.NewRequest(http.MethodGet, url, nil)
+	resp, err := http.DefaultClient.Get(url)
 	if err != nil {
-		return err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return err
+		return fmt.Errorf("while downloading %s: %w", url, err)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("got HTTP status '%s' from %s", resp.Status, url)
+		return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
 	}
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return err
+		return fmt.Errorf("while downloading %s: %w", url, err)
 	}
 
 	file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644)

pkg/cwhub/enable.go

@@ -27,7 +27,7 @@ func (h *Hub) EnableItem(target *Item) error {
 		}
 
 		// if it's a collection, check sub-items even if the collection file itself is up-to-date
-		if target.UpToDate && target.Type != COLLECTIONS {
+		if target.UpToDate && !target.HasSubItems() {
 			log.Tracef("%s is installed and up-to-date, skip.", target.Name)
 			return nil
 		}
@@ -41,7 +41,7 @@ func (h *Hub) EnableItem(target *Item) error {
 		}
 	}
 
-	// install sub-items if it's a collection
+	// install sub-items if any
 	for _, sub := range target.SubItems() {
 		val, ok := h.Items[sub.Type][sub.Name]
 		if !ok {

pkg/cwhub/helpers.go

@@ -158,12 +158,13 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro
 	// XXX: should return the path of the downloaded file (taken from DownloadItem)
 	log.Debugf("Downloading %s %s", target.Type, target.Name)
 
-	if target.Type != COLLECTIONS {
+	if !target.HasSubItems() {
 		if !target.Installed && updateOnly && target.Downloaded {
 			log.Debugf("skipping upgrade of %s: not installed", target.Name)
 			return nil
 		}
 
+		// XXX:
 		return h.DownloadItem(target, overwrite)
 	}
 
@@ -182,7 +183,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro
 		log.Debugf("Download %s sub-item: %s %s (%t -> %t)", target.Name, sub.Type, sub.Name, target.Installed, updateOnly)
 
 		// recurse as it's a collection
-		if sub.Type == COLLECTIONS {
+		if sub.HasSubItems() {
 			log.Tracef("collection, recurse")
 
 			if err := h.DownloadLatest(&val, overwrite, updateOnly); err != nil {
@@ -235,24 +236,19 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool) error {
 		}
 	}
 
-	req, err := http.NewRequest(http.MethodGet, url, nil)
+	resp, err := http.DefaultClient.Get(url)
 	if err != nil {
-		return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
+		return fmt.Errorf("while downloading %s: %w", url, err)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String())
+		return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
 	}
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return fmt.Errorf("while reading %s: %w", req.URL.String(), err)
+		return fmt.Errorf("while downloading %s: %w", url, err)
 	}
 
 	hash := sha256.New()

pkg/cwhub/helpers_test.go

@@ -180,7 +180,7 @@ func assertCollectionDepsInstalled(t *testing.T, collection string) {
 	require.NoError(t, err)
 
 	c := hub.Items[COLLECTIONS][collection]
-	require.NoError(t, hub.CollectDepsCheck(&c))
+	require.NoError(t, hub.checkSubItems(&c))
 }
 
 func pushUpdateToCollectionInHub() {

pkg/cwhub/items.go

@@ -74,6 +74,14 @@ type SubItem struct {
 	Name string
 }
 
+func (i *Item) HasSubItems() bool {
+	return i.Type == COLLECTIONS
+}
+
+func (i *SubItem) HasSubItems() bool {
+	return i.Type == COLLECTIONS
+}
+
 func (i *Item) IsLocal() bool {
 	return i.Installed && !i.Downloaded
 }

pkg/cwhub/remote.go

@@ -40,23 +40,18 @@ func (r *RemoteHubCfg) downloadIndex(localPath string) error {
 		return fmt.Errorf("failed to build hub index request: %w", err)
 	}
 
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return fmt.Errorf("failed to build request for hub index: %w", err)
-	}
-
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := http.DefaultClient.Get(url)
 	if err != nil {
 		return fmt.Errorf("failed http request for hub index: %w", err)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
 		if resp.StatusCode == http.StatusNotFound {
-			return IndexNotFoundError{req.URL.String(), r.Branch}
+			return IndexNotFoundError{url, r.Branch}
 		}
 
-		return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String())
+		return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
 	}
 
 	body, err := io.ReadAll(resp.Body)

pkg/cwhub/sync.go

@@ -206,7 +206,6 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
 	if local && !inhub {
 		log.Tracef("%s is a local file, skip", path)
 		h.skippedLocal++
-		//	log.Infof("local scenario, skip.")
 
 		_, fileName := filepath.Split(path)
 
@@ -335,17 +334,18 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
 	return nil
 }
 
-func (h *Hub) CollectDepsCheck(v *Item) error {
-	if v.Type != COLLECTIONS {
+// checkSubItems checks for the presence, taint and version state of sub-items
+func (h *Hub) checkSubItems(v *Item) error {
+	if !v.HasSubItems() {
 		return nil
 	}
 
-	if v.versionStatus() != VersionUpToDate { // not up-to-date
+	if v.versionStatus() != VersionUpToDate {
 		log.Debugf("%s dependencies not checked: not up-to-date", v.Name)
 		return nil
 	}
 
-	// if it's a collection, ensure all the items are installed, or tag it as tainted
+	// ensure all the sub-items are installed, or tag the parent as tainted
 	log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed)
 
 	for _, sub := range v.SubItems() {
@@ -360,21 +360,10 @@ func (h *Hub) CollectDepsCheck(v *Item) error {
 			continue
 		}
 
-		if subItem.Type == COLLECTIONS {
-			log.Tracef("collec, recurse.")
-
-			if err := h.CollectDepsCheck(&subItem); err != nil {
-				if subItem.Tainted {
-					v.Tainted = true
-				}
-
-				return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
-			}
-
-			h.Items[sub.Type][sub.Name] = subItem
+		if err := h.checkSubItems(&subItem); err != nil {
+			return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
 		}
 
-		// propagate the state of sub-items to set
 		if subItem.Tainted {
 			v.Tainted = true
 			return fmt.Errorf("tainted %s %s, tainted", sub.Type, sub.Name)
@@ -431,7 +420,7 @@ func (h *Hub) SyncDir(dir string) ([]string, error) {
 		vs := item.versionStatus()
 		switch vs {
 		case VersionUpToDate: // latest
-			if err := h.CollectDepsCheck(&item); err != nil {
+			if err := h.checkSubItems(&item); err != nil {
 				warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
 				h.Items[COLLECTIONS][name] = item
 			}