Added support for date range in listing the recycle bin, and implemen…

…ted for eos drivers
glpatcern · Jan 12, 2024 · 93e9703 · 93e9703
1 parent 6c2a033
commit 93e9703
Show file tree

Hide file tree

Showing 13 changed files with 76 additions and 31 deletions.
diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go
@@ -1116,7 +1116,7 @@ func (s *service) ListRecycleStream(req *provider.ListRecycleStreamRequest, ss p
 	}
 
 	key, itemPath := router.ShiftPath(req.Key)
-	items, err := s.storage.ListRecycle(ctx, ref.GetPath(), key, itemPath)
+	items, err := s.storage.ListRecycle(ctx, ref.GetPath(), key, itemPath, req.FromTs.String(), req.ToTs.String())
 	if err != nil {
 		var st *rpc.Status
 		switch err.(type) {
@@ -1157,7 +1157,7 @@ func (s *service) ListRecycle(ctx context.Context, req *provider.ListRecycleRequ
 		return nil, err
 	}
 	key, itemPath := router.ShiftPath(req.Key)
-	items, err := s.storage.ListRecycle(ctx, ref.GetPath(), key, itemPath)
+	items, err := s.storage.ListRecycle(ctx, ref.GetPath(), key, itemPath, req.FromTs.String(), req.ToTs.String())
 	// TODO(labkode): CRITICAL: fill recycle info with storage provider.
 	if err != nil {
 		var st *rpc.Status

diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go
@@ -798,15 +798,25 @@ func (c *Client) GetRecyclePath(ctx context.Context, auth eosclient.Authorizatio
 }
 
 // ListDeletedEntries returns a list of the deleted entries.
-func (c *Client) ListDeletedEntries(ctx context.Context, auth eosclient.Authorization) ([]*eosclient.DeletedEntry, error) {
-	// Note that this may time out if the recycle has too many items:
+func (c *Client) ListDeletedEntries(ctx context.Context, auth eosclient.Authorization, from, to time.Time) ([]*eosclient.DeletedEntry, error) {
+	// Note that this may time out if the recycle has too many items or the time range is too large:
 	// the CS3API call ListRecycle includes a check to prevent that
-	args := []string{"recycle", "ls", "-m"}
-	stdout, _, err := c.executeEOS(ctx, args, auth)
-	if err != nil {
-		return nil, err
+	deleted := []*eosclient.DeletedEntry{}
+	for d := from; !d.After(to); d = d.AddDate(0, 0, 1) {
+		args := []string{"recycle", "ls", d.Format("2006/01/02"), "-m"}
+		stdout, _, err := c.executeEOS(ctx, args, auth)
+		if err != nil {
+			return nil, err
+		}
+
+		list, err := parseRecycleList(stdout)
+		if err != nil {
+			return nil, err
+		}
+		deleted = append(deleted, list...)
 	}
-	return parseRecycleList(stdout)
+
+	return deleted, nil
 }
 
 // RestoreDeletedEntry restores a deleted entry.

diff --git a/pkg/eosclient/eosclient.go b/pkg/eosclient/eosclient.go
@@ -21,6 +21,7 @@ package eosclient
 import (
 	"context"
 	"io"
+	"time"
 
 	"github.com/cs3org/reva/pkg/errtypes"
 	"github.com/cs3org/reva/pkg/storage/utils/acl"
@@ -53,7 +54,7 @@ type EOSClient interface {
 	Write(ctx context.Context, auth Authorization, path string, stream io.ReadCloser) error
 	WriteFile(ctx context.Context, auth Authorization, path, source string) error
 	GetRecyclePath(ctx context.Context, auth Authorization) (string, error)
-	ListDeletedEntries(ctx context.Context, auth Authorization) ([]*DeletedEntry, error)
+	ListDeletedEntries(ctx context.Context, auth Authorization, from, to time.Time) ([]*DeletedEntry, error)
 	RestoreDeletedEntry(ctx context.Context, auth Authorization, key string) error
 	PurgeDeletedEntries(ctx context.Context, auth Authorization) error
 	ListVersions(ctx context.Context, auth Authorization, p string) ([]*FileInfo, error)

diff --git a/pkg/eosclient/eosgrpc/eosgrpc.go b/pkg/eosclient/eosgrpc/eosgrpc.go
@@ -35,6 +35,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
 	"github.com/cs3org/reva/pkg/appctx"
@@ -1414,7 +1415,7 @@ func (c *Client) GetRecyclePath(ctx context.Context, auth eosclient.Authorizatio
 }
 
 // ListDeletedEntries returns a list of the deleted entries.
-func (c *Client) ListDeletedEntries(ctx context.Context, auth eosclient.Authorization) ([]*eosclient.DeletedEntry, error) {
+func (c *Client) ListDeletedEntries(ctx context.Context, auth eosclient.Authorization, from, to time.Time) ([]*eosclient.DeletedEntry, error) {
 	log := appctx.GetLogger(ctx)
 	log.Info().Str("func", "ListDeletedEntries").Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Msg("")
 

diff --git a/pkg/ocm/storage/outcoming/ocm.go b/pkg/ocm/storage/outcoming/ocm.go
@@ -668,7 +668,7 @@ func (d *driver) RestoreRevision(ctx context.Context, ref *provider.Reference, k
 	return errtypes.NotSupported("operation not supported")
 }
 
-func (d *driver) ListRecycle(ctx context.Context, basePath, key, relativePath string) ([]*provider.RecycleItem, error) {
+func (d *driver) ListRecycle(ctx context.Context, basePath, key, relativePath, from, to string) ([]*provider.RecycleItem, error) {
 	return nil, errtypes.NotSupported("operation not supported")
 }
 

diff --git a/pkg/ocm/storage/received/ocm.go b/pkg/ocm/storage/received/ocm.go
@@ -332,7 +332,7 @@ func (d *driver) RestoreRevision(ctx context.Context, ref *provider.Reference, k
 	return errtypes.NotSupported("operation not supported")
 }
 
-func (d *driver) ListRecycle(ctx context.Context, basePath, key, relativePath string) ([]*provider.RecycleItem, error) {
+func (d *driver) ListRecycle(ctx context.Context, basePath, key, relativePath, from, to string) ([]*provider.RecycleItem, error) {
 	return nil, errtypes.NotSupported("operation not supported")
 }
 

diff --git a/pkg/storage/fs/cephfs/cephfs.go b/pkg/storage/fs/cephfs/cephfs.go
@@ -604,7 +604,7 @@ func (fs *cephfs) CreateStorageSpace(ctx context.Context, req *provider.CreateSt
 	return nil, errtypes.NotSupported("unimplemented")
 }
 
-func (fs *cephfs) ListRecycle(ctx context.Context, basePath, key, relativePath string) ([]*provider.RecycleItem, error) {
+func (fs *cephfs) ListRecycle(ctx context.Context, basePath, key, relativePath, from, to string) ([]*provider.RecycleItem, error) {
 	return nil, errtypes.NotSupported("unimplemented")
 }
 

diff --git a/pkg/storage/fs/nextcloud/nextcloud.go b/pkg/storage/fs/nextcloud/nextcloud.go
@@ -374,7 +374,7 @@ type ListRecycleRequest struct {
 }
 
 // ListRecycle as defined in the storage.FS interface.
-func (nc *StorageDriver) ListRecycle(ctx context.Context, basePath, key, relativePath string) ([]*provider.RecycleItem, error) {
+func (nc *StorageDriver) ListRecycle(ctx context.Context, basePath, key, relativePath, from, to string) ([]*provider.RecycleItem, error) {
 	var items []*provider.RecycleItem
 	err := nc.do(ctx, http.MethodPost, "ListRecycle", ListRecycleRequest{Key: key, Path: relativePath}, &items)
 	return items, err

diff --git a/pkg/storage/fs/nextcloud/nextcloud_test.go b/pkg/storage/fs/nextcloud/nextcloud_test.go
@@ -491,13 +491,13 @@ var _ = Describe("Nextcloud", func() {
 		})
 	})
 
-	// ListRecycle(ctx context.Context, key, path string) ([]*provider.RecycleItem, error)
+	// ListRecycle(ctx context.Context, key, path, from, to string) ([]*provider.RecycleItem, error)
 	Describe("ListRecycle", func() {
 		It("calls the ListRecycle endpoint", func() {
 			nc, called, teardown := setUpNextcloudServer()
 			defer teardown()
 
-			results, err := nc.ListRecycle(ctx, "/", "asdf", "/some/file.txt")
+			results, err := nc.ListRecycle(ctx, "/", "asdf", "/some/file.txt", "", "")
 			Expect(err).ToNot(HaveOccurred())
 			// https://github.com/cs3org/go-cs3apis/blob/970eec3/cs3/storage/provider/v1beta1/resources.pb.go#L1085-L1110
 			Expect(len(results)).To(Equal(1))

diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go
@@ -43,7 +43,7 @@ type FS interface {
 	ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error)
 	DownloadRevision(ctx context.Context, ref *provider.Reference, key string) (io.ReadCloser, error)
 	RestoreRevision(ctx context.Context, ref *provider.Reference, key string) error
-	ListRecycle(ctx context.Context, basePath, key, relativePath string) ([]*provider.RecycleItem, error)
+	ListRecycle(ctx context.Context, basePath, key, relativePath, from, to string) ([]*provider.RecycleItem, error)
 	RestoreRecycleItem(ctx context.Context, basePath, key, relativePath string, restoreRef *provider.Reference) error
 	PurgeRecycleItem(ctx context.Context, basePath, key, relativePath string) error
 	EmptyRecycle(ctx context.Context) error

diff --git a/pkg/storage/utils/eosfs/config.go b/pkg/storage/utils/eosfs/config.go
@@ -180,4 +180,8 @@ type Config struct {
 	// Maximum entries count a ListRecycle call may return: if exceeded, ListRecycle
 	// will return a BadRequest error
 	MaxRecycleEntries uint64 `mapstructure:"max_recycle_entries"`
+
+	// Maximum time span in days a ListRecycle call may return: if exceeded, ListRecycle
+	// will override the "to" date with "from" + this value
+	MaxDaysInRecycleList int `mapstructure:"max_days_in_recycle_list"`
 }
diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go
@@ -151,6 +151,10 @@ func (c *Config) ApplyDefaults() {
 		c.MaxRecycleEntries = 5000
 	}
 
+	if c.MaxDaysInRecycleList == 0 {
+		c.MaxDaysInRecycleList = 14
+	}
+
 	c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc)
 }
 
@@ -1918,23 +1922,28 @@ func (fs *eosfs) EmptyRecycle(ctx context.Context) error {
 	return fs.c.PurgeDeletedEntries(ctx, auth)
 }
 
-func (fs *eosfs) countDeletedEntries(ctx context.Context, auth eosclient.Authorization) (uint64, error) {
-	// Look for the recycle path, typically /eos/<instance>/proc/recycle/uid:<UID>
+func (fs *eosfs) countDeletedEntries(ctx context.Context, auth eosclient.Authorization, from, to time.Time) (uint64, error) {
 	recyclePath, err := fs.c.GetRecyclePath(ctx, auth)
 	if err != nil {
 		return 0, err
 	}
 
-	// TODO: treeCount is not recursive, once we implement a calendar view we should do this on the '0' bucket
-	eosmd, err := fs.c.GetFileInfoByPath(ctx, auth, fmt.Sprintf("%s/uid:%s", recyclePath, auth.Role.UID))
-	if err != nil {
-		return 0, err
+	var total uint64
+	total = 0
+	for d := from; !d.After(to); d = d.AddDate(0, 0, 1) {
+		// stat the bucket 0, which includes by default the first 100K entries at most: if the user has more,
+		// the obtained count is a lower bound that is likely good enough to fail the request anyway
+		eosmd, err := fs.c.GetFileInfoByPath(ctx, auth, fmt.Sprintf("%s/uid:%s/%s/0", recyclePath, auth.Role.UID, d.Format("2006/01/31")))
+		if err != nil {
+			return 0, err
+		}
+		total += eosmd.TreeCount
 	}
 
-	return eosmd.TreeCount, nil
+	return total, nil
 }
 
-func (fs *eosfs) ListRecycle(ctx context.Context, basePath, key, relativePath string) ([]*provider.RecycleItem, error) {
+func (fs *eosfs) ListRecycle(ctx context.Context, basePath, key, relativePath, from, to string) ([]*provider.RecycleItem, error) {
 	var auth eosclient.Authorization
 
 	if !fs.conf.EnableHome && fs.conf.AllowPathRecycleOperations && basePath != "/" {
@@ -1965,13 +1974,33 @@ func (fs *eosfs) ListRecycle(ctx context.Context, basePath, key, relativePath st
 		}
 	}
 
-	// ignore errors for this check and optimistically move on with the listing
-	rcount, _ := fs.countDeletedEntries(ctx, auth)
+	var dateFrom, dateTo time.Time
+	if from == "" || to == "" {
+		// list recent (up to max days) entries by default
+		dateTo = time.Now()
+		dateFrom = dateTo.AddDate(0, 0, -fs.conf.MaxDaysInRecycleList)
+	} else {
+		dateFrom, err := time.Parse("2006/01/02", from)
+		if err != nil {
+			dateFrom = time.Now()
+		}
+		dateTo, err = time.Parse("2006/01/02", to)
+		if err != nil {
+			dateTo = time.Now()
+		}
+		maxDate := dateFrom.AddDate(0, 0, fs.conf.MaxDaysInRecycleList) // limit to avoid overloading EOS
+		if maxDate.Before(dateTo) {
+			dateTo = maxDate
+		}
+	}
+
+	// ignore errors here and optimistically move on with the listing
+	rcount, _ := fs.countDeletedEntries(ctx, auth, dateFrom, dateTo)
 	if rcount > fs.conf.MaxRecycleEntries {
-		return nil, errtypes.BadRequest("eosfs: too many entries found in listing recycle bin")
+		return nil, errtypes.BadRequest("eosfs: too many entries found in listing the recycle bin")
 	}
 
-	eosDeletedEntries, err := fs.c.ListDeletedEntries(ctx, auth)
+	eosDeletedEntries, err := fs.c.ListDeletedEntries(ctx, auth, dateFrom, dateTo)
 	if err != nil {
 		return nil, errors.Wrap(err, "eosfs: error listing deleted entries")
 	}

diff --git a/pkg/storage/utils/localfs/localfs.go b/pkg/storage/utils/localfs/localfs.go
@@ -1238,7 +1238,7 @@ func (fs *localfs) convertToRecycleItem(ctx context.Context, rp string, md os.Fi
 	}
 }
 
-func (fs *localfs) ListRecycle(ctx context.Context, basePath, key, relativePath string) ([]*provider.RecycleItem, error) {
+func (fs *localfs) ListRecycle(ctx context.Context, basePath, key, relativePath, from, to string) ([]*provider.RecycleItem, error) {
 	rp := fs.wrapRecycleBin(ctx, "/")
 
 	entries, err := os.ReadDir(rp)