From 042aa3e7934d7967905fec53f87e2780f4e9d1c3 Mon Sep 17 00:00:00 2001 From: Samuel Alfageme Sainz Date: Mon, 5 Dec 2022 13:46:08 +0100 Subject: [PATCH 1/9] search: basic REPORT functionality - SearchDir() with searchstring and basic input validation - log pattern on REPORT - ListFolder() with search capabilities via opaque - add the searchString from opaque to ListFolder mdKeys --- .../storageprovider/storageprovider.go | 5 + .../http/services/owncloud/ocdav/report.go | 135 +++++++++++++----- pkg/eosclient/eosbinary/eosbinary.go | 28 +++- pkg/eosclient/eosclient.go | 1 + pkg/eosclient/eosgrpc/eosgrpc.go | 4 + pkg/storage/utils/eosfs/eosfs.go | 49 +++++++ 6 files changed, 186 insertions(+), 36 deletions(-) diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index c116604362..cc7b8ccb8e 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -938,6 +938,11 @@ func (s *service) ListContainer(ctx context.Context, req *provider.ListContainer }, nil } + // Ugly hack to enable search + if req.Opaque.Map["search"] != nil { + s.storage.ListFolder(ctx, newRef, []string{"search", string(req.Opaque.Map["searchString"].GetValue())}) + } + mds, err := s.storage.ListFolder(ctx, newRef, req.ArbitraryMetadataKeys) if err != nil { var st *rpc.Status diff --git a/internal/http/services/owncloud/ocdav/report.go b/internal/http/services/owncloud/ocdav/report.go index 70fb2b7dfa..048af9b94b 100644 --- a/internal/http/services/owncloud/ocdav/report.go +++ b/internal/http/services/owncloud/ocdav/report.go @@ -20,12 +20,15 @@ package ocdav import ( "encoding/xml" + "fmt" "io" + "io/ioutil" "net/http" "strings" rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" ctxpkg "github.com/cs3org/reva/pkg/ctx" ) @@ -38,14 +41,17 @@ const ( func (s *svc) handleReport(w http.ResponseWriter, r *http.Request, ns string) { ctx := r.Context() log := appctx.GetLogger(ctx) - // fn := path.Join(ns, r.URL.Path) + // TODO(salfagem): catch empty request body rep, status, err := readReport(r.Body) if err != nil { log.Error().Err(err).Msg("error reading report") w.WriteHeader(status) return } + + log.Info().Msgf("hugo: searching in path: %s with pattern: %s", r.URL.Path, rep.SearchFiles.Search.Pattern) + if rep.SearchFiles != nil { s.doSearchFiles(w, r, rep.SearchFiles) return @@ -56,21 +62,98 @@ func (s *svc) handleReport(w http.ResponseWriter, r *http.Request, ns string) { return } - // TODO(jfd): implement report - w.WriteHeader(http.StatusNotImplemented) } func (s *svc) doSearchFiles(w http.ResponseWriter, r *http.Request, sf *reportSearchFiles) { ctx := r.Context() log := appctx.GetLogger(ctx) - _, err := s.getClient() + + log.Info().Msgf("hugo: search is: %+v", sf) + + client, err := s.getClient() if err != nil { - log.Error().Err(err).Msg("error getting grpc client") + log.Error().Err(err).Msg("search: error getting grpc client") w.WriteHeader(http.StatusInternalServerError) return } - w.WriteHeader(http.StatusNotImplemented) + + opaqueMap := map[string]*typespb.OpaqueEntry{ + "search": { + Decoder: "plain", + Value: []byte("search"), + }, + "searchString": { + Decoder: "plain", + Value: []byte(sf.Search.Pattern), + }, + } + + // TODO(salfagem): hardcoded path for the time being: + ref := &provider.Reference{Path: "/eos/project/c/cernbox"} + + req := &provider.ListContainerRequest{Opaque: &typespb.Opaque{ + Map: opaqueMap, + }, Ref: ref} + + res, err := client.ListContainer(ctx, req) + + if err != nil { + log.Error().Err(err).Msg("search: error listing container") + w.WriteHeader(http.StatusInternalServerError) + return + } + + for _, v := range res.Infos { + fmt.Println(v) + } + + data := ` + + + + /remote.php/dav/spaces/1284d238-aa92-42ce-bdc4-0b0000009157$4c510ada-c86b-4815-8820-42cdf82c3d51/asd.txt + + + 1284d238-aa92-42ce-bdc4-0b0000009157$4c510ada-c86b-4815-8820-42cdf82c3d51!d5613880-307c-4e3e-b56d-97839fcf6d03 + 1284d238-aa92-42ce-bdc4-0b0000009157$4c510ada-c86b-4815-8820-42cdf82c3d51!4c510ada-c86b-4815-8820-42cdf82c3d51 + asd.txt + 2022-11-08T10:45:16Z + text/plain + RDNVW + + + 0 + 0.4809828996658325 + + HTTP/1.1 200 OK + + + + /remote.php/dav/spaces/1284d238-aa92-42ce-bdc4-0b0000009157$4c510ada-c86b-4815-8820-42cdf82c3d51/asdddddd + + + 1284d238-aa92-42ce-bdc4-0b0000009157$4c510ada-c86b-4815-8820-42cdf82c3d51!fa3c7a85-5fc4-46d3-a0b1-293a282da1b7 + 1284d238-aa92-42ce-bdc4-0b0000009157$4c510ada-c86b-4815-8820-42cdf82c3d51!4c510ada-c86b-4815-8820-42cdf82c3d51 + asdddddd + 2022-11-08T10:45:32Z + httpd/unix-directory + RDNVCK + + + + + 0 + 0.4809828996658325 + + HTTP/1.1 200 OK + + + +` + w.Write([]byte(data)) + w.WriteHeader(207) + return } func (s *svc) doFilterFiles(w http.ResponseWriter, r *http.Request, ff *reportFilterFiles, namespace string) { @@ -148,7 +231,7 @@ type reportSearchFiles struct { Search reportSearchFilesSearch `xml:"search"` } type reportSearchFilesSearch struct { - Pattern string `xml:"search"` + Pattern string `xml:"pattern"` Limit int `xml:"limit"` Offset int `xml:"offset"` } @@ -166,34 +249,16 @@ type reportFilterFilesRules struct { } func readReport(r io.Reader) (rep *report, status int, err error) { - decoder := xml.NewDecoder(r) - rep = &report{} - for { - t, err := decoder.Token() - if err == io.EOF { - // io.EOF is a successful end - return rep, 0, nil - } - if err != nil { - return nil, http.StatusBadRequest, err - } + content, err := ioutil.ReadAll(r) + if err != nil { + return nil, 0, err + } - if v, ok := t.(xml.StartElement); ok { - if v.Name.Local == elementNameSearchFiles { - var repSF reportSearchFiles - err = decoder.DecodeElement(&repSF, &v) - if err != nil { - return nil, http.StatusBadRequest, err - } - rep.SearchFiles = &repSF - } else if v.Name.Local == elementNameFilterFiles { - var repFF reportFilterFiles - err = decoder.DecodeElement(&repFF, &v) - if err != nil { - return nil, http.StatusBadRequest, err - } - rep.FilterFiles = &repFF - } - } + s := &reportSearchFiles{Search: reportSearchFilesSearch{}} + err = xml.Unmarshal(content, s) + if err != nil { + return nil, 0, err } + + return &report{SearchFiles: s}, 0, err } diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index f61f724336..57795cf51d 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -27,6 +27,7 @@ import ( "os/exec" "path" "path/filepath" + "regexp" "strconv" "strings" "syscall" @@ -715,7 +716,32 @@ func (c *Client) List(ctx context.Context, auth eosclient.Authorization, path st return c.parseFind(ctx, auth, path, stdout) } -// Read reads a file from the mgm. +// List the contents of the directory given by path with depth infinity +func (c *Client) SearchDir(ctx context.Context, auth eosclient.Authorization, searchString string, path string) ([]*eosclient.FileInfo, error) { + // TODO(salfagem): path is truncated - i.e. /c/cernbox (not absolute) + args := []string{"find", "--fileinfo", "-name", searchString, path} + log := appctx.GetLogger(ctx) + log.Debug().Msgf("eosbinary search with args: %s", args) + // Safeguard #2 to prevent the search to go to undesired places: + if !strings.HasPrefix(path, "/eos/project/c/cernbox") { + log.Debug().Msgf("eosbinary - prefix doesn't match") + return nil, errors.Errorf("eosclient: search path out of bounds fn=%s", path) + } + // For the moment, just file names are supported, no paths (and no parentheses, braquets...) + searchStringMatch, _ := regexp.MatchString(`^[\w\d\s\*\-\.]+$`, searchString) + if !searchStringMatch { + log.Debug().Msgf("eosbinary - searchstring is not valid") + return nil, errors.Errorf("eosclient: ilegal search string: %s", searchString) + } + // TODO: set a timeout for the find in case it goes out of hand + stdout, _, err := c.executeEOS(ctx, args, auth) + if err != nil { + return nil, errors.Wrapf(err, "eosclient: error listing fn=%s", path) + } + return c.parseFind(ctx, auth, path, stdout) +} + +// Read reads a file from the mgm func (c *Client) Read(ctx context.Context, auth eosclient.Authorization, path string) (io.ReadCloser, error) { rand := "eosread-" + uuid.New().String() localTarget := fmt.Sprintf("%s/%s", c.opt.CacheDirectory, rand) diff --git a/pkg/eosclient/eosclient.go b/pkg/eosclient/eosclient.go index 7bf5fce242..9feaa9c210 100644 --- a/pkg/eosclient/eosclient.go +++ b/pkg/eosclient/eosclient.go @@ -49,6 +49,7 @@ type EOSClient interface { Remove(ctx context.Context, auth Authorization, path string, noRecycle bool) error Rename(ctx context.Context, auth Authorization, oldPath, newPath string) error List(ctx context.Context, auth Authorization, path string) ([]*FileInfo, error) + SearchDir(ctx context.Context, auth Authorization, searchString string, path string) ([]*FileInfo, error) Read(ctx context.Context, auth Authorization, path string) (io.ReadCloser, error) Write(ctx context.Context, auth Authorization, path string, stream io.ReadCloser) error WriteFile(ctx context.Context, auth Authorization, path, source string) error diff --git a/pkg/eosclient/eosgrpc/eosgrpc.go b/pkg/eosclient/eosgrpc/eosgrpc.go index ffc888106c..b04166e702 100644 --- a/pkg/eosclient/eosgrpc/eosgrpc.go +++ b/pkg/eosclient/eosgrpc/eosgrpc.go @@ -1221,6 +1221,10 @@ func (c *Client) List(ctx context.Context, auth eosclient.Authorization, dpath s return mylst, nil } +func (c *Client) SearchDir(ctx context.Context, auth eosclient.Authorization, searchString string, dpath string) ([]*eosclient.FileInfo, error) { + return nil, errtypes.NotSupported("eosgrpc: search is not implemented") +} + // Read reads a file from the mgm and returns a handle to read it // This handle could be directly the body of the response or a local tmp file // diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index d495f1e5f9..1e2e7d6282 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -1307,11 +1307,60 @@ func (fs *eosfs) getMDShareFolder(ctx context.Context, p string, mdKeys []string } func (fs *eosfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error) { + log := appctx.GetLogger(ctx) + p, err := fs.resolve(ctx, ref) + if err != nil { return nil, errors.Wrap(err, "eosfs: error resolving reference") } + p = fs.wrap(ctx, p) + + u, err := getUser(ctx) + if err != nil { + return nil, errors.Wrap(err, "eosfs: no user in ctx") + } + // lightweight accounts don't have share folders, so we're passing an empty string as path + auth, err := fs.getUserAuth(ctx, u, "") + if err != nil { + return nil, err + } + + searchString := "" + finfos := []*provider.ResourceInfo{} + + for i, key := range mdKeys { + if key == "search" { + // TODO(salfagem): Ugly hack due to mdKeys being an array. + // - also check that searchString not empty: + searchString = mdKeys[i+1] + + log.Info().Msgf("eosfs: running search: path=%s searchString=%s", p, searchString) + + eosFileInfos, err := fs.c.SearchDir(ctx, auth, searchString, p) + if err != nil { + return nil, errors.Wrap(err, "eosfs: error searching") + } + + for _, eosFileInfo := range eosFileInfos { + // filter out sys files + if !fs.conf.ShowHiddenSysFiles { + base := path.Base(eosFileInfo.File) + if hiddenReg.MatchString(base) { + continue + } + } + + if finfo, err := fs.convertToFileReference(ctx, eosFileInfo); err == nil { + finfos = append(finfos, finfo) + } + } + + return finfos, nil + } + } + if fs.conf.EnableHome { return fs.listWithHome(ctx, p) } From 974032ce718c53a07c4fe6b1f02b3b9e7342062e Mon Sep 17 00:00:00 2001 From: Samuel Alfageme Sainz Date: Mon, 5 Dec 2022 14:26:06 +0100 Subject: [PATCH 2/9] search: use newfind instead and catch errcodes 2/13 --- pkg/eosclient/eosbinary/eosbinary.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index 57795cf51d..7f840e5ec0 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -251,9 +251,10 @@ func (c *Client) executeEOS(ctx context.Context, cmdArgs []string, auth eosclien err = nil case int(syscall.ENOENT): err = errtypes.NotFound(errBuf.String()) - case int(syscall.EPERM), int(syscall.E2BIG), int(syscall.EINVAL): + case int(syscall.EPERM), int(syscall.E2BIG), int(syscall.EINVAL), int(syscall.EACCES): // eos reports back error code 1 (EPERM) when ? // eos reports back error code 7 (E2BIG) when the user is not allowed to read the directory + // eos reports back error code 13 (EACCES) wheb the user is not allowed to read the directory in e.g. eos newfind // eos reports back error code 22 (EINVAL) when the user is not allowed to enter the instance err = errtypes.PermissionDenied(errBuf.String()) } @@ -719,7 +720,7 @@ func (c *Client) List(ctx context.Context, auth eosclient.Authorization, path st // List the contents of the directory given by path with depth infinity func (c *Client) SearchDir(ctx context.Context, auth eosclient.Authorization, searchString string, path string) ([]*eosclient.FileInfo, error) { // TODO(salfagem): path is truncated - i.e. /c/cernbox (not absolute) - args := []string{"find", "--fileinfo", "-name", searchString, path} + args := []string{"newfind", "--fileinfo", "-name", searchString, path} log := appctx.GetLogger(ctx) log.Debug().Msgf("eosbinary search with args: %s", args) // Safeguard #2 to prevent the search to go to undesired places: @@ -736,7 +737,12 @@ func (c *Client) SearchDir(ctx context.Context, auth eosclient.Authorization, se // TODO: set a timeout for the find in case it goes out of hand stdout, _, err := c.executeEOS(ctx, args, auth) if err != nil { - return nil, errors.Wrapf(err, "eosclient: error listing fn=%s", path) + switch err.(type) { + case errtypes.NotFound, errtypes.PermissionDenied: + log.Debug().Msgf("eosbinary - user had insufficient permissions to search part of the directory") + default: + return nil, errors.Wrapf(err, "eosclient: error listing fn=%s", path) + } } return c.parseFind(ctx, auth, path, stdout) } From 7e5e172283d0abc3cf99c3ea5be306179ecf9b90 Mon Sep 17 00:00:00 2001 From: Samuel Alfageme Sainz Date: Mon, 5 Dec 2022 15:18:49 +0100 Subject: [PATCH 3/9] eosfs: wrap only inside the search --- pkg/storage/utils/eosfs/eosfs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 1e2e7d6282..53209e50fc 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -1315,8 +1315,6 @@ func (fs *eosfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys return nil, errors.Wrap(err, "eosfs: error resolving reference") } - p = fs.wrap(ctx, p) - u, err := getUser(ctx) if err != nil { return nil, errors.Wrap(err, "eosfs: no user in ctx") @@ -1332,6 +1330,8 @@ func (fs *eosfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys for i, key := range mdKeys { if key == "search" { + p = fs.wrap(ctx, p) + // TODO(salfagem): Ugly hack due to mdKeys being an array. // - also check that searchString not empty: searchString = mdKeys[i+1] From 7f4f29ff6606570e9d97ea985e636d40bba53489 Mon Sep 17 00:00:00 2001 From: Samuel Alfageme Sainz Date: Mon, 5 Dec 2022 15:50:25 +0100 Subject: [PATCH 4/9] search: initialize opaque if nil for ListContainer --- internal/grpc/services/storageprovider/storageprovider.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index cc7b8ccb8e..8afa6eb630 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -32,6 +32,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/mime" @@ -939,6 +940,10 @@ func (s *service) ListContainer(ctx context.Context, req *provider.ListContainer } // Ugly hack to enable search + if req.Opaque == nil { + req.Opaque = &typesv1beta1.Opaque{Map: make(map[string]*typesv1beta1.OpaqueEntry)} + } + if req.Opaque.Map["search"] != nil { s.storage.ListFolder(ctx, newRef, []string{"search", string(req.Opaque.Map["searchString"].GetValue())}) } From 799351d886e0f3ee5a5327545a4fe592e7243e5f Mon Sep 17 00:00:00 2001 From: Samuel Alfageme Sainz Date: Tue, 13 Dec 2022 12:08:02 +0100 Subject: [PATCH 5/9] report: log match path --- internal/http/services/owncloud/ocdav/report.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/http/services/owncloud/ocdav/report.go b/internal/http/services/owncloud/ocdav/report.go index 048af9b94b..7ec2123568 100644 --- a/internal/http/services/owncloud/ocdav/report.go +++ b/internal/http/services/owncloud/ocdav/report.go @@ -20,7 +20,6 @@ package ocdav import ( "encoding/xml" - "fmt" "io" "io/ioutil" "net/http" @@ -105,7 +104,7 @@ func (s *svc) doSearchFiles(w http.ResponseWriter, r *http.Request, sf *reportSe } for _, v := range res.Infos { - fmt.Println(v) + log.Debug().Msg(v.Path) } data := ` From 732bd8bf3997da2e376a1d9aa49b38fb7bb8883a Mon Sep 17 00:00:00 2001 From: Samuel Alfageme Sainz Date: Tue, 13 Dec 2022 13:55:16 +0100 Subject: [PATCH 6/9] eosbinary: change name parameter and sandbox project --- .../http/services/owncloud/ocdav/report.go | 177 +++++++++++++++--- pkg/eosclient/eosbinary/eosbinary.go | 9 +- pkg/storage/utils/eosfs/eosfs.go | 3 +- 3 files changed, 154 insertions(+), 35 deletions(-) diff --git a/internal/http/services/owncloud/ocdav/report.go b/internal/http/services/owncloud/ocdav/report.go index 7ec2123568..ad8e1a3743 100644 --- a/internal/http/services/owncloud/ocdav/report.go +++ b/internal/http/services/owncloud/ocdav/report.go @@ -89,7 +89,7 @@ func (s *svc) doSearchFiles(w http.ResponseWriter, r *http.Request, sf *reportSe } // TODO(salfagem): hardcoded path for the time being: - ref := &provider.Reference{Path: "/eos/project/c/cernbox"} + ref := &provider.Reference{Path: "/eos/project/t/test"} req := &provider.ListContainerRequest{Opaque: &typespb.Opaque{ Map: opaqueMap, @@ -103,52 +103,167 @@ func (s *svc) doSearchFiles(w http.ResponseWriter, r *http.Request, sf *reportSe return } + log.Debug().Msgf("search: found %d matches", len(res.Infos)) + for _, v := range res.Infos { log.Debug().Msg(v.Path) } data := ` - - - /remote.php/dav/spaces/1284d238-aa92-42ce-bdc4-0b0000009157$4c510ada-c86b-4815-8820-42cdf82c3d51/asd.txt + + /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/New%20Text%20Document.txt + + + newproject-a!420307062 + newproject-a!616510 + New Text Document.txt + 2022-12-05T16:12:20Z + text/plain + SRDNVCKZ + + + + + 0 + + HTTP/1.1 200 OK + + + + /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/New%20text%20file.txt - - 1284d238-aa92-42ce-bdc4-0b0000009157$4c510ada-c86b-4815-8820-42cdf82c3d51!d5613880-307c-4e3e-b56d-97839fcf6d03 - 1284d238-aa92-42ce-bdc4-0b0000009157$4c510ada-c86b-4815-8820-42cdf82c3d51!4c510ada-c86b-4815-8820-42cdf82c3d51 - asd.txt - 2022-11-08T10:45:16Z + + newproject-a!166399624 + newproject-a!616510 + New text file.txt + 2022-12-05T16:12:20Z text/plain - RDNVW + SRDNVCKZ - - 0 - 0.4809828996658325 - - HTTP/1.1 200 OK + + + + 5 + + HTTP/1.1 200 OK - - - /remote.php/dav/spaces/1284d238-aa92-42ce-bdc4-0b0000009157$4c510ada-c86b-4815-8820-42cdf82c3d51/asdddddd + + + /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/recover.txt - - 1284d238-aa92-42ce-bdc4-0b0000009157$4c510ada-c86b-4815-8820-42cdf82c3d51!fa3c7a85-5fc4-46d3-a0b1-293a282da1b7 - 1284d238-aa92-42ce-bdc4-0b0000009157$4c510ada-c86b-4815-8820-42cdf82c3d51!4c510ada-c86b-4815-8820-42cdf82c3d51 - asdddddd - 2022-11-08T10:45:32Z - httpd/unix-directory - RDNVCK + + newproject-a!493427700 + newproject-a!616510 + recover.txt + 2022-12-05T16:12:20Z + text/plain + SRDNVCKZ - + 0 - 0.4809828996658325 - - HTTP/1.1 200 OK + + HTTP/1.1 200 OK + + + + /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/test_support.txt + + + newproject-a!420300390 + newproject-a!616510 + test_support.txt + 2022-12-05T16:12:20Z + text/plain + SRDNVCKZ + + + + + 0 + + HTTP/1.1 200 OK + + + + /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/testing.txt + + + newproject-a!420300167 + newproject-a!616510 + testing.txt + 2022-12-05T16:12:20Z + text/plain + SRDNVCKZ + + + + + 0 + + HTTP/1.1 200 OK + + + + /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/testing2.txt + + + newproject-a!420300326 + newproject-a!616510 + testing2.txt + 2022-12-05T16:12:20Z + text/plain + SRDNVCKZ + + + + + 0 + + HTTP/1.1 200 OK + + + + /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/Denied%20Folder/New%20file.txt + + + newproject-a!437033942 + newproject-a!67942728 + New file.txt + 2022-12-05T16:12:20Z + text/plain + SRDNVCKZ + + + + + 0 + + HTTP/1.1 200 OK + + + + /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/test_versions/test.txt + + + newproject-a!506354963 + newproject-a!73758625 + test.txt + 2022-12-05T16:12:20Z + text/plain + SRDNVCKZ + + + + + 9 + + HTTP/1.1 200 OK - - + + ` w.Write([]byte(data)) w.WriteHeader(207) diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index 7f840e5ec0..74ba672b5d 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -720,11 +720,11 @@ func (c *Client) List(ctx context.Context, auth eosclient.Authorization, path st // List the contents of the directory given by path with depth infinity func (c *Client) SearchDir(ctx context.Context, auth eosclient.Authorization, searchString string, path string) ([]*eosclient.FileInfo, error) { // TODO(salfagem): path is truncated - i.e. /c/cernbox (not absolute) - args := []string{"newfind", "--fileinfo", "-name", searchString, path} + args := []string{"newfind", "--fileinfo", "--name", searchString, path} log := appctx.GetLogger(ctx) log.Debug().Msgf("eosbinary search with args: %s", args) // Safeguard #2 to prevent the search to go to undesired places: - if !strings.HasPrefix(path, "/eos/project/c/cernbox") { + if !strings.HasPrefix(path, "/eos/project/t/test") { log.Debug().Msgf("eosbinary - prefix doesn't match") return nil, errors.Errorf("eosclient: search path out of bounds fn=%s", path) } @@ -741,9 +741,12 @@ func (c *Client) SearchDir(ctx context.Context, auth eosclient.Authorization, se case errtypes.NotFound, errtypes.PermissionDenied: log.Debug().Msgf("eosbinary - user had insufficient permissions to search part of the directory") default: - return nil, errors.Wrapf(err, "eosclient: error listing fn=%s", path) + log.Error().Msgf("ERRROR LISTING= %s", stdout) + // There will be errors: + //return nil, errors.Wrapf(err, "eosclient: error listing fn=%s", path) } } + log.Debug().Msgf("eos find stdout= %s", stdout) return c.parseFind(ctx, auth, path, stdout) } diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 53209e50fc..511626f1c0 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -1336,7 +1336,7 @@ func (fs *eosfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys // - also check that searchString not empty: searchString = mdKeys[i+1] - log.Info().Msgf("eosfs: running search: path=%s searchString=%s", p, searchString) + log.Debug().Msgf("eosfs: running search: path=%s searchString=%s", p, searchString) eosFileInfos, err := fs.c.SearchDir(ctx, auth, searchString, p) if err != nil { @@ -1353,6 +1353,7 @@ func (fs *eosfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys } if finfo, err := fs.convertToFileReference(ctx, eosFileInfo); err == nil { + log.Debug().Msgf("eosfs: file name from search %s", finfo.Name) finfos = append(finfos, finfo) } } From d8a4e55b0cb852e6500ed4a296d05fb2af5f61ba Mon Sep 17 00:00:00 2001 From: Samuel Alfageme Sainz Date: Tue, 24 Jan 2023 12:22:39 +0100 Subject: [PATCH 7/9] search: report - ContentType: application/xml --- .../http/services/owncloud/ocdav/report.go | 158 +----------------- 1 file changed, 3 insertions(+), 155 deletions(-) diff --git a/internal/http/services/owncloud/ocdav/report.go b/internal/http/services/owncloud/ocdav/report.go index ad8e1a3743..ef82b63e0f 100644 --- a/internal/http/services/owncloud/ocdav/report.go +++ b/internal/http/services/owncloud/ocdav/report.go @@ -110,163 +110,11 @@ func (s *svc) doSearchFiles(w http.ResponseWriter, r *http.Request, sf *reportSe } data := ` - - - /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/New%20Text%20Document.txt - - - newproject-a!420307062 - newproject-a!616510 - New Text Document.txt - 2022-12-05T16:12:20Z - text/plain - SRDNVCKZ - - - - - 0 - - HTTP/1.1 200 OK - - - - /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/New%20text%20file.txt - - - newproject-a!166399624 - newproject-a!616510 - New text file.txt - 2022-12-05T16:12:20Z - text/plain - SRDNVCKZ - - - - - 5 - - HTTP/1.1 200 OK - - - - /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/recover.txt - - - newproject-a!493427700 - newproject-a!616510 - recover.txt - 2022-12-05T16:12:20Z - text/plain - SRDNVCKZ - - - - - 0 - - HTTP/1.1 200 OK - - - - /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/test_support.txt - - - newproject-a!420300390 - newproject-a!616510 - test_support.txt - 2022-12-05T16:12:20Z - text/plain - SRDNVCKZ - - - - - 0 - - HTTP/1.1 200 OK - - - - /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/testing.txt - - - newproject-a!420300167 - newproject-a!616510 - testing.txt - 2022-12-05T16:12:20Z - text/plain - SRDNVCKZ - - - - - 0 - - HTTP/1.1 200 OK - - - - /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/testing2.txt - - - newproject-a!420300326 - newproject-a!616510 - testing2.txt - 2022-12-05T16:12:20Z - text/plain - SRDNVCKZ - - - - - 0 - - HTTP/1.1 200 OK - - - - /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/Denied%20Folder/New%20file.txt - - - newproject-a!437033942 - newproject-a!67942728 - New file.txt - 2022-12-05T16:12:20Z - text/plain - SRDNVCKZ - - - - - 0 - - HTTP/1.1 200 OK - - - - /remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/test_versions/test.txt - - - newproject-a!506354963 - newproject-a!73758625 - test.txt - 2022-12-05T16:12:20Z - text/plain - SRDNVCKZ - - - - - 9 - - HTTP/1.1 200 OK - - - +/remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/New%20Text%20Document.txtnewproject-a!420307062newproject-a!616510New Text Document.txt2022-12-05T16:12:20Ztext/plainSRDNVCKZ0HTTP/1.1 200 OK/remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/New%20text%20file.txtnewproject-a!166399624newproject-a!616510New text file.txt2022-12-05T16:12:20Ztext/plainSRDNVCKZ5HTTP/1.1 200 OK/remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/recover.txtnewproject-a!493427700newproject-a!616510recover.txt2022-12-05T16:12:20Ztext/plainSRDNVCKZ0HTTP/1.1 200 OK/remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/test_support.txtnewproject-a!420300390newproject-a!616510test_support.txt2022-12-05T16:12:20Ztext/plainSRDNVCKZ0HTTP/1.1 200 OK/remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/testing.txtnewproject-a!420300167newproject-a!616510testing.txt2022-12-05T16:12:20Ztext/plainSRDNVCKZ0HTTP/1.1 200 OK/remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/testing2.txtnewproject-a!420300326newproject-a!616510testing2.txt2022-12-05T16:12:20Ztext/plainSRDNVCKZ0HTTP/1.1 200 OK/remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/Denied%20Folder/New%20file.txtnewproject-a!437033942newproject-a!67942728New file.txt2022-12-05T16:12:20Ztext/plainSRDNVCKZ0HTTP/1.1 200 OK/remote.php/dav/files/cboxbot/eos/project/a/awesomeproject/test_versions/test.txtnewproject-a!506354963newproject-a!73758625test.txt2022-12-05T16:12:20Ztext/plainSRDNVCKZ9HTTP/1.1 200 OK ` + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusMultiStatus) w.Write([]byte(data)) - w.WriteHeader(207) return } From d1f253060f286b8a13204082c6a92dc86c76ee22 Mon Sep 17 00:00:00 2001 From: Samuel Alfageme Sainz Date: Wed, 25 Jan 2023 11:18:13 +0100 Subject: [PATCH 8/9] search: log stderr from 'eos find' and replace mock response --- .../http/services/owncloud/ocdav/report.go | 14 ++++++------- pkg/eosclient/eosbinary/eosbinary.go | 20 ++++++++++--------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/internal/http/services/owncloud/ocdav/report.go b/internal/http/services/owncloud/ocdav/report.go index ef82b63e0f..edef86dde1 100644 --- a/internal/http/services/owncloud/ocdav/report.go +++ b/internal/http/services/owncloud/ocdav/report.go @@ -27,7 +27,7 @@ import ( rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" ctxpkg "github.com/cs3org/reva/pkg/ctx" ) @@ -49,7 +49,7 @@ func (s *svc) handleReport(w http.ResponseWriter, r *http.Request, ns string) { return } - log.Info().Msgf("hugo: searching in path: %s with pattern: %s", r.URL.Path, rep.SearchFiles.Search.Pattern) + log.Info().Msgf("searching in path: %s with pattern: %s", r.URL.Path, rep.SearchFiles.Search.Pattern) if rep.SearchFiles != nil { s.doSearchFiles(w, r, rep.SearchFiles) @@ -68,7 +68,7 @@ func (s *svc) doSearchFiles(w http.ResponseWriter, r *http.Request, sf *reportSe ctx := r.Context() log := appctx.GetLogger(ctx) - log.Info().Msgf("hugo: search is: %+v", sf) + log.Info().Msgf("search is: %+v", sf) client, err := s.getClient() if err != nil { @@ -77,7 +77,7 @@ func (s *svc) doSearchFiles(w http.ResponseWriter, r *http.Request, sf *reportSe return } - opaqueMap := map[string]*typespb.OpaqueEntry{ + opaqueMap := map[string]*types.OpaqueEntry{ "search": { Decoder: "plain", Value: []byte("search"), @@ -88,10 +88,10 @@ func (s *svc) doSearchFiles(w http.ResponseWriter, r *http.Request, sf *reportSe }, } - // TODO(salfagem): hardcoded path for the time being: - ref := &provider.Reference{Path: "/eos/project/t/test"} + // TODO(salfagem): hardcoded path for the time being, enable a list: + ref := &provider.Reference{Path: "/eos/project/a/awesomeproject"} - req := &provider.ListContainerRequest{Opaque: &typespb.Opaque{ + req := &provider.ListContainerRequest{Opaque: &types.Opaque{ Map: opaqueMap, }, Ref: ref} diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index 74ba672b5d..2217ce6d28 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -720,11 +720,11 @@ func (c *Client) List(ctx context.Context, auth eosclient.Authorization, path st // List the contents of the directory given by path with depth infinity func (c *Client) SearchDir(ctx context.Context, auth eosclient.Authorization, searchString string, path string) ([]*eosclient.FileInfo, error) { // TODO(salfagem): path is truncated - i.e. /c/cernbox (not absolute) - args := []string{"newfind", "--fileinfo", "--name", searchString, path} + args := []string{"find", "--fileinfo", "-name", searchString, path} log := appctx.GetLogger(ctx) log.Debug().Msgf("eosbinary search with args: %s", args) // Safeguard #2 to prevent the search to go to undesired places: - if !strings.HasPrefix(path, "/eos/project/t/test") { + if !strings.HasPrefix(path, "/eos/project/a/awesomeproject") { log.Debug().Msgf("eosbinary - prefix doesn't match") return nil, errors.Errorf("eosclient: search path out of bounds fn=%s", path) } @@ -735,15 +735,17 @@ func (c *Client) SearchDir(ctx context.Context, auth eosclient.Authorization, se return nil, errors.Errorf("eosclient: ilegal search string: %s", searchString) } // TODO: set a timeout for the find in case it goes out of hand - stdout, _, err := c.executeEOS(ctx, args, auth) + stdout, stderr, err := c.executeEOS(ctx, args, auth) if err != nil { - switch err.(type) { - case errtypes.NotFound, errtypes.PermissionDenied: + // Errcode 7: "TOOBIG" + if stdout != "" { + log.Debug().Msgf("eos find stdout= %s", stdout) + log.Debug().Msgf("=========== eos find stderr =========== %s", stderr) + return c.parseFind(ctx, auth, path, stdout) + } else { log.Debug().Msgf("eosbinary - user had insufficient permissions to search part of the directory") - default: - log.Error().Msgf("ERRROR LISTING= %s", stdout) - // There will be errors: - //return nil, errors.Wrapf(err, "eosclient: error listing fn=%s", path) + // There will be errors; we cannot ignore them: + return nil, errors.Wrapf(err, "eosclient: error listing fn=%s", path) } } log.Debug().Msgf("eos find stdout= %s", stdout) From 73a716d94231839634c6d5232df9a181dba6b08b Mon Sep 17 00:00:00 2001 From: Samuel Alfageme Sainz Date: Wed, 8 Feb 2023 16:46:00 +0100 Subject: [PATCH 9/9] search: fix search behavior, return match list --- pkg/eosclient/eosbinary/eosbinary.go | 15 ++------------- pkg/storage/utils/eosfs/eosfs.go | 26 ++++++++++++++++++-------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index 2217ce6d28..8c36061053 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -735,19 +735,8 @@ func (c *Client) SearchDir(ctx context.Context, auth eosclient.Authorization, se return nil, errors.Errorf("eosclient: ilegal search string: %s", searchString) } // TODO: set a timeout for the find in case it goes out of hand - stdout, stderr, err := c.executeEOS(ctx, args, auth) - if err != nil { - // Errcode 7: "TOOBIG" - if stdout != "" { - log.Debug().Msgf("eos find stdout= %s", stdout) - log.Debug().Msgf("=========== eos find stderr =========== %s", stderr) - return c.parseFind(ctx, auth, path, stdout) - } else { - log.Debug().Msgf("eosbinary - user had insufficient permissions to search part of the directory") - // There will be errors; we cannot ignore them: - return nil, errors.Wrapf(err, "eosclient: error listing fn=%s", path) - } - } + stdout, _, _ := c.executeEOS(ctx, args, auth) + // We can ignore errors and stderr, we're just interested on the EOS output log.Debug().Msgf("eos find stdout= %s", stdout) return c.parseFind(ctx, auth, path, stdout) } diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 511626f1c0..bcb80a57e3 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -1336,25 +1336,29 @@ func (fs *eosfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys // - also check that searchString not empty: searchString = mdKeys[i+1] + if searchString == "" { + return nil, errtypes.NotSupported("Search requires a search string") + } + log.Debug().Msgf("eosfs: running search: path=%s searchString=%s", p, searchString) eosFileInfos, err := fs.c.SearchDir(ctx, auth, searchString, p) + if err != nil { return nil, errors.Wrap(err, "eosfs: error searching") } for _, eosFileInfo := range eosFileInfos { - // filter out sys files - if !fs.conf.ShowHiddenSysFiles { - base := path.Base(eosFileInfo.File) - if hiddenReg.MatchString(base) { - continue - } - } - if finfo, err := fs.convertToFileReference(ctx, eosFileInfo); err == nil { + log.Debug().Msgf("seach: eosFileInfo %s", eosFileInfo.File) + + // Search all files, hidden and not, for the time being + + if finfo, err := fs.convertToResourceInfo(ctx, eosFileInfo); err == nil { log.Debug().Msgf("eosfs: file name from search %s", finfo.Name) finfos = append(finfos, finfo) + } else { + log.Error().Err(err).Msg(" wtf 🥘") } } @@ -2120,6 +2124,8 @@ func (fs *eosfs) convertToResourceInfo(ctx context.Context, eosFileInfo *eosclie } func (fs *eosfs) convertToFileReference(ctx context.Context, eosFileInfo *eosclient.FileInfo) (*provider.ResourceInfo, error) { + log := appctx.GetLogger(ctx) + log.Debug().Msg("convertToFileReference") info, err := fs.convert(ctx, eosFileInfo) if err != nil { return nil, err @@ -2258,6 +2264,8 @@ func mergePermissions(l *provider.ResourcePermissions, r *provider.ResourcePermi } func (fs *eosfs) convert(ctx context.Context, eosFileInfo *eosclient.FileInfo) (*provider.ResourceInfo, error) { + log := appctx.GetLogger(ctx) + log.Debug().Msg("convert") path, err := fs.unwrap(ctx, eosFileInfo.File) if err != nil { return nil, err @@ -2293,6 +2301,8 @@ func (fs *eosfs) convert(ctx context.Context, eosFileInfo *eosclient.FileInfo) ( } } + log.Debug().Msgf("convert: marshalling the eosFileInfo %s", eosFileInfo) + info := &provider.ResourceInfo{ Id: &provider.ResourceId{OpaqueId: fmt.Sprintf("%d", eosFileInfo.Inode)}, Path: path,