diff --git a/cmd/reva/upload.go b/cmd/reva/upload.go index d6a3d1c9ab..1040708ffd 100644 --- a/cmd/reva/upload.go +++ b/cmd/reva/upload.go @@ -22,12 +22,18 @@ import ( "fmt" "io" "math" - "net/http" "os" + "path/filepath" + "strconv" "github.com/cheggaaa/pb" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + + tokenpkg "github.com/cs3org/reva/pkg/token" + "github.com/eventials/go-tus" + "github.com/eventials/go-tus/memorystore" // TODO(labkode): this should not come from this package. "github.com/cs3org/reva/internal/grpc/services/storageprovider" @@ -65,7 +71,7 @@ func uploadCommand() *command { fmt.Printf("Local file size: %d bytes\n", md.Size()) - client, err := getClient() + gwc, err := getClient() if err != nil { return err } @@ -76,9 +82,17 @@ func uploadCommand() *command { Path: target, }, }, + Opaque: &typespb.Opaque{ + Map: map[string]*typespb.OpaqueEntry{ + "Upload-Length": { + Decoder: "plain", + Value: []byte(strconv.FormatInt(md.Size(), 10)), + }, + }, + }, } - res, err := client.InitiateFileUpload(ctx, req) + res, err := gwc.InitiateFileUpload(ctx, req) if err != nil { return err } @@ -109,35 +123,54 @@ func uploadCommand() *command { } dataServerURL := res.UploadEndpoint - bar := pb.New(int(md.Size())).SetUnits(pb.U_BYTES) - bar.Start() - reader := bar.NewProxyReader(fd) - httpReq, err := rhttp.NewRequest(ctx, "PUT", dataServerURL, reader) + // create the tus client. + c := tus.DefaultConfig() + c.Resume = true + c.HttpClient = rhttp.GetHTTPClient(ctx) + c.Store, err = memorystore.NewMemoryStore() + if err != nil { + return err + } + if res.Token != "" { + fmt.Printf("using X-Reva-Transfer header\n") + c.Header.Add("X-Reva-Transfer", res.Token) + } else if token, ok := tokenpkg.ContextGetToken(ctx); ok { + fmt.Printf("using %s header\n", tokenpkg.TokenHeader) + c.Header.Add(tokenpkg.TokenHeader, token) + } + tusc, err := tus.NewClient(dataServerURL, c) if err != nil { return err } - httpReq.Header.Set("X-Reva-Transfer", res.Token) - q := httpReq.URL.Query() - q.Add("xs", xs) - q.Add("xs_type", storageprovider.GRPC2PKGXS(xsType).String()) - httpReq.URL.RawQuery = q.Encode() + metadata := map[string]string{ + "filename": filepath.Base(target), + "dir": filepath.Dir(target), + "checksum": fmt.Sprintf("%s %s", storageprovider.GRPC2PKGXS(xsType).String(), xs), + } + + fingerprint := fmt.Sprintf("%s-%d-%s-%s", md.Name(), md.Size(), md.ModTime(), xs) + + bar := pb.New(int(md.Size())).SetUnits(pb.U_BYTES) + bar.Start() + reader := bar.NewProxyReader(fd) + + // create an upload from a file. + upload := tus.NewUpload(reader, md.Size(), metadata, fingerprint) - httpClient := rhttp.GetHTTPClient(ctx) + // create the uploader. + c.Store.Set(upload.Fingerprint, dataServerURL) + uploader := tus.NewUploader(tusc, dataServerURL, upload, 0) - httpRes, err := httpClient.Do(httpReq) + // start the uploading process. + err = uploader.Upload() if err != nil { return err } - defer httpRes.Body.Close() bar.Finish() - if httpRes.StatusCode != http.StatusOK { - return err - } - req2 := &provider.StatRequest{ Ref: &provider.Reference{ Spec: &provider.Reference_Path{ @@ -145,7 +178,7 @@ func uploadCommand() *command { }, }, } - res2, err := client.Stat(ctx, req2) + res2, err := gwc.Stat(ctx, req2) if err != nil { return err } diff --git a/go.mod b/go.mod index 3b823f14e2..14e17323c9 100644 --- a/go.mod +++ b/go.mod @@ -12,13 +12,14 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/go-cs3apis v0.0.0-20200422090600-d9e5166bebfe github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/eventials/go-tus v0.0.0-20190617130015-9db47421f6a0 github.com/fatih/color v1.7.0 // indirect github.com/go-openapi/strfmt v0.19.2 // indirect github.com/gofrs/uuid v3.2.0+incompatible github.com/golang/protobuf v1.3.5 github.com/gomodule/redigo v2.0.0+incompatible github.com/google/uuid v1.1.1 - github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 + github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 github.com/huandu/xstrings v1.3.0 // indirect github.com/imdario/mergo v0.3.8 // indirect github.com/jedib0t/go-pretty v4.3.0+incompatible @@ -31,12 +32,12 @@ require ( github.com/pkg/errors v0.9.1 github.com/pkg/xattr v0.4.1 github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect - github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 // indirect github.com/rs/cors v1.7.0 github.com/rs/zerolog v1.18.0 + github.com/tus/tusd v1.1.1-0.20200416115059-9deabf9d80c2 go.opencensus.io v0.22.3 golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a - golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 + golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 google.golang.org/grpc v1.29.0 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/cheggaaa/pb.v1 v1.0.27 // indirect @@ -45,3 +46,5 @@ require ( ) go 1.13 + +replace github.com/eventials/go-tus => github.com/andrewmostello/go-tus v0.0.0-20200314041820-904a9904af9a diff --git a/go.sum b/go.sum index b29ec818e5..af00f0474c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro= contrib.go.opencensus.io/exporter/jaeger v0.2.0 h1:nhTv/Ry3lGmqbJ/JGvCjWxBl5ozRfqo86Ngz59UAlfk= contrib.go.opencensus.io/exporter/jaeger v0.2.0/go.mod h1:ukdzwIYYHgZ7QYtwVFQUjiT28BJHiMhTERo32s6qVgM= contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg= @@ -16,24 +18,19 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5Vpd github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andrewmostello/go-tus v0.0.0-20200314041820-904a9904af9a h1:6tD4saJb8wmYF6Llz96ZJwUQ5r2GyTBFA2VEB5z8gVY= +github.com/andrewmostello/go-tus v0.0.0-20200314041820-904a9904af9a/go.mod h1:XYuK1S5+kS6FGhlIUFuZFPvWiSrOIoLk6+ro33Xce3Y= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.30.1 h1:cUMxtoFvIHhScZgv17tGxw15r6rVKJHR1hsIFRx9hcA= -github.com/aws/aws-sdk-go v1.30.1/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.30.2 h1:0vuroAsbPwVbP91MMaUmFLnrQcFBhmjQnnXaH1kcnPw= -github.com/aws/aws-sdk-go v1.30.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.30.3 h1:tmaR+qpBSig6RfhP9IoxALJEE1m0vfLy5tlnEIXu6WI= -github.com/aws/aws-sdk-go v1.30.3/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.30.4 h1:dpQgypC3rld2Uuz+/2u+0nbfmmyEWxau6v1hdAlvoc8= -github.com/aws/aws-sdk-go v1.30.4/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.30.7 h1:IaXfqtioP6p9SFAnNfsqdNczbR5UNbYqvcZUSsCAdTY= -github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.20.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.30.12 h1:KrjyosZvkpJjcwMk0RNxMZewQ47v7+ZkbQDXjWsJMs8= github.com/aws/aws-sdk-go v1.30.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cheggaaa/pb v1.0.28 h1:kWGpdAcSp3MxMU9CCHOwz/8V0kCHN4+9yQm2MzWuI98= github.com/cheggaaa/pb v1.0.28/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= @@ -42,12 +39,6 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/cs3org/go-cs3apis v0.0.0-20200324115356-e04b4fd75f03 h1:JGANezYNs/VM1Mpqu/noYqAT8HzsheysyArEaz4OnwM= -github.com/cs3org/go-cs3apis v0.0.0-20200324115356-e04b4fd75f03/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/go-cs3apis v0.0.0-20200403073321-0519c6823b48 h1:iR2JGsyZwRg6N1k9QuCJhXRH4CQUnCZNoJ0VmnjF2xE= -github.com/cs3org/go-cs3apis v0.0.0-20200403073321-0519c6823b48/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/go-cs3apis v0.0.0-20200408065125-6e23f3ecec0a h1:+enFdliTCV/aaLAvLmeka/r9wUoEypngV4JD5Gy92Uc= -github.com/cs3org/go-cs3apis v0.0.0-20200408065125-6e23f3ecec0a/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cs3org/go-cs3apis v0.0.0-20200422090600-d9e5166bebfe h1:gQgullcz2/nn1epVPhX3805nszWkSolM8m6CYLF9Y04= github.com/cs3org/go-cs3apis v0.0.0-20200422090600-d9e5166bebfe/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -62,6 +53,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-openapi/errors v0.19.2 h1:a2kIyV3w+OS3S97zxUndRVD46+FhGOUBDFY7nmu4CsY= @@ -74,12 +66,15 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -90,32 +85,44 @@ github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/gopherjs/gopherjs v0.0.0-20181004151105-1babbf986f6f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.0 h1:gvV6jG9dTgFEncxo+AF7PH6MZXi/vZl25owA/8Dg8Wo= github.com/huandu/xstrings v1.3.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo= github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -135,12 +142,19 @@ github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/moul/http2curl v0.0.0-20170919181001-9ac6cf4d929b/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/oleiade/reflections v1.0.0 h1:0ir4pc6v8/PJ0yw5AEtMddfXpWBXg9cnG7SgSoJuCgY= github.com/oleiade/reflections v1.0.0/go.mod h1:RbATFBbKYkVdqmSFtx13Bb/tVhR0lgOBXunWTZKeL4w= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/ory/fosite v0.31.0 h1:NZ0FA4ywPEYrCGLNVBAz2dq8vTacLDbbO4Iiy68WCKQ= github.com/ory/fosite v0.31.0/go.mod h1:lSSqjo8Kr/U1P3kJWxsNGHmq7TnH/7pS1ijvQRT7G+g= github.com/ory/go-convenience v0.1.0 h1:zouLKfF2GoSGnJwGq+PE/nJAE6dj2Zj5QlTgmMTsTS8= @@ -162,29 +176,27 @@ github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prY github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 h1:D+CiwcpGTW6pL6bv6KI3KbyEyCKyS+1JWS2h8PNDnGA= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f h1:BVwpUVJDADN2ufcGik7W992pyps0wZ888b/y9GXcLTU= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8= github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I= +github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0/go.mod h1:Ad7IjTpvzZO8Fl0vh9AzQ+j/jYZfyp2diGwI8m5q+ns= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -198,16 +210,25 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tus/tusd v1.1.0 h1:y2oBFGeOyqlGgyqD0CloH8FuBrjDk0Tq1IQWvAZnyG8= +github.com/tus/tusd v1.1.0/go.mod h1:3DWPOdeCnjBwKtv98y5dSws3itPqfce5TVa0s59LRiA= +github.com/tus/tusd v1.1.1-0.20200416115059-9deabf9d80c2 h1:rcji4q9wMuSrz0tZt3kgIr/3WsB5kUqFja6RrgeCGEo= +github.com/tus/tusd v1.1.1-0.20200416115059-9deabf9d80c2/go.mod h1:ygrT4B9ZSb27dx3uTnobX5nOFDnutBL6iWKLH4+KpA0= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/vimeo/go-util v1.2.0/go.mod h1:s13SMDTSO7AjH1nbgp707mfN5JFIWUFDU5MDDuRRtKs= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a h1:Igim7XhdOpBnWPuYJ70XcNpq8q3BCACtVgNfoJxOV7g= @@ -217,15 +238,17 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -235,6 +258,7 @@ golang.org/x/oauth2 v0.0.0-20181003184128-c57b0facaced h1:4oqSq7eft7MdPKBGQK11X9 golang.org/x/oauth2 v0.0.0-20181003184128-c57b0facaced/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -243,40 +267,51 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEha golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181021155630-eda9bb28ed51/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190415081028-16da32be82c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd h1:r7DufRZuZbWB7j439YfAzP8RPDa9unLkpwQKUYbIMPI= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0 h1:KKgc1aqhV8wDPbDzlDtpvyjZFY3vjz85FP7p4wcQUyI= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107 h1:xtNn7qFlagY2mQNFHMSRPjT2RkOV4OXM7P5TVy9xATo= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb h1:i1Ppqkc3WQXikh8bXiwHqAN5Rv3/qDCcRk0/Otx73BY= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -286,29 +321,26 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.28.1 h1:C1QC6KzgSiLyBabDi87BbjaGreoRgGUF5nOyvfrAZ1k= -google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.0 h1:2pJjwYOdkZ9HlN4sWRYBg9ttH5bCOlsueaM+b/oYjwo= google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +gopkg.in/Acconut/lockfile.v1 v1.1.0/go.mod h1:6UCz3wJ8tSFUsPR6uP/j8uegEtDuEEqFxlpi0JI4Umw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.27 h1:kJdccidYzt3CaHD1crCFTS1hxyhSi059NhOFUf03YFo= gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/ldap.v2 v2.5.1 h1:wiu0okdNfjlBzg6UWvd1Hn8Y+Ux17/u/4nlk4CQr6tU= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/ldap.v2 v2.5.1/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk= gopkg.in/square/go-jose.v2 v2.1.9/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index e767d3a99d..a7ddfb4e3b 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -24,6 +24,7 @@ import ( "net/url" "os" "path" + "strconv" "strings" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -243,14 +244,30 @@ func (s *service) InitiateFileDownload(ctx context.Context, req *provider.Initia func (s *service) InitiateFileUpload(ctx context.Context, req *provider.InitiateFileUploadRequest) (*provider.InitiateFileUploadResponse, error) { // TODO(labkode): same considerations as download log := appctx.GetLogger(ctx) - url := *s.dataServerURL newRef, err := s.unwrap(ctx, req.Ref) if err != nil { return &provider.InitiateFileUploadResponse{ Status: status.NewInternal(ctx, err, "error unwrapping path"), }, nil } - url.Path = path.Join("/", url.Path, newRef.GetPath()) + var uploadLength int64 + if req.Opaque != nil && req.Opaque.Map != nil && req.Opaque.Map["Upload-Length"] != nil { + var err error + uploadLength, err = strconv.ParseInt(string(req.Opaque.Map["Upload-Length"].Value), 10, 64) + if err != nil { + return &provider.InitiateFileUploadResponse{ + Status: status.NewInternal(ctx, err, "error parsing upload length"), + }, nil + } + } + uploadID, err := s.storage.InitiateUpload(ctx, newRef, uploadLength) + if err != nil { + return &provider.InitiateFileUploadResponse{ + Status: status.NewInternal(ctx, err, "error getting upload id"), + }, nil + } + url := *s.dataServerURL + url.Path = path.Join("/", url.Path, uploadID) log.Info().Str("data-server", url.String()). Str("fn", req.Ref.GetPath()). Str("xs", fmt.Sprintf("%+v", s.conf.AvailableXS)). diff --git a/internal/http/interceptors/cors/cors.go b/internal/http/interceptors/cors/cors.go index 67e2e12853..ac150d8e3a 100644 --- a/internal/http/interceptors/cors/cors.go +++ b/internal/http/interceptors/cors/cors.go @@ -61,11 +61,54 @@ func New(m map[string]interface{}) (global.Middleware, int, error) { } if len(conf.AllowedMethods) == 0 { - conf.AllowedMethods = []string{"OPTIONS", "GET", "PUT", "POST", "DELETE", "MKCOL", "PROPFIND", "PROPPATCH", "MOVE", "COPY", "REPORT", "SEARCH"} + conf.AllowedMethods = []string{ + "OPTIONS", + "HEAD", + "GET", + "PUT", + "POST", + "DELETE", + "MKCOL", + "PROPFIND", + "PROPPATCH", + "MOVE", + "COPY", + "REPORT", + "SEARCH", + } } if len(conf.AllowedHeaders) == 0 { - conf.AllowedHeaders = []string{"Origin", "Accept", "Content-Type", "Depth", "Authorization", "Ocs-Apirequest", "If-None-Match", "If-Match", "Destination", "Overwrite", "X-Request-Id", "X-Requested-With"} + conf.AllowedHeaders = []string{ + "Origin", + "Accept", + "Content-Type", + "Depth", + "Authorization", + "Ocs-Apirequest", + "If-None-Match", + "If-Match", + "Destination", + "Overwrite", + "X-Request-Id", + "X-Requested-With", + "Tus-Resumable", + "Tus-Checksum-Algorithm", + "Upload-Concat", + "Upload-Length", + "Upload-Metadata", + "Upload-Defer-Length", + "Upload-Expires", + "Upload-Checksum", + "Upload-Offset", + "X-HTTP-Method-Override", + } + } + + if len(conf.ExposedHeaders) == 0 { + conf.ExposedHeaders = []string{ + "Location", + } } // TODO(jfd): use log from request context, otherwise fmt will be used to log, diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index 831aa23622..519755283e 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -21,13 +21,13 @@ package dataprovider import ( "fmt" "net/http" - "os" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + tusd "github.com/tus/tusd/pkg/handler" ) func init() { @@ -35,10 +35,9 @@ func init() { } type config struct { - Prefix string `mapstructure:"prefix"` - Driver string `mapstructure:"driver"` - TmpFolder string `mapstructure:"tmp_folder"` - Drivers map[string]map[string]interface{} `mapstructure:"drivers"` + Prefix string `mapstructure:"prefix"` + Driver string `mapstructure:"driver"` + Drivers map[string]map[string]interface{} `mapstructure:"drivers"` } type svc struct { @@ -58,14 +57,6 @@ func New(m map[string]interface{}) (global.Service, error) { conf.Prefix = "data" } - if conf.TmpFolder == "" { - conf.TmpFolder = os.TempDir() - } - - if err := os.MkdirAll(conf.TmpFolder, 0755); err != nil { - return nil, errors.Wrap(err, "could not create tmp dir") - } - fs, err := getFS(conf) if err != nil { return nil, err @@ -75,8 +66,8 @@ func New(m map[string]interface{}) (global.Service, error) { storage: fs, conf: conf, } - s.setHandler() - return s, nil + err = s.setHandler() + return s, err } // Close performs cleanup. @@ -88,6 +79,12 @@ func (s *svc) Unprotected() []string { return []string{} } +// Create a new DataStore instance which is responsible for +// storing the uploaded file on disk in the specified directory. +// This path _must_ exist before tusd will store uploads in it. +// If you want to save them on a different medium, for example +// a remote FTP server, you can implement your own storage backend +// by implementing the tusd.DataStore interface. func getFS(c *config) (storage.FS, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { return f(c.Drivers[c.Driver]) @@ -103,29 +100,80 @@ func (s *svc) Handler() http.Handler { return s.handler } -func (s *svc) setHandler() { - s.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case "HEAD": - addCorsHeader(w) - w.WriteHeader(http.StatusOK) - return - case "GET": - s.doGet(w, r) - return - case "PUT": - s.doPut(w, r) - return - default: - w.WriteHeader(http.StatusNotImplemented) - return - } - }) +// Composable is the interface that a struct needs to implement to be composable by this composer +type Composable interface { + UseIn(composer *tusd.StoreComposer) } -func addCorsHeader(res http.ResponseWriter) { - headers := res.Header() - headers.Set("Access-Control-Allow-Origin", "*") - headers.Set("Access-Control-Allow-Headers", "Content-Type, Origin, Authorization") - headers.Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS, HEAD") +func (s *svc) setHandler() (err error) { + composable, ok := s.storage.(Composable) + if ok { + // A storage backend for tusd may consist of multiple different parts which + // handle upload creation, locking, termination and so on. The composer is a + // place where all those separated pieces are joined together. In this example + // we only use the file store but you may plug in multiple. + composer := tusd.NewStoreComposer() + + // let the composable storage tell tus which extensions it supports + composable.UseIn(composer) + + config := tusd.Config{ + BasePath: s.conf.Prefix, + StoreComposer: composer, + //Logger: logger, // TODO use logger + } + + handler, err := tusd.NewUnroutedHandler(config) + if err != nil { + return err + } + + s.handler = handler.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + log := appctx.GetLogger(r.Context()) + log.Info().Msgf("tusd routing: path=%s", r.URL.Path) + + switch r.Method { + // old fashioned download. + + // GET is not part of the tus.io protocol + // currently there is no way to GET an upload that is in progress + // TODO allow range based get requests? that end before the current offset + case "GET": + s.doGet(w, r) + + // tus.io based upload + + // uploads are initiated using the CS3 APIs Initiate Download call + case "POST": + handler.PostFile(w, r) + case "HEAD": + handler.HeadFile(w, r) + case "PATCH": + handler.PatchFile(w, r) + // TODO Only attach the DELETE handler if the Terminate() method is provided + case "DELETE": + handler.DelFile(w, r) + } + })) + } else { + s.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case "HEAD": + w.WriteHeader(http.StatusOK) + return + case "GET": + s.doGet(w, r) + return + case "PUT": + s.doPut(w, r) + return + default: + w.WriteHeader(http.StatusNotImplemented) + return + } + }) + } + + return err } diff --git a/internal/http/services/owncloud/ocdav/copy.go b/internal/http/services/owncloud/ocdav/copy.go index d517415250..3308723228 100644 --- a/internal/http/services/owncloud/ocdav/copy.go +++ b/internal/http/services/owncloud/ocdav/copy.go @@ -36,6 +36,9 @@ import ( func (s *svc) handleCopy(w http.ResponseWriter, r *http.Request, ns string) { ctx := r.Context() log := appctx.GetLogger(ctx) + + ns = applyLayout(ctx, ns) + src := path.Join(ns, r.URL.Path) dstHeader := r.Header.Get("Destination") overwrite := r.Header.Get("Overwrite") diff --git a/internal/http/services/owncloud/ocdav/delete.go b/internal/http/services/owncloud/ocdav/delete.go index aa48ced3c7..9816e9a364 100644 --- a/internal/http/services/owncloud/ocdav/delete.go +++ b/internal/http/services/owncloud/ocdav/delete.go @@ -30,6 +30,9 @@ import ( func (s *svc) handleDelete(w http.ResponseWriter, r *http.Request, ns string) { ctx := r.Context() log := appctx.GetLogger(ctx) + + ns = applyLayout(ctx, ns) + fn := path.Join(ns, r.URL.Path) client, err := s.getClient() diff --git a/internal/http/services/owncloud/ocdav/get.go b/internal/http/services/owncloud/ocdav/get.go index 1d6efa8473..4d2cc28a09 100644 --- a/internal/http/services/owncloud/ocdav/get.go +++ b/internal/http/services/owncloud/ocdav/get.go @@ -34,6 +34,9 @@ import ( func (s *svc) handleGet(w http.ResponseWriter, r *http.Request, ns string) { ctx := r.Context() log := appctx.GetLogger(ctx) + + ns = applyLayout(ctx, ns) + fn := path.Join(ns, r.URL.Path) client, err := s.getClient() diff --git a/internal/http/services/owncloud/ocdav/head.go b/internal/http/services/owncloud/ocdav/head.go index 97d970dfe7..d40bdee9a5 100644 --- a/internal/http/services/owncloud/ocdav/head.go +++ b/internal/http/services/owncloud/ocdav/head.go @@ -32,6 +32,9 @@ import ( func (s *svc) handleHead(w http.ResponseWriter, r *http.Request, ns string) { ctx := r.Context() log := appctx.GetLogger(ctx) + + ns = applyLayout(ctx, ns) + fn := path.Join(ns, r.URL.Path) client, err := s.getClient() diff --git a/internal/http/services/owncloud/ocdav/mkcol.go b/internal/http/services/owncloud/ocdav/mkcol.go index 68df0f861a..6c9e26207f 100644 --- a/internal/http/services/owncloud/ocdav/mkcol.go +++ b/internal/http/services/owncloud/ocdav/mkcol.go @@ -31,6 +31,9 @@ import ( func (s *svc) handleMkcol(w http.ResponseWriter, r *http.Request, ns string) { ctx := r.Context() log := appctx.GetLogger(ctx) + + ns = applyLayout(ctx, ns) + fn := path.Join(ns, r.URL.Path) buf := make([]byte, 1) diff --git a/internal/http/services/owncloud/ocdav/move.go b/internal/http/services/owncloud/ocdav/move.go index a487b2ebcd..bc28c9a2a6 100644 --- a/internal/http/services/owncloud/ocdav/move.go +++ b/internal/http/services/owncloud/ocdav/move.go @@ -32,6 +32,9 @@ import ( func (s *svc) handleMove(w http.ResponseWriter, r *http.Request, ns string) { ctx := r.Context() log := appctx.GetLogger(ctx) + + ns = applyLayout(ctx, ns) + src := path.Join(ns, r.URL.Path) dstHeader := r.Header.Get("Destination") overwrite := r.Header.Get("Overwrite") diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go index bb36032222..192bf8ef76 100644 --- a/internal/http/services/owncloud/ocdav/ocdav.go +++ b/internal/http/services/owncloud/ocdav/ocdav.go @@ -34,6 +34,8 @@ import ( "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/storage/templates" + ctxuser "github.com/cs3org/reva/pkg/user" "github.com/mitchellh/mapstructure" ) @@ -49,8 +51,16 @@ func init() { // Config holds the config options that need to be passed down to all ocdav handlers type Config struct { - Prefix string `mapstructure:"prefix"` - FilesNamespace string `mapstructure:"files_namespace"` + Prefix string `mapstructure:"prefix"` + // FilesNamespace prefixes the namespace, optionally with user information. + // Example: if FilesNamespace is /users/{{substr 0 1 .Username}}/{{.Username}} + // and received path is /docs the internal path will be: + // /users///docs + FilesNamespace string `mapstructure:"files_namespace"` + // WebdavNamespace prefixes the namespace, optionally with user information. + // Example: if WebdavNamespace is /users/{{substr 0 1 .Username}}/{{.Username}} + // and received path is /docs the internal path will be: + // /users///docs WebdavNamespace string `mapstructure:"webdav_namespace"` ChunkFolder string `mapstructure:"chunk_folder"` GatewaySvc string `mapstructure:"gatewaysvc"` @@ -170,6 +180,10 @@ func (s *svc) getClient() (gateway.GatewayAPIClient, error) { return pool.GetGatewayServiceClient(s.c.GatewaySvc) } +func applyLayout(ctx context.Context, ns string) string { + return templates.WithUser(ctxuser.ContextMustGetUser(ctx), ns) +} + func wrapResourceID(r *provider.ResourceId) string { return wrap(r.StorageId, r.OpaqueId) } diff --git a/internal/http/services/owncloud/ocdav/options.go b/internal/http/services/owncloud/ocdav/options.go index 1de5e9e115..b0ef55812d 100644 --- a/internal/http/services/owncloud/ocdav/options.go +++ b/internal/http/services/owncloud/ocdav/options.go @@ -27,9 +27,15 @@ func (s *svc) handleOptions(w http.ResponseWriter, r *http.Request, ns string) { allow += " MOVE, UNLOCK, PROPFIND, MKCOL, REPORT, SEARCH," allow += " PUT" // TODO(jfd): only for files ... but we cannot create the full path without a user ... which we only have when credentials are sent + w.Header().Add("Access-Control-Allow-Headers", "Tus-Resumable") + w.Header().Add("Access-Control-Expose-Headers", "Tus-Resumable, Tus-Version, Tus-Extension") + w.Header().Set("Content-Type", "application/xml") w.Header().Set("Allow", allow) w.Header().Set("DAV", "1, 2") w.Header().Set("MS-Author-Via", "DAV") + w.Header().Set("Tus-Resumable", "1.0.0") // TODO(jfd): only for dirs? + w.Header().Set("Tus-Version", "1.0.0") + w.Header().Set("Tus-Extension", "creation,creation-with-upload") w.WriteHeader(http.StatusOK) } diff --git a/internal/http/services/owncloud/ocdav/propfind.go b/internal/http/services/owncloud/ocdav/propfind.go index cc3fa605e0..c6b452b6a4 100644 --- a/internal/http/services/owncloud/ocdav/propfind.go +++ b/internal/http/services/owncloud/ocdav/propfind.go @@ -46,6 +46,8 @@ func (s *svc) handlePropfind(w http.ResponseWriter, r *http.Request, ns string) defer span.End() log := appctx.GetLogger(ctx) + ns = applyLayout(ctx, ns) + fn := path.Join(ns, r.URL.Path) listChildren := r.Header.Get("Depth") != "0" @@ -112,6 +114,13 @@ func (s *svc) handlePropfind(w http.ResponseWriter, r *http.Request, ns string) } w.Header().Set("DAV", "1, 3, extended-mkcol") w.Header().Set("Content-Type", "application/xml; charset=utf-8") + // let clients know this collection supports tus.io POST requests to start uploads + if info.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER { + w.Header().Add("Access-Control-Expose-Headers", "Tus-Resumable, Tus-Version, Tus-Extension") + w.Header().Set("Tus-Resumable", "1.0.0") + w.Header().Set("Tus-Version", "1.0.0") + w.Header().Set("Tus-Extension", "creation,creation-with-upload") + } w.WriteHeader(http.StatusMultiStatus) if _, err := w.Write([]byte(propRes)); err != nil { log.Err(err).Msg("error writing response") diff --git a/internal/http/services/owncloud/ocdav/proppatch.go b/internal/http/services/owncloud/ocdav/proppatch.go index 1e1e4ccc09..a56a373c7e 100644 --- a/internal/http/services/owncloud/ocdav/proppatch.go +++ b/internal/http/services/owncloud/ocdav/proppatch.go @@ -41,6 +41,8 @@ func (s *svc) handleProppatch(w http.ResponseWriter, r *http.Request, ns string) defer span.End() log := appctx.GetLogger(ctx) + ns = applyLayout(ctx, ns) + fn := path.Join(ns, r.URL.Path) pp, status, err := readProppatch(r.Body) diff --git a/internal/http/services/owncloud/ocdav/put.go b/internal/http/services/owncloud/ocdav/put.go index d5f59cc561..d22a7765ff 100644 --- a/internal/http/services/owncloud/ocdav/put.go +++ b/internal/http/services/owncloud/ocdav/put.go @@ -27,9 +27,13 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/utils" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp" + tokenpkg "github.com/cs3org/reva/pkg/token" + "github.com/eventials/go-tus" + "github.com/eventials/go-tus/memorystore" ) func isChunked(fn string) (bool, error) { @@ -104,6 +108,9 @@ func isContentRange(r *http.Request) bool { func (s *svc) handlePut(w http.ResponseWriter, r *http.Request, ns string) { ctx := r.Context() log := appctx.GetLogger(ctx) + + ns = applyLayout(ctx, ns) + fn := path.Join(ns, r.URL.Path) if r.Body == nil { @@ -184,10 +191,24 @@ func (s *svc) handlePut(w http.ResponseWriter, r *http.Request, ns string) { } } + length, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + uReq := &provider.InitiateFileUploadRequest{ Ref: &provider.Reference{ Spec: &provider.Reference_Path{Path: fn}, }, + Opaque: &typespb.Opaque{ + Map: map[string]*typespb.OpaqueEntry{ + "Upload-Length": { + Decoder: "plain", + Value: []byte(r.Header.Get("Content-Length")), + }, + }, + }, } // where to upload the file? @@ -204,25 +225,44 @@ func (s *svc) handlePut(w http.ResponseWriter, r *http.Request, ns string) { } dataServerURL := uRes.UploadEndpoint - // TODO(labkode): do a protocol switch - httpReq, err := rhttp.NewRequest(ctx, "PUT", dataServerURL, r.Body) + + // create the tus client. + c := tus.DefaultConfig() + c.Resume = true + c.HttpClient = rhttp.GetHTTPClient(ctx) + c.Store, err = memorystore.NewMemoryStore() if err != nil { - log.Error().Err(err).Msg("error creating http request") w.WriteHeader(http.StatusInternalServerError) return } - httpReq.Header.Set("X-Reva-Transfer", uRes.Token) - httpClient := rhttp.GetHTTPClient(ctx) - httpRes, err := httpClient.Do(httpReq) + log.Debug(). + Str("header", tokenpkg.TokenHeader). + Str("token", tokenpkg.ContextMustGetToken(ctx)). + Msg("adding token to header") + c.Header.Set(tokenpkg.TokenHeader, tokenpkg.ContextMustGetToken(ctx)) + + tusc, err := tus.NewClient(dataServerURL, c) if err != nil { - log.Error().Err(err).Msg("error doing http request") w.WriteHeader(http.StatusInternalServerError) return } - defer httpRes.Body.Close() - if httpRes.StatusCode != http.StatusOK { + metadata := map[string]string{ + "filename": path.Base(fn), + "dir": path.Dir(fn), + //"checksum": fmt.Sprintf("%s %s", storageprovider.GRPC2PKGXS(xsType).String(), xs), + } + + upload := tus.NewUpload(r.Body, length, metadata, "") + + // create the uploader. + c.Store.Set(upload.Fingerprint, dataServerURL) + uploader := tus.NewUploader(tusc, dataServerURL, upload, 0) + + // start the uploading process. + err = uploader.Upload() + if err != nil { w.WriteHeader(http.StatusInternalServerError) return } diff --git a/internal/http/services/owncloud/ocdav/tus.go b/internal/http/services/owncloud/ocdav/tus.go new file mode 100644 index 0000000000..743780c223 --- /dev/null +++ b/internal/http/services/owncloud/ocdav/tus.go @@ -0,0 +1,178 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocdav + +import ( + "net/http" + "path" + + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/rhttp" + tusd "github.com/tus/tusd/pkg/handler" +) + +func (s *svc) handleTusPost(w http.ResponseWriter, r *http.Request, ns string) { + ctx := r.Context() + log := appctx.GetLogger(ctx) + + w.Header().Add("Access-Control-Allow-Headers", "Tus-Resumable, Upload-Length, Upload-Metadata, If-Match") + w.Header().Add("Access-Control-Expose-Headers", "Tus-Resumable, Location") + + w.Header().Set("Tus-Resumable", "1.0.0") + + // Test if the version sent by the client is supported + // GET methods are not checked since a browser may visit this URL and does + // not include this header. This request is not part of the specification. + if r.Header.Get("Tus-Resumable") != "1.0.0" { + w.WriteHeader(http.StatusPreconditionFailed) + return + } + if r.Header.Get("Upload-Length") == "" { + w.WriteHeader(http.StatusPreconditionFailed) + return + } + //TODO check Expect: 100-continue + + // read filename from metadata + meta := tusd.ParseMetadataHeader(r.Header.Get("Upload-Metadata")) + if meta["filename"] == "" { + w.WriteHeader(http.StatusPreconditionFailed) + return + } + + ns = applyLayout(ctx, ns) + + // append filename to current dir + fn := path.Join(ns, r.URL.Path, meta["filename"]) + + // check tus headers? + + // check if destination exists or is a file + client, err := s.getClient() + if err != nil { + log.Error().Err(err).Msg("error getting grpc client") + w.WriteHeader(http.StatusInternalServerError) + return + } + + sReq := &provider.StatRequest{ + Ref: &provider.Reference{ + Spec: &provider.Reference_Path{Path: fn}, + }, + } + sRes, err := client.Stat(ctx, sReq) + if err != nil { + log.Error().Err(err).Msg("error sending grpc stat request") + w.WriteHeader(http.StatusInternalServerError) + return + } + + if sRes.Status.Code != rpc.Code_CODE_OK { + if sRes.Status.Code != rpc.Code_CODE_NOT_FOUND { + w.WriteHeader(http.StatusInternalServerError) + return + } + } + + info := sRes.Info + if info != nil && info.Type != provider.ResourceType_RESOURCE_TYPE_FILE { + log.Warn().Msg("resource is not a file") + w.WriteHeader(http.StatusConflict) + return + } + + if info != nil { + clientETag := r.Header.Get("If-Match") + serverETag := info.Etag + if clientETag != "" { + if clientETag != serverETag { + log.Warn().Str("client-etag", clientETag).Str("server-etag", serverETag).Msg("etags mismatch") + w.WriteHeader(http.StatusPreconditionFailed) + return + } + } + } + + // initiateUpload + + uReq := &provider.InitiateFileUploadRequest{ + Ref: &provider.Reference{ + Spec: &provider.Reference_Path{Path: fn}, + }, + Opaque: &typespb.Opaque{ + Map: map[string]*typespb.OpaqueEntry{ + "Upload-Length": { + Decoder: "plain", + Value: []byte(r.Header.Get("Upload-Length")), + }, + }, + }, + } + + uRes, err := client.InitiateFileUpload(ctx, uReq) + if err != nil { + log.Error().Err(err).Msg("error initiating file upload") + w.WriteHeader(http.StatusInternalServerError) + return + } + + if uRes.Status.Code != rpc.Code_CODE_OK { + w.WriteHeader(http.StatusInternalServerError) + return + } + + w.Header().Set("Location", uRes.UploadEndpoint) + + // for creation-with-upload extension forward bytes to dataprovider + // TODO check this really streams + if r.Header.Get("Content-Type") == "application/offset+octet-stream" { + + httpClient := rhttp.GetHTTPClient(ctx) + httpReq, err := rhttp.NewRequest(ctx, "PATCH", uRes.UploadEndpoint, r.Body) + if err != nil { + log.Err(err).Msg("wrong request") + w.WriteHeader(http.StatusInternalServerError) + return + } + + httpReq.Header.Set("Content-Type", r.Header.Get("Content-Type")) + httpReq.Header.Set("Content-Length", r.Header.Get("Content-Length")) + httpReq.Header.Set("Upload-Offset", r.Header.Get("Upload-Offset")) + httpReq.Header.Set("Tus-Resumable", r.Header.Get("Tus-Resumable")) + + httpRes, err := httpClient.Do(httpReq) + if err != nil { + log.Err(err).Msg("error doing GET request to data service") + w.WriteHeader(http.StatusInternalServerError) + return + } + defer httpRes.Body.Close() + + httpRes.Header.Set("Upload-Offset", httpRes.Header.Get("Upload-Offset")) + httpRes.Header.Set("Tus-Resumable", httpRes.Header.Get("Tus-Resumable")) + if httpRes.StatusCode != http.StatusNoContent { + w.WriteHeader(httpRes.StatusCode) + return + } + } + w.WriteHeader(http.StatusCreated) +} diff --git a/internal/http/services/owncloud/ocdav/webdav.go b/internal/http/services/owncloud/ocdav/webdav.go index c14efc204a..5c7e5be7c1 100644 --- a/internal/http/services/owncloud/ocdav/webdav.go +++ b/internal/http/services/owncloud/ocdav/webdav.go @@ -57,6 +57,8 @@ func (h *WebDavHandler) Handler(s *svc) http.Handler { s.handleGet(w, r, h.namespace) case http.MethodPut: s.handlePut(w, r, h.namespace) + case http.MethodPost: + s.handleTusPost(w, r, h.namespace) case http.MethodOptions: s.handleOptions(w, r, h.namespace) case http.MethodHead: diff --git a/pkg/eosclient/eosclient.go b/pkg/eosclient/eosclient.go index 767a9e2621..c9a0c38395 100644 --- a/pkg/eosclient/eosclient.go +++ b/pkg/eosclient/eosclient.go @@ -591,12 +591,8 @@ func (c *Client) Read(ctx context.Context, username, path string) (io.ReadCloser return os.Open(localTarget) } -// Write writes a file to the mgm +// Write writes a stream to the mgm func (c *Client) Write(ctx context.Context, username, path string, stream io.ReadCloser) error { - unixUser, err := c.getUnixUser(username) - if err != nil { - return err - } fd, err := ioutil.TempFile(c.opt.CacheDirectory, "eoswrite-") if err != nil { return err @@ -609,8 +605,18 @@ func (c *Client) Write(ctx context.Context, username, path string, stream io.Rea if err != nil { return err } + + return c.WriteFile(ctx, username, path, fd.Name()) +} + +// WriteFile writes an existing file to the mgm +func (c *Client) WriteFile(ctx context.Context, username, path, source string) error { + unixUser, err := c.getUnixUser(username) + if err != nil { + return err + } xrdPath := fmt.Sprintf("%s//%s", c.opt.URL, path) - cmd := exec.CommandContext(ctx, c.opt.XrdcopyBinary, "--nopbar", "--silent", "-f", fd.Name(), xrdPath, fmt.Sprintf("-ODeos.ruid=%s&eos.rgid=%s", unixUser.Uid, unixUser.Gid)) + cmd := exec.CommandContext(ctx, c.opt.XrdcopyBinary, "--nopbar", "--silent", "-f", source, xrdPath, fmt.Sprintf("-ODeos.ruid=%s&eos.rgid=%s", unixUser.Uid, unixUser.Gid)) _, _, err = c.execute(ctx, cmd) return err } diff --git a/pkg/storage/fs/eos/eos.go b/pkg/storage/fs/eos/eos.go index 3f8c56e7e0..a4e8633ee5 100644 --- a/pkg/storage/fs/eos/eos.go +++ b/pkg/storage/fs/eos/eos.go @@ -79,6 +79,9 @@ type config struct { // ShadowNamespace for storing shadow data ShadowNamespace string `mapstructure:"shadow_namespace"` + // UploadsNamespace for storing upload data + UploadsNamespace string `mapstructure:"uploads_namespace"` + // ShareFolder defines the name of the folder in the // shadowed namespace. Ex: /eos/user/.shadow/h/hugo/MyShares ShareFolder string `mapstructure:"share_folder"` @@ -1130,26 +1133,6 @@ func (fs *eosfs) Download(ctx context.Context, ref *provider.Reference) (io.Read return fs.c.Read(ctx, u.Username, fn) } -func (fs *eosfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - - p, err := fs.resolve(ctx, u, ref) - if err != nil { - return errors.Wrap(err, "eos: error resolving reference") - } - - if fs.isShareFolder(ctx, p) { - return errtypes.PermissionDenied("eos: cannot download under the virtual share folder") - } - - fn := fs.wrap(ctx, p) - - return fs.c.Write(ctx, u.Username, fn, r) -} - func (fs *eosfs) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { u, err := getUser(ctx) if err != nil { diff --git a/pkg/storage/fs/eos/upload.go b/pkg/storage/fs/eos/upload.go new file mode 100644 index 0000000000..0a024727a5 --- /dev/null +++ b/pkg/storage/fs/eos/upload.go @@ -0,0 +1,319 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package eos + +import ( + "context" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/google/uuid" + "github.com/pkg/errors" + tusd "github.com/tus/tusd/pkg/handler" +) + +var defaultFilePerm = os.FileMode(0664) + +// TODO deprecated ... use tus +func (fs *eosfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + + p, err := fs.resolve(ctx, u, ref) + if err != nil { + return errors.Wrap(err, "eos: error resolving reference") + } + + if fs.isShareFolder(ctx, p) { + return errtypes.PermissionDenied("eos: cannot upload under the virtual share folder") + } + + fn := fs.wrap(ctx, p) + + return fs.c.Write(ctx, u.Username, fn, r) +} + +// InitiateUpload returns an upload id that can be used for uploads with tus +// TODO read optional content for small files in this request +func (fs *eosfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64) (uploadID string, err error) { + u, err := getUser(ctx) + if err != nil { + return "", errors.Wrap(err, "eos: no user in ctx") + } + + np, err := fs.resolve(ctx, u, ref) + if err != nil { + return "", errors.Wrap(err, "eos: error resolving reference") + } + + p := fs.wrap(ctx, np) + + info := tusd.FileInfo{ + MetaData: tusd.MetaData{ + "filename": filepath.Base(p), + "dir": filepath.Dir(p), + }, + Size: uploadLength, + } + + upload, err := fs.NewUpload(ctx, info) + if err != nil { + return "", err + } + + info, _ = upload.GetInfo(ctx) + + return info.ID, nil +} + +// UseIn tells the tus upload middleware which extensions it supports. +func (fs *eosfs) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(fs) + composer.UseTerminater(fs) +} + +// NewUpload creates a new upload using the size as the file's length. To determine where to write the binary data +// the Fileinfo metadata must contain a dir and a filename. +// returns a unique id which is used to identify the upload. The properties Size and MetaData will be filled. +func (fs *eosfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { + + log := appctx.GetLogger(ctx) + log.Debug().Interface("info", info).Msg("eos: NewUpload") + + fn := info.MetaData["filename"] + if fn == "" { + return nil, errors.New("eos: missing filename in metadata") + } + info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"]) + + dir := info.MetaData["dir"] + if dir == "" { + return nil, errors.New("eos: missing dir in metadata") + } + info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) + + log.Debug().Interface("info", info).Msg("eos: resolved filename") + + info.ID = uuid.New().String() + + binPath, err := fs.getUploadPath(ctx, info.ID) + if err != nil { + return nil, errors.Wrap(err, "eos: error resolving upload path") + } + user, err := getUser(ctx) + if err != nil { + return nil, errors.Wrap(err, "eos: no user in ctx") + } + info.Storage = map[string]string{ + "Type": "EOSStore", + "Username": user.Username, + } + // Create binary file with no content + + file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return nil, err + } + defer file.Close() + + u := &fileUpload{ + info: info, + binPath: binPath, + infoPath: binPath + ".info", + fs: fs, + } + + // writeInfo creates the file by itself if necessary + err = u.writeInfo() + if err != nil { + return nil, err + } + + return u, nil +} + +// TODO use a subdirectory in the shadow tree +func (fs *eosfs) getUploadPath(ctx context.Context, uploadID string) (string, error) { + return filepath.Join(fs.conf.CacheDirectory, uploadID), nil +} + +// GetUpload returns the Upload for the given upload id +func (fs *eosfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + binPath, err := fs.getUploadPath(ctx, id) + if err != nil { + return nil, err + } + infoPath := binPath + ".info" + info := tusd.FileInfo{} + data, err := ioutil.ReadFile(infoPath) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + stat, err := os.Stat(binPath) + if err != nil { + return nil, err + } + + info.Offset = stat.Size() + + return &fileUpload{ + info: info, + binPath: binPath, + infoPath: infoPath, + fs: fs, + }, nil +} + +type fileUpload struct { + // info stores the current information about the upload + info tusd.FileInfo + // infoPath is the path to the .info file + infoPath string + // binPath is the path to the binary file (which has no extension) + binPath string + // only fs knows how to handle metadata and versions + fs *eosfs +} + +// GetInfo returns the FileInfo +func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return upload.info, nil +} + +// GetReader returns an io.Readerfor the upload +func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) { + return os.Open(upload.binPath) +} + +// WriteChunk writes the stream from the reader to the given offset of the upload +// TODO use the grpc api to directly stream to a temporary uploads location in the eos shadow tree +func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return 0, err + } + defer file.Close() + + n, err := io.Copy(file, src) + + // If the HTTP PATCH request gets interrupted in the middle (e.g. because + // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. + // However, for OwnCloudStore it's not important whether the stream has ended + // on purpose or accidentally. + if err != nil { + if err != io.ErrUnexpectedEOF { + return n, err + } + } + + upload.info.Offset += n + err = upload.writeInfo() + + return n, err +} + +// writeInfo updates the entire information. Everything will be overwritten. +func (upload *fileUpload) writeInfo() error { + data, err := json.Marshal(upload.info) + if err != nil { + return err + } + return ioutil.WriteFile(upload.infoPath, data, defaultFilePerm) +} + +// FinishUpload finishes an upload and moves the file to the internal destination +func (upload *fileUpload) FinishUpload(ctx context.Context) error { + + checksum := upload.info.MetaData["checksum"] + if checksum != "" { + // check checksum + s := strings.SplitN(checksum, " ", 2) + if len(s) == 2 { + alg, hash := s[0], s[1] + + log := appctx.GetLogger(ctx) + log.Debug(). + Interface("info", upload.info). + Str("alg", alg). + Str("hash", hash). + Msg("eos: TODO check checksum") // TODO this is done by eos if we write chunks to it directly + + } + } + np := filepath.Join(upload.info.MetaData["dir"], upload.info.MetaData["filename"]) + + // TODO check etag with If-Match header + // if destination exists + //if _, err := os.Stat(np); err == nil { + // copy attributes of existing file to tmp file befor overwriting the target? + // eos creates revisions internally + //} + + err := upload.fs.c.WriteFile(ctx, upload.info.Storage["Username"], np, upload.binPath) + + // only delete the upload if it was successfully written to eos + if err == nil { + // cleanup in the background, delete might take a while and we don't need to wait for it to finish + go func() { + if err := os.Remove(upload.infoPath); err != nil { + log := appctx.GetLogger(ctx) + log.Err(err).Interface("info", upload.info).Msg("eos: could not delete upload info") + } + if err := os.Remove(upload.binPath); err != nil { + log := appctx.GetLogger(ctx) + log.Err(err).Interface("info", upload.info).Msg("eos: could not delete upload binary") + } + }() + } + // metadata propagation is left to the storage implementation + return err +} + +// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination +// - the storage needs to implement AsTerminatableUpload +// - the upload needs to implement Terminate + +// AsTerminatableUpload returnsa a TerminatableUpload +func (fs *eosfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload { + return upload.(*fileUpload) +} + +// Terminate terminates the upload +func (upload *fileUpload) Terminate(ctx context.Context) error { + if err := os.Remove(upload.infoPath); err != nil { + return err + } + if err := os.Remove(upload.binPath); err != nil { + return err + } + return nil +} diff --git a/pkg/storage/fs/local/local.go b/pkg/storage/fs/local/local.go index 573dffdcc8..ad03253cc2 100644 --- a/pkg/storage/fs/local/local.go +++ b/pkg/storage/fs/local/local.go @@ -49,6 +49,8 @@ type config struct { Root string `mapstructure:"root"` EnableHome bool `mapstructure:"enable_home"` UserLayout string `mapstructure:"user_layout"` + // Uploads fsolder should be on the same partition as root to make the final rename not fall back to a copy and delete + Uploads string `mapstructure:"uploads"` } func parseConfig(m map[string]interface{}) (*config, error) { @@ -82,6 +84,14 @@ func New(m map[string]interface{}) (storage.FS, error) { return nil, errors.Wrap(err, "local: could not create namespace dir") } + if c.Uploads == "" { + c.Uploads = path.Join(c.Root, ".uploads") + } + + if err := os.MkdirAll(c.Uploads, 0700); err != nil { + return nil, errors.Wrap(err, "could not create uploads dir "+c.Uploads) + } + return &localfs{root: c.Root, conf: c}, nil } @@ -335,33 +345,6 @@ func (fs *localfs) ListFolder(ctx context.Context, ref *provider.Reference) ([]* return finfos, nil } -func (fs *localfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { - fn, err := fs.resolve(ctx, ref) - if err != nil { - return errors.Wrap(err, "error resolving ref") - } - - // we cannot rely on /tmp as it can live in another partition and we can - // hit invalid cross-device link errors, so we create the tmp file in the same directory - // the file is supposed to be written. - tmp, err := ioutil.TempFile(path.Dir(fn), "._reva_atomic_upload") - if err != nil { - return errors.Wrap(err, "localfs: error creating tmp fn at "+path.Dir(fn)) - } - - _, err = io.Copy(tmp, r) - if err != nil { - return errors.Wrap(err, "localfs: eror writing to tmp file "+tmp.Name()) - } - - // TODO(labkode): make sure rename is atomic, missing fsync ... - if err := os.Rename(tmp.Name(), fn); err != nil { - return errors.Wrap(err, "localfs: error renaming from "+tmp.Name()+" to "+fn) - } - - return nil -} - func (fs *localfs) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { fn, err := fs.resolve(ctx, ref) if err != nil { diff --git a/pkg/storage/fs/local/upload.go b/pkg/storage/fs/local/upload.go new file mode 100644 index 0000000000..d43b152c5b --- /dev/null +++ b/pkg/storage/fs/local/upload.go @@ -0,0 +1,296 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package local + +import ( + "context" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/google/uuid" + "github.com/pkg/errors" + tusd "github.com/tus/tusd/pkg/handler" +) + +var defaultFilePerm = os.FileMode(0664) + +// TODO deprecated ... use tus +func (fs *localfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { + fn, err := fs.resolve(ctx, ref) + if err != nil { + return errors.Wrap(err, "error resolving ref") + } + + // we cannot rely on /tmp as it can live in another partition and we can + // hit invalid cross-device link errors, so we create the tmp file in the same directory + // the file is supposed to be written. + tmp, err := ioutil.TempFile(filepath.Dir(fn), "._reva_atomic_upload") + if err != nil { + return errors.Wrap(err, "localfs: error creating tmp fn at "+filepath.Dir(fn)) + } + + _, err = io.Copy(tmp, r) + if err != nil { + return errors.Wrap(err, "localfs: eror writing to tmp file "+tmp.Name()) + } + + // TODO(labkode): make sure rename is atomic, missing fsync ... + if err := os.Rename(tmp.Name(), fn); err != nil { + return errors.Wrap(err, "localfs: error renaming from "+tmp.Name()+" to "+fn) + } + + return nil +} + +// InitiateUpload returns an upload id that can be used for uploads with tus +// It resolves the resurce and then reuses the NewUpload function +// Currently requires the uploadLength to be set +// TODO to implement LengthDeferrerDataStore make size optional +// TODO read optional content for small files in this request +func (fs *localfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64) (uploadID string, err error) { + np, err := fs.resolve(ctx, ref) + if err != nil { + return "", errors.Wrap(err, "localfs: error resolving reference") + } + + p := fs.unwrap(ctx, np) + + info := tusd.FileInfo{ + MetaData: tusd.MetaData{ + "filename": filepath.Base(p), + "dir": filepath.Dir(p), + }, + Size: uploadLength, + } + + upload, err := fs.NewUpload(ctx, info) + if err != nil { + return "", err + } + + info, _ = upload.GetInfo(ctx) + + return info.ID, nil +} + +// UseIn tells the tus upload middleware which extensions it supports. +func (fs *localfs) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(fs) + composer.UseTerminater(fs) + // TODO composer.UseConcater(fs) + // TODO composer.UseLengthDeferrer(fs) +} + +// NewUpload creates a new upload using the size as the file's length. To determine where to write the binary data +// the Fileinfo metadata must contain a dir and a filename. +// returns a unique id which is used to identify the upload. The properties Size and MetaData will be filled. +func (fs *localfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { + + log := appctx.GetLogger(ctx) + log.Debug().Interface("info", info).Msg("localfs: NewUpload") + + fn := info.MetaData["filename"] + if fn == "" { + return nil, errors.New("localfs: missing filename in metadata") + } + info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"]) + + dir := info.MetaData["dir"] + if dir == "" { + return nil, errors.New("localfs: missing dir in metadata") + } + info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) + + np := fs.wrap(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"])) + + log.Debug().Interface("info", info).Msg("localfs: resolved filename") + + info.ID = uuid.New().String() + + binPath, err := fs.getUploadPath(ctx, info.ID) + if err != nil { + return nil, errors.Wrap(err, "localfs: error resolving upload path") + } + info.Storage = map[string]string{ + "Type": "LocalStore", + "InternalDestination": np, + } + // Create binary file with no content + file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return nil, err + } + defer file.Close() + + u := &fileUpload{ + info: info, + binPath: binPath, + infoPath: binPath + ".info", + fs: fs, + } + + // writeInfo creates the file by itself if necessary + err = u.writeInfo() + if err != nil { + return nil, err + } + + return u, nil +} + +func (fs *localfs) getUploadPath(ctx context.Context, uploadID string) (string, error) { + return filepath.Join(fs.conf.Uploads, uploadID), nil +} + +// GetUpload returns the Upload for the given upload id +func (fs *localfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + binPath, err := fs.getUploadPath(ctx, id) + if err != nil { + return nil, err + } + infoPath := binPath + ".info" + info := tusd.FileInfo{} + data, err := ioutil.ReadFile(infoPath) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + stat, err := os.Stat(binPath) + if err != nil { + return nil, err + } + + info.Offset = stat.Size() + + return &fileUpload{ + info: info, + binPath: binPath, + infoPath: infoPath, + fs: fs, + }, nil +} + +type fileUpload struct { + // info stores the current information about the upload + info tusd.FileInfo + // infoPath is the path to the .info file + infoPath string + // binPath is the path to the binary file (which has no extension) + binPath string + // only fs knows how to handle metadata and versions + fs *localfs +} + +// GetInfo returns the FileInfo +func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return upload.info, nil +} + +// GetReader returns an io.Readerfor the upload +func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) { + return os.Open(upload.binPath) +} + +// WriteChunk writes the stream from the reader to the given offset of the upload +func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return 0, err + } + defer file.Close() + + n, err := io.Copy(file, src) + + // If the HTTP PATCH request gets interrupted in the middle (e.g. because + // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. + // However, for OwnCloudStore it's not important whether the stream has ended + // on purpose or accidentally. + if err != nil { + if err != io.ErrUnexpectedEOF { + return n, err + } + } + + upload.info.Offset += n + err = upload.writeInfo() + + return n, err +} + +// writeInfo updates the entire information. Everything will be overwritten. +func (upload *fileUpload) writeInfo() error { + data, err := json.Marshal(upload.info) + if err != nil { + return err + } + return ioutil.WriteFile(upload.infoPath, data, defaultFilePerm) +} + +// FinishUpload finishes an upload and moves the file to the internal destination +func (upload *fileUpload) FinishUpload(ctx context.Context) error { + + np := upload.info.Storage["InternalDestination"] + + // TODO check etag with If-Match header + // if destination exists + //if _, err := os.Stat(np); err == nil { + // the local storage does not store metadata + // the fileid is based on the path, so no we do not need to copy it to the new file + // the local storage does not track revisions + //} + + err := os.Rename(upload.binPath, np) + + // only delete the upload if it was successfully written to eos + if err := os.Remove(upload.infoPath); err != nil { + log := appctx.GetLogger(ctx) + log.Err(err).Interface("info", upload.info).Msg("eos: could not delete upload info") + } + + // metadata propagation is left to the storage implementation + return err +} + +// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination +// - the storage needs to implement AsTerminatableUpload +// - the upload needs to implement Terminate + +// AsTerminatableUpload returnsa a TerminatableUpload +func (fs *localfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload { + return upload.(*fileUpload) +} + +// Terminate terminates the upload +func (upload *fileUpload) Terminate(ctx context.Context) error { + if err := os.Remove(upload.infoPath); err != nil { + return err + } + if err := os.Remove(upload.binPath); err != nil { + return err + } + return nil +} diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index 0380ac7f0c..b015f47da8 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -144,6 +144,7 @@ const ( mdPrefix string = "user.oc.md." // arbitrary metadada favPrefix string = "user.oc.fav." // favorite flag, per user etagPrefix string = "user.oc.etag." // allow overriding a calculated etag with one from the extended attributes + //checksumPrefix string = "user.oc.cs." // TODO add checksum support ) func init() { @@ -152,6 +153,7 @@ func init() { type config struct { DataDirectory string `mapstructure:"datadirectory"` + UploadInfoDir string `mapstructure:"upload_info_dir"` UserLayout string `mapstructure:"user_layout"` Redis string `mapstructure:"redis"` EnableHome bool `mapstructure:"enable_home"` @@ -174,6 +176,9 @@ func (c *config) init(m map[string]interface{}) { if c.UserLayout == "" { c.UserLayout = "{{.Username}}" } + if c.UploadInfoDir == "" { + c.UploadInfoDir = "/var/tmp/reva/uploadinfo" + } // default to scanning if not configured if _, ok := m["scan"]; !ok { c.Scan = true @@ -200,6 +205,13 @@ func New(m map[string]interface{}) (storage.FS, error) { Msg("could not create datadir") } + err = os.MkdirAll(c.UploadInfoDir, 0700) + if err != nil { + logger.New().Error().Err(err). + Str("path", c.UploadInfoDir). + Msg("could not create uploadinfo dir") + } + pool := &redis.Pool{ MaxIdle: 3, @@ -539,7 +551,11 @@ func (fs *ocfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provid attr = sharePrefix + "u:" + e.Principal } - return xattr.Set(np, attr, getValue(e)) + if err := xattr.Set(np, attr, getValue(e)); err != nil { + return err + } + fs.propagate(ctx, np) + return nil } func getValue(e *ace) []byte { @@ -877,6 +893,7 @@ func (fs *ocfs) CreateHome(ctx context.Context) error { path.Join(fs.c.DataDirectory, layout, "files"), path.Join(fs.c.DataDirectory, layout, "files_trashbin"), path.Join(fs.c.DataDirectory, layout, "files_versions"), + path.Join(fs.c.DataDirectory, layout, "uploads"), } for _, v := range homePaths { @@ -905,6 +922,7 @@ func (fs *ocfs) CreateDir(ctx context.Context, fn string) (err error) { // FIXME we also need already exists error, webdav expects 405 MethodNotAllowed return errors.Wrap(err, "ocfs: error creating dir "+np) } + fs.propagate(ctx, np) return nil } @@ -1035,6 +1053,7 @@ func (fs *ocfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Referenc } switch len(errs) { case 0: + fs.propagate(ctx, np) return nil case 1: return errs[0] @@ -1114,6 +1133,7 @@ func (fs *ocfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refere switch len(errs) { case 0: + fs.propagate(ctx, np) return nil case 1: return errs[0] @@ -1175,6 +1195,8 @@ func (fs *ocfs) Delete(ctx context.Context, ref *provider.Reference) (err error) return errors.Wrap(err, "ocfs: could not restore item") } + fs.propagate(ctx, path.Dir(np)) + // TODO(jfd) move versions to trash return nil } @@ -1191,6 +1213,8 @@ func (fs *ocfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) (e if err = os.Rename(oldName, newName); err != nil { return errors.Wrap(err, "ocfs: error moving "+oldName+" to "+newName) } + fs.propagate(ctx, newName) + fs.propagate(ctx, oldName) return nil } @@ -1240,50 +1264,9 @@ func (fs *ocfs) ListFolder(ctx context.Context, ref *provider.Reference) ([]*pro return finfos, nil } -func (fs *ocfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { - np, err := fs.resolve(ctx, ref) - if err != nil { - return errors.Wrap(err, "ocfs: error resolving reference") - } - - // we cannot rely on /tmp as it can live in another partition and we can - // hit invalid cross-device link errors, so we create the tmp file in the same directory - // the file is supposed to be written. - tmp, err := ioutil.TempFile(path.Dir(np), "._reva_atomic_upload") - if err != nil { - return errors.Wrap(err, "ocfs: error creating tmp fn at "+path.Dir(np)) - } - defer os.RemoveAll(tmp.Name()) - - _, err = io.Copy(tmp, r) - tmp.Close() - if err != nil { - return errors.Wrap(err, "ocfs: error writing to tmp file "+tmp.Name()) - } - - // if destination exists - if _, err := os.Stat(np); err == nil { - // copy attributes of existing file to tmp file - if err := fs.copyMD(np, tmp.Name()); err != nil { - return errors.Wrap(err, "ocfs: error copying metadata from "+np+" to "+tmp.Name()) - } - // create revision - if err := fs.archiveRevision(ctx, np); err != nil { - return err - } - } - - // TODO(jfd): make sure rename is atomic, missing fsync ... - if err := os.Rename(tmp.Name(), np); err != nil { - return errors.Wrap(err, "ocfs: error renaming from "+tmp.Name()+" to "+np) - } - - return nil -} - -func (fs *ocfs) archiveRevision(ctx context.Context, np string) error { +func (fs *ocfs) archiveRevision(ctx context.Context, vbp string, np string) error { // move existing file to versions dir - vp := fmt.Sprintf("%s.v%d", fs.getVersionsPath(ctx, np), time.Now().Unix()) + vp := fmt.Sprintf("%s.v%d", vbp, time.Now().Unix()) if err := os.MkdirAll(path.Dir(vp), 0700); err != nil { return errors.Wrap(err, "ocfs: error creating versions dir "+vp) } @@ -1403,7 +1386,7 @@ func (fs *ocfs) RestoreRevision(ctx context.Context, ref *provider.Reference, re defer source.Close() // destination should be available, otherwise we could not have navigated to its revisions - if err := fs.archiveRevision(ctx, np); err != nil { + if err := fs.archiveRevision(ctx, fs.getVersionsPath(ctx, np), np); err != nil { return err } @@ -1415,6 +1398,9 @@ func (fs *ocfs) RestoreRevision(ctx context.Context, ref *provider.Reference, re defer destination.Close() _, err = io.Copy(destination, source) + + fs.propagate(ctx, np) + // TODO(jfd) bring back revision in case sth goes wrong? return err } @@ -1556,5 +1542,58 @@ func (fs *ocfs) RestoreRecycleItem(ctx context.Context, key string) error { log.Warn().Err(err).Str("path", tgt).Msg("could not unset origin") } // TODO(jfd) restore versions + + fs.propagate(ctx, tgt) + return nil } + +func (fs *ocfs) propagate(ctx context.Context, leafPath string) { + var root string + if fs.c.EnableHome { + root = fs.wrap(ctx, "/") + } else { + u := user.ContextMustGetUser(ctx) + root = fs.wrap(ctx, path.Join("/", u.GetUsername())) + } + if !strings.HasPrefix(leafPath, root) { + appctx.GetLogger(ctx).Error(). + Err(errors.New("internal path outside root")). + Str("leafPath", leafPath). + Str("root", root). + Msg("could not propagate change") + return + } + + fi, err := os.Stat(leafPath) + if err != nil { + appctx.GetLogger(ctx).Error(). + Err(err). + Str("leafPath", leafPath). + Str("root", root). + Msg("could not propagate change") + } + + parts := strings.Split(strings.TrimPrefix(leafPath, root), "/") + // root never ents in / so the split returns an empty first element, which we can skip + // we do not need to chmod the last element because it is the leaf path (< and not <= comparison) + for i := 1; i < len(parts); i++ { + appctx.GetLogger(ctx).Debug(). + Str("leafPath", leafPath). + Str("root", root). + Int("i", i). + Interface("parts", parts). + Msg("propagating change") + if err := os.Chtimes(path.Join(root), fi.ModTime(), fi.ModTime()); err != nil { + appctx.GetLogger(ctx).Error(). + Err(err). + Str("leafPath", leafPath). + Str("root", root). + Msg("could not propagate change") + } + root = path.Join(root, parts[i]) + } +} + +// TODO propagate etag and mtime or append event to history? propagate on disk ... +// - but propagation is a separate task. only if upload was successful ... diff --git a/pkg/storage/fs/owncloud/owncloud_unix.go b/pkg/storage/fs/owncloud/owncloud_unix.go index 24760ffdde..26bb3b0cea 100755 --- a/pkg/storage/fs/owncloud/owncloud_unix.go +++ b/pkg/storage/fs/owncloud/owncloud_unix.go @@ -31,6 +31,8 @@ import ( "github.com/cs3org/reva/pkg/appctx" ) +// TODO(jfd) get rid of the differences between unix and windows. the inode and dev should never be used for the etag because it interferes with backups + // calcEtag will create an etag based on the md5 of // - mtime, // - inode (if available), diff --git a/pkg/storage/fs/owncloud/upload.go b/pkg/storage/fs/owncloud/upload.go new file mode 100644 index 0000000000..2cad971f83 --- /dev/null +++ b/pkg/storage/fs/owncloud/upload.go @@ -0,0 +1,412 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package owncloud + +import ( + "context" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/logger" + "github.com/cs3org/reva/pkg/user" + "github.com/google/uuid" + "github.com/pkg/errors" + tusd "github.com/tus/tusd/pkg/handler" +) + +var defaultFilePerm = os.FileMode(0664) + +// TODO deprecated ... use tus +func (fs *ocfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { + np, err := fs.resolve(ctx, ref) + if err != nil { + return errors.Wrap(err, "ocfs: error resolving reference") + } + + // we cannot rely on /tmp as it can live in another partition and we can + // hit invalid cross-device link errors, so we create the tmp file in the same directory + // the file is supposed to be written. + tmp, err := ioutil.TempFile(filepath.Dir(np), "._reva_atomic_upload") + if err != nil { + return errors.Wrap(err, "ocfs: error creating tmp fn at "+filepath.Dir(np)) + } + + _, err = io.Copy(tmp, r) + if err != nil { + return errors.Wrap(err, "ocfs: error writing to tmp file "+tmp.Name()) + } + + // if destination exists + if _, err := os.Stat(np); err == nil { + // copy attributes of existing file to tmp file + if err := fs.copyMD(np, tmp.Name()); err != nil { + return errors.Wrap(err, "ocfs: error copying metadata from "+np+" to "+tmp.Name()) + } + // create revision + if err := fs.archiveRevision(ctx, fs.getVersionsPath(ctx, np), np); err != nil { + return err + } + } + + // TODO(jfd): make sure rename is atomic, missing fsync ... + if err := os.Rename(tmp.Name(), np); err != nil { + return errors.Wrap(err, "ocfs: error renaming from "+tmp.Name()+" to "+np) + } + + return nil +} + +// InitiateUpload returns an upload id that can be used for uploads with tus +// TODO read optional content for small files in this request +func (fs *ocfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64) (uploadID string, err error) { + np, err := fs.resolve(ctx, ref) + if err != nil { + return "", errors.Wrap(err, "ocfs: error resolving reference") + } + + p := fs.unwrap(ctx, np) + + info := tusd.FileInfo{ + MetaData: tusd.MetaData{ + "filename": filepath.Base(p), + "dir": filepath.Dir(p), + }, + Size: uploadLength, + } + + upload, err := fs.NewUpload(ctx, info) + if err != nil { + return "", err + } + + info, _ = upload.GetInfo(ctx) + + return info.ID, nil +} + +// UseIn tells the tus upload middleware which extensions it supports. +func (fs *ocfs) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(fs) + composer.UseTerminater(fs) + composer.UseConcater(fs) + composer.UseLengthDeferrer(fs) +} + +// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol +// - the storage needs to implement NewUpload and GetUpload +// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload + +func (fs *ocfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { + + log := appctx.GetLogger(ctx) + log.Debug().Interface("info", info).Msg("ocfs: NewUpload") + + fn := info.MetaData["filename"] + if fn == "" { + return nil, errors.New("ocfs: missing filename in metadata") + } + info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"]) + + dir := info.MetaData["dir"] + if dir == "" { + return nil, errors.New("ocfs: missing dir in metadata") + } + info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) + + np := fs.wrap(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"])) + + log.Debug().Interface("info", info).Msg("ocfs: resolved filename") + + info.ID = uuid.New().String() + + binPath, err := fs.getUploadPath(ctx, info.ID) + if err != nil { + return nil, errors.Wrap(err, "ocfs: error resolving upload path") + } + usr := user.ContextMustGetUser(ctx) + info.Storage = map[string]string{ + "Type": "OwnCloudStore", + "BinPath": binPath, + "InternalDestination": np, + + "Idp": usr.Id.Idp, + "UserId": usr.Id.OpaqueId, + "UserName": usr.Username, + + "LogLevel": log.GetLevel().String(), + } + // Create binary file in the upload folder with no content + file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return nil, err + } + defer file.Close() + + u := &fileUpload{ + info: info, + binPath: binPath, + infoPath: filepath.Join(fs.c.UploadInfoDir, info.ID+".info"), + fs: fs, + } + + // writeInfo creates the file by itself if necessary + err = u.writeInfo() + if err != nil { + return nil, err + } + + return u, nil +} + +func (fs *ocfs) getUploadPath(ctx context.Context, uploadID string) (string, error) { + u, ok := user.ContextGetUser(ctx) + if !ok { + err := errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") + return "", err + } + return filepath.Join(fs.c.DataDirectory, u.Username, "uploads", uploadID), nil +} + +// GetUpload returns the Upload for the given upload id +func (fs *ocfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + infoPath := filepath.Join(fs.c.UploadInfoDir, id+".info") + + info := tusd.FileInfo{} + data, err := ioutil.ReadFile(infoPath) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + stat, err := os.Stat(info.Storage["BinPath"]) + if err != nil { + return nil, err + } + + info.Offset = stat.Size() + + u := &userpb.User{ + Id: &userpb.UserId{ + Idp: info.Storage["Idp"], + OpaqueId: info.Storage["UserId"], + }, + Username: info.Storage["UserName"], + } + + ctx = user.ContextSetUser(ctx, u) + // TODO configure the logger the same way ... store and add traceid in file info + + var opts []logger.Option + opts = append(opts, logger.WithLevel(info.Storage["LogLevel"])) + opts = append(opts, logger.WithWriter(os.Stderr, logger.ConsoleMode)) + l := logger.New(opts...) + + sub := l.With().Int("pid", os.Getpid()).Logger() + + ctx = appctx.WithLogger(ctx, &sub) + + return &fileUpload{ + info: info, + binPath: info.Storage["BinPath"], + infoPath: infoPath, + fs: fs, + ctx: ctx, + }, nil +} + +type fileUpload struct { + // info stores the current information about the upload + info tusd.FileInfo + // infoPath is the path to the .info file + infoPath string + // binPath is the path to the binary file (which has no extension) + binPath string + // only fs knows how to handle metadata and versions + fs *ocfs + // a context with a user + // TODO add logger as well? + ctx context.Context +} + +// GetInfo returns the FileInfo +func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return upload.info, nil +} + +// WriteChunk writes the stream from the reader to the given offset of the upload +func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return 0, err + } + defer file.Close() + + n, err := io.Copy(file, src) + + // If the HTTP PATCH request gets interrupted in the middle (e.g. because + // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. + // However, for OwnCloudStore it's not important whether the stream has ended + // on purpose or accidentally. + if err != nil { + if err != io.ErrUnexpectedEOF { + return n, err + } + } + + upload.info.Offset += n + err = upload.writeInfo() // TODO info is written here ... we need to truncate in DiscardChunk + + return n, err +} + +// GetReader returns an io.Readerfor the upload +func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) { + return os.Open(upload.binPath) +} + +// writeInfo updates the entire information. Everything will be overwritten. +func (upload *fileUpload) writeInfo() error { + data, err := json.Marshal(upload.info) + if err != nil { + return err + } + return ioutil.WriteFile(upload.infoPath, data, defaultFilePerm) +} + +// FinishUpload finishes an upload and moves the file to the internal destination +func (upload *fileUpload) FinishUpload(ctx context.Context) error { + + /* + checksum := upload.info.MetaData["checksum"] + if checksum != "" { + // TODO check checksum + s := strings.SplitN(checksum, " ", 2) + if len(s) == 2 { + alg, hash := s[0], s[1] + + } + } + */ + + np := upload.info.Storage["InternalDestination"] + + // if destination exists + // TODO check etag with If-Match header + if _, err := os.Stat(np); err == nil { + // copy attributes of existing file to tmp file + if err := upload.fs.copyMD(np, upload.binPath); err != nil { + return errors.Wrap(err, "ocfs: error copying metadata from "+np+" to "+upload.binPath) + } + // create revision + if err := upload.fs.archiveRevision(upload.ctx, upload.fs.getVersionsPath(upload.ctx, np), np); err != nil { + return err + } + } + + err := os.Rename(upload.binPath, np) + + // only delete the upload if it was successfully written to eos + if err := os.Remove(upload.infoPath); err != nil { + log := appctx.GetLogger(upload.ctx) + log.Err(err).Interface("info", upload.info).Msg("eos: could not delete upload info") + } + + upload.fs.propagate(upload.ctx, np) + + // FIXME metadata propagation is left to the storage implementation + return err +} + +// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination +// - the storage needs to implement AsTerminatableUpload +// - the upload needs to implement Terminate + +// AsTerminatableUpload returnsa a TerminatableUpload +func (fs *ocfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload { + return upload.(*fileUpload) +} + +// Terminate terminates the upload +func (upload *fileUpload) Terminate(ctx context.Context) error { + if err := os.Remove(upload.infoPath); err != nil { + return err + } + if err := os.Remove(upload.binPath); err != nil { + return err + } + return nil +} + +// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation +// - the storage needs to implement AsLengthDeclarableUpload +// - the upload needs to implement DeclareLength + +// AsLengthDeclarableUpload returnsa a LengthDeclarableUpload +func (fs *ocfs) AsLengthDeclarableUpload(upload tusd.Upload) tusd.LengthDeclarableUpload { + return upload.(*fileUpload) +} + +// DeclareLength updates the upload length information +func (upload *fileUpload) DeclareLength(ctx context.Context, length int64) error { + upload.info.Size = length + upload.info.SizeIsDeferred = false + return upload.writeInfo() +} + +// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation +// - the storage needs to implement AsConcatableUpload +// - the upload needs to implement ConcatUploads + +// AsConcatableUpload returnsa a ConcatableUpload +func (fs *ocfs) AsConcatableUpload(upload tusd.Upload) tusd.ConcatableUpload { + return upload.(*fileUpload) +} + +// ConcatUploads concatenates multiple uploads +func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []tusd.Upload) (err error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return err + } + defer file.Close() + + for _, partialUpload := range uploads { + fileUpload := partialUpload.(*fileUpload) + + src, err := os.Open(fileUpload.binPath) + if err != nil { + return err + } + + if _, err := io.Copy(file, src); err != nil { + return err + } + } + + return +} diff --git a/pkg/storage/fs/s3/upload.go b/pkg/storage/fs/s3/upload.go new file mode 100644 index 0000000000..a87a96d7ff --- /dev/null +++ b/pkg/storage/fs/s3/upload.go @@ -0,0 +1,31 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package s3 + +import ( + "context" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/errtypes" +) + +// InitiateUpload returns an upload id that can be used for uploads with tus +func (fs *s3FS) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64) (uploadID string, err error) { + return "", errtypes.NotSupported("op not supported") +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 3457c0c552..776af7a17b 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -36,6 +36,7 @@ type FS interface { Move(ctx context.Context, oldRef, newRef *provider.Reference) error GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) + InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64) (string, error) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error)