tus support (#674)

cmd/reva/upload.go

@@ -22,12 +22,18 @@ import (
 	"fmt"
 	"io"
 	"math"
-	"net/http"
 	"os"
+	"path/filepath"
+	"strconv"
 
 	"github.com/cheggaaa/pb"
 	rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
+
+	tokenpkg "github.com/cs3org/reva/pkg/token"
+	"github.com/eventials/go-tus"
+	"github.com/eventials/go-tus/memorystore"
 
 	// TODO(labkode): this should not come from this package.
 	"github.com/cs3org/reva/internal/grpc/services/storageprovider"
@@ -65,7 +71,7 @@ func uploadCommand() *command {
 
 		fmt.Printf("Local file size: %d bytes\n", md.Size())
 
-		client, err := getClient()
+		gwc, err := getClient()
 		if err != nil {
 			return err
 		}
@@ -76,9 +82,17 @@ func uploadCommand() *command {
 					Path: target,
 				},
 			},
+			Opaque: &typespb.Opaque{
+				Map: map[string]*typespb.OpaqueEntry{
+					"Upload-Length": {
+						Decoder: "plain",
+						Value:   []byte(strconv.FormatInt(md.Size(), 10)),
+					},
+				},
+			},
 		}
 
-		res, err := client.InitiateFileUpload(ctx, req)
+		res, err := gwc.InitiateFileUpload(ctx, req)
 		if err != nil {
 			return err
 		}
@@ -109,43 +123,62 @@ func uploadCommand() *command {
 		}
 
 		dataServerURL := res.UploadEndpoint
-		bar := pb.New(int(md.Size())).SetUnits(pb.U_BYTES)
-		bar.Start()
-		reader := bar.NewProxyReader(fd)
 
-		httpReq, err := rhttp.NewRequest(ctx, "PUT", dataServerURL, reader)
+		// create the tus client.
+		c := tus.DefaultConfig()
+		c.Resume = true
+		c.HttpClient = rhttp.GetHTTPClient(ctx)
+		c.Store, err = memorystore.NewMemoryStore()
+		if err != nil {
+			return err
+		}
+		if res.Token != "" {
+			fmt.Printf("using X-Reva-Transfer header\n")
+			c.Header.Add("X-Reva-Transfer", res.Token)
+		} else if token, ok := tokenpkg.ContextGetToken(ctx); ok {
+			fmt.Printf("using %s header\n", tokenpkg.TokenHeader)
+			c.Header.Add(tokenpkg.TokenHeader, token)
+		}
+		tusc, err := tus.NewClient(dataServerURL, c)
 		if err != nil {
 			return err
 		}
 
-		httpReq.Header.Set("X-Reva-Transfer", res.Token)
-		q := httpReq.URL.Query()
-		q.Add("xs", xs)
-		q.Add("xs_type", storageprovider.GRPC2PKGXS(xsType).String())
-		httpReq.URL.RawQuery = q.Encode()
+		metadata := map[string]string{
+			"filename": filepath.Base(target),
+			"dir":      filepath.Dir(target),
+			"checksum": fmt.Sprintf("%s %s", storageprovider.GRPC2PKGXS(xsType).String(), xs),
+		}
+
+		fingerprint := fmt.Sprintf("%s-%d-%s-%s", md.Name(), md.Size(), md.ModTime(), xs)
+
+		bar := pb.New(int(md.Size())).SetUnits(pb.U_BYTES)
+		bar.Start()
+		reader := bar.NewProxyReader(fd)
+
+		// create an upload from a file.
+		upload := tus.NewUpload(reader, md.Size(), metadata, fingerprint)
 
-		httpClient := rhttp.GetHTTPClient(ctx)
+		// create the uploader.
+		c.Store.Set(upload.Fingerprint, dataServerURL)
+		uploader := tus.NewUploader(tusc, dataServerURL, upload, 0)
 
-		httpRes, err := httpClient.Do(httpReq)
+		// start the uploading process.
+		err = uploader.Upload()
 		if err != nil {
 			return err
 		}
-		defer httpRes.Body.Close()
 
 		bar.Finish()
 
-		if httpRes.StatusCode != http.StatusOK {
-			return err
-		}
-
 		req2 := &provider.StatRequest{
 			Ref: &provider.Reference{
 				Spec: &provider.Reference_Path{
 					Path: target,
 				},
 			},
 		}
-		res2, err := client.Stat(ctx, req2)
+		res2, err := gwc.Stat(ctx, req2)
 		if err != nil {
 			return err
 		}

go.mod

@@ -12,13 +12,14 @@ require (
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/cs3org/go-cs3apis v0.0.0-20200423154403-462ce7762d4a
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/eventials/go-tus v0.0.0-20190617130015-9db47421f6a0
 	github.com/fatih/color v1.7.0 // indirect
 	github.com/go-openapi/strfmt v0.19.2 // indirect
 	github.com/gofrs/uuid v3.2.0+incompatible
 	github.com/golang/protobuf v1.3.5
 	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/google/uuid v1.1.1
-	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
+	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
 	github.com/huandu/xstrings v1.3.0 // indirect
 	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
@@ -31,12 +32,12 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/xattr v0.4.1
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
-	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 // indirect
 	github.com/rs/cors v1.7.0
 	github.com/rs/zerolog v1.18.0
+	github.com/tus/tusd v1.1.1-0.20200416115059-9deabf9d80c2
 	go.opencensus.io v0.22.3
 	golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a
-	golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421
+	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	google.golang.org/grpc v1.29.0
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.27 // indirect
@@ -45,3 +46,5 @@ require (
 )
 
 go 1.13
+
+replace github.com/eventials/go-tus => github.com/andrewmostello/go-tus v0.0.0-20200314041820-904a9904af9a