From 3b70b613276ca5389df6e3e26d5423260b5eb1d5 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 2 Jun 2023 16:47:45 +1000 Subject: [PATCH 1/2] feat: add stable etag Ref: https://github.com/ipfs/boxo/pull/303/commits/f61f95481041406df46a1781b1daab34b6605650#r1213918777 --- go.mod | 2 +- pkg/internal/itest/http_fetch_test.go | 5 +- pkg/server/http/ipfs.go | 21 ++-- pkg/types/request.go | 27 ++++++ pkg/types/request_test.go | 134 ++++++++++++++++++++++++++ 5 files changed, 177 insertions(+), 12 deletions(-) create mode 100644 pkg/types/request_test.go diff --git a/go.mod b/go.mod index 6d7c1239..dcf871d6 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bep/debounce v1.2.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect diff --git a/pkg/internal/itest/http_fetch_test.go b/pkg/internal/itest/http_fetch_test.go index 7cdfcd2c..0bfd8766 100644 --- a/pkg/internal/itest/http_fetch_test.go +++ b/pkg/internal/itest/http_fetch_test.go @@ -982,7 +982,10 @@ func TestHttpFetch(t *testing.T) { req.Equal("public, max-age=29030400, immutable", resp.Header.Get("Cache-Control")) req.Equal("application/vnd.ipld.car; version=1", resp.Header.Get("Content-Type")) req.Equal("nosniff", resp.Header.Get("X-Content-Type-Options")) - req.Equal(fmt.Sprintf(`%s.car`, srcData[i].Root.String()), resp.Header.Get("ETag")) // TODO: needs scope and path too + etagStart := fmt.Sprintf(`"%s.car.`, srcData[i].Root.String()) + etagGot := resp.Header.Get("ETag") + req.True(strings.HasPrefix(etagGot, etagStart), "ETag should start with [%s], got [%s]", etagStart, etagGot) + req.Equal(`"`, etagGot[len(etagGot)-1:], "ETag should end with a quote") var path string if testCase.paths != nil && testCase.paths[i] != "" { path = testCase.paths[i] diff --git a/pkg/server/http/ipfs.go b/pkg/server/http/ipfs.go index 4d3f649b..89239f0c 100644 --- a/pkg/server/http/ipfs.go +++ b/pkg/server/http/ipfs.go @@ -138,6 +138,16 @@ func ipfsHandler(lassie *lassie.Lassie, cfg HttpServerConfig) func(http.Response }() var store types.ReadableWritableStorage = carStore + request, err := types.NewRequestForPath(store, rootCid, path.String(), dagScope) + if err != nil { + errorResponse(res, statusLogger, http.StatusInternalServerError, fmt.Errorf("failed to create request: %w", err)) + return + } + request.Protocols = protocols + request.FixedPeers = fixedPeers + request.RetrievalID = retrievalId + request.Duplicates = includeDupes // needed for etag + carWriter.OnPut(func(int) { // called once we start writing blocks into the CAR (on the first Put()) res.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName)) @@ -145,7 +155,7 @@ func ipfsHandler(lassie *lassie.Lassie, cfg HttpServerConfig) func(http.Response res.Header().Set("Cache-Control", ResponseCacheControlHeader) res.Header().Set("Content-Type", ResponseContentTypeHeader) // TODO: needs scope and path - res.Header().Set("Etag", fmt.Sprintf("%s.car", rootCid.String())) + res.Header().Set("Etag", request.Etag()) res.Header().Set("X-Content-Type-Options", "nosniff") res.Header().Set("X-Ipfs-Path", "/"+datamodel.ParsePath(req.URL.Path).String()) // TODO: set X-Ipfs-Roots header when we support root+path @@ -155,15 +165,6 @@ func ipfsHandler(lassie *lassie.Lassie, cfg HttpServerConfig) func(http.Response close(bytesWritten) }, true) - request, err := types.NewRequestForPath(store, rootCid, path.String(), dagScope) - - if err != nil { - errorResponse(res, statusLogger, http.StatusInternalServerError, fmt.Errorf("failed to create request: %w", err)) - return - } - request.Protocols = protocols - request.FixedPeers = fixedPeers - request.RetrievalID = retrievalId // setup preload storage for bitswap, the temporary CAR store can set up a // separate preload space in its storage request.PreloadLinkSystem = cidlink.DefaultLinkSystem() diff --git a/pkg/types/request.go b/pkg/types/request.go index 5f235a1d..81ec3692 100644 --- a/pkg/types/request.go +++ b/pkg/types/request.go @@ -3,12 +3,15 @@ package types import ( "errors" "fmt" + "strconv" "strings" + "github.com/cespare/xxhash/v2" "github.com/google/uuid" "github.com/ipfs/go-cid" "github.com/ipfs/go-unixfsnode" "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" ipldstorage "github.com/ipld/go-ipld-prime/storage" "github.com/libp2p/go-libp2p/core/peer" @@ -52,6 +55,7 @@ type RetrievalRequest struct { Selector ipld.Node Path string Scope DagScope + Duplicates bool Protocols []multicodec.Code PreloadLinkSystem ipld.LinkSystem MaxBlocks uint64 @@ -142,6 +146,29 @@ func (r RetrievalRequest) GetSupportedProtocols(allSupportedProtocols []multicod return supportedProtocols } +func (r RetrievalRequest) Etag() string { + // https://github.com/ipfs/boxo/pull/303/commits/f61f95481041406df46a1781b1daab34b6605650#r1213918777 + sb := strings.Builder{} + sb.WriteString("/ipfs/") + sb.WriteString(r.Cid.String()) + if r.Path != "" { + sb.WriteString("/") + sb.WriteString(datamodel.ParsePath(r.Path).String()) + } + if r.Scope != DagScopeAll { + sb.WriteString(".") + sb.WriteString(string(r.Scope)) + } + if r.Duplicates { + sb.WriteString(".dups") + } + sb.WriteString(".dfs") + // range bytes would go here: `.from.to` + suffix := strconv.FormatUint(xxhash.Sum64([]byte(sb.String())), 32) + fmt.Println("sb.String()", sb.String(), xxhash.Sum64([]byte(sb.String()))) + return `"` + r.Cid.String() + ".car." + suffix + `"` +} + func ParseProtocolsString(v string) ([]multicodec.Code, error) { vs := strings.Split(v, ",") protocols := make([]multicodec.Code, 0, len(vs)) diff --git a/pkg/types/request_test.go b/pkg/types/request_test.go new file mode 100644 index 00000000..174cf7ae --- /dev/null +++ b/pkg/types/request_test.go @@ -0,0 +1,134 @@ +package types_test + +import ( + "fmt" + "testing" + + "github.com/filecoin-project/lassie/pkg/types" + "github.com/ipfs/go-cid" +) + +func TestEtag(t *testing.T) { + // To generate independent fixtures using Node.js, `npm install xxhash` then + // in a REPL: + // + // xx = (s) => require('xxhash').hash64(Buffer.from(s), 0).readBigUInt64LE(0).toString(32) + // + // then generate the suffix with the expected construction: + // + // xx('/ipfs/QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.dfs') + + testCases := []struct { + cid cid.Cid + path string + scope types.DagScope + dups bool + expected string + }{ + { + cid: cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"), + scope: types.DagScopeAll, + expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.58mf8vcmd2eo8"`, + }, + { + cid: cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"), + scope: types.DagScopeEntity, + expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.3t6g88g8u04i6"`, + }, + { + cid: cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"), + scope: types.DagScopeBlock, + expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.1fe71ua3km0b5"`, + }, + { + cid: cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"), + scope: types.DagScopeAll, + dups: true, + expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.4mglp6etuagob"`, + }, + { + cid: cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"), + scope: types.DagScopeEntity, + dups: true, + expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.fqhsp0g4l66m1"`, + }, + { + cid: cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"), + scope: types.DagScopeBlock, + dups: true, + expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.8u1ga109k62pp"`, + }, + { + cid: cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"), + scope: types.DagScopeAll, + path: "/some/path/to/thing", + expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.8q5lna3r43lgj"`, + }, + { + cid: cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"), + scope: types.DagScopeEntity, + path: "/some/path/to/thing", + expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.e4hni8qqgeove"`, + }, + { + cid: cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"), + scope: types.DagScopeBlock, + path: "/some/path/to/thing", + expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.7pdc786smhd1n"`, + }, + { + cid: cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"), + scope: types.DagScopeAll, + path: "/some/path/to/thing", + dups: true, + expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.bdfv1q76a1oem"`, + }, + { + cid: cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"), + scope: types.DagScopeEntity, + path: "/some/path/to/thing", + dups: true, + expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.790m13mh0recp"`, + }, + { + cid: cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"), + scope: types.DagScopeBlock, + path: "/some/path/to/thing", + dups: true, + expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.972jmjvd3o3"`, + }, + // path variations should be normalised + { + cid: cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"), + scope: types.DagScopeAll, + path: "some/path/to/thing", + expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.8q5lna3r43lgj"`, + }, + { + cid: cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"), + scope: types.DagScopeAll, + path: "///some//path//to/thing/", + expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.8q5lna3r43lgj"`, + }, + { + cid: cid.MustParse("bafyrgqhai26anf3i7pips7q22coa4sz2fr4gk4q4sqdtymvvjyginfzaqewveaeqdh524nsktaq43j65v22xxrybrtertmcfxufdam3da3hbk"), + scope: types.DagScopeAll, + expected: `"bafyrgqhai26anf3i7pips7q22coa4sz2fr4gk4q4sqdtymvvjyginfzaqewveaeqdh524nsktaq43j65v22xxrybrtertmcfxufdam3da3hbk.car.9lumqv26cg30t"`, + }, + } + + for _, tc := range testCases { + t.Run(fmt.Sprintf("%s:%s:%s:%v", tc.cid.String(), tc.path, tc.scope, tc.dups), func(t *testing.T) { + rr := types.RetrievalRequest{ + Cid: tc.cid, + Path: tc.path, + Scope: tc.scope, + Duplicates: tc.dups, + } + actual := rr.Etag() + if actual != tc.expected { + t.Errorf("expected %s, got %s", tc.expected, actual) + } + }) + } +} From d8acafcdd2f003952a184b941c9f84b458dd6e10 Mon Sep 17 00:00:00 2001 From: Kyle Huntsman <3432646+kylehuntsman@users.noreply.github.com> Date: Fri, 2 Jun 2023 18:07:19 -0700 Subject: [PATCH 2/2] chore: remove etag todo --- pkg/server/http/ipfs.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/server/http/ipfs.go b/pkg/server/http/ipfs.go index 89239f0c..02252062 100644 --- a/pkg/server/http/ipfs.go +++ b/pkg/server/http/ipfs.go @@ -154,7 +154,6 @@ func ipfsHandler(lassie *lassie.Lassie, cfg HttpServerConfig) func(http.Response res.Header().Set("Accept-Ranges", ResponseAcceptRangesHeader) res.Header().Set("Cache-Control", ResponseCacheControlHeader) res.Header().Set("Content-Type", ResponseContentTypeHeader) - // TODO: needs scope and path res.Header().Set("Etag", request.Etag()) res.Header().Set("X-Content-Type-Options", "nosniff") res.Header().Set("X-Ipfs-Path", "/"+datamodel.ParsePath(req.URL.Path).String())