feat: add stable etag

go.mod

@@ -55,7 +55,7 @@ require (
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bep/debounce v1.2.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/containerd/cgroups v1.1.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect

pkg/internal/itest/http_fetch_test.go

@@ -982,7 +982,10 @@ func TestHttpFetch(t *testing.T) {
 					req.Equal("public, max-age=29030400, immutable", resp.Header.Get("Cache-Control"))
 					req.Equal("application/vnd.ipld.car; version=1", resp.Header.Get("Content-Type"))
 					req.Equal("nosniff", resp.Header.Get("X-Content-Type-Options"))
-					req.Equal(fmt.Sprintf(`%s.car`, srcData[i].Root.String()), resp.Header.Get("ETag")) // TODO: needs scope and path too
+					etagStart := fmt.Sprintf(`"%s.car.`, srcData[i].Root.String())
+					etagGot := resp.Header.Get("ETag")
+					req.True(strings.HasPrefix(etagGot, etagStart), "ETag should start with [%s], got [%s]", etagStart, etagGot)
+					req.Equal(`"`, etagGot[len(etagGot)-1:], "ETag should end with a quote")
 					var path string
 					if testCase.paths != nil && testCase.paths[i] != "" {
 						path = testCase.paths[i]

pkg/server/http/ipfs.go

@@ -138,14 +138,23 @@ func ipfsHandler(lassie *lassie.Lassie, cfg HttpServerConfig) func(http.Response
 		}()
 		var store types.ReadableWritableStorage = carStore
 
+		request, err := types.NewRequestForPath(store, rootCid, path.String(), dagScope)
+		if err != nil {
+			errorResponse(res, statusLogger, http.StatusInternalServerError, fmt.Errorf("failed to create request: %w", err))
+			return
+		}
+		request.Protocols = protocols
+		request.FixedPeers = fixedPeers
+		request.RetrievalID = retrievalId
+		request.Duplicates = includeDupes // needed for etag
+
 		carWriter.OnPut(func(int) {
 			// called once we start writing blocks into the CAR (on the first Put())
 			res.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName))
 			res.Header().Set("Accept-Ranges", ResponseAcceptRangesHeader)
 			res.Header().Set("Cache-Control", ResponseCacheControlHeader)
 			res.Header().Set("Content-Type", ResponseContentTypeHeader)
-			// TODO: needs scope and path
-			res.Header().Set("Etag", fmt.Sprintf("%s.car", rootCid.String()))
+			res.Header().Set("Etag", request.Etag())
 			res.Header().Set("X-Content-Type-Options", "nosniff")
 			res.Header().Set("X-Ipfs-Path", "/"+datamodel.ParsePath(req.URL.Path).String())
 			// TODO: set X-Ipfs-Roots header when we support root+path
@@ -155,15 +164,6 @@ func ipfsHandler(lassie *lassie.Lassie, cfg HttpServerConfig) func(http.Response
 			close(bytesWritten)
 		}, true)
 
-		request, err := types.NewRequestForPath(store, rootCid, path.String(), dagScope)
-
-		if err != nil {
-			errorResponse(res, statusLogger, http.StatusInternalServerError, fmt.Errorf("failed to create request: %w", err))
-			return
-		}
-		request.Protocols = protocols
-		request.FixedPeers = fixedPeers
-		request.RetrievalID = retrievalId
 		// setup preload storage for bitswap, the temporary CAR store can set up a
 		// separate preload space in its storage
 		request.PreloadLinkSystem = cidlink.DefaultLinkSystem()

pkg/types/request.go

@@ -3,12 +3,15 @@ package types
 import (
 	"errors"
 	"fmt"
+	"strconv"
 	"strings"
 
+	"github.com/cespare/xxhash/v2"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-unixfsnode"
 	"github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/datamodel"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
 	ipldstorage "github.com/ipld/go-ipld-prime/storage"
 	"github.com/libp2p/go-libp2p/core/peer"
@@ -52,6 +55,7 @@ type RetrievalRequest struct {
 	Selector          ipld.Node
 	Path              string
 	Scope             DagScope
+	Duplicates        bool
 	Protocols         []multicodec.Code
 	PreloadLinkSystem ipld.LinkSystem
 	MaxBlocks         uint64
@@ -142,6 +146,29 @@ func (r RetrievalRequest) GetSupportedProtocols(allSupportedProtocols []multicod
 	return supportedProtocols
 }
 
+func (r RetrievalRequest) Etag() string {
+	// https://github.com/ipfs/boxo/pull/303/commits/f61f95481041406df46a1781b1daab34b6605650#r1213918777
+	sb := strings.Builder{}
+	sb.WriteString("/ipfs/")
+	sb.WriteString(r.Cid.String())
+	if r.Path != "" {
+		sb.WriteString("/")
+		sb.WriteString(datamodel.ParsePath(r.Path).String())
+	}
+	if r.Scope != DagScopeAll {
+		sb.WriteString(".")
+		sb.WriteString(string(r.Scope))
+	}
+	if r.Duplicates {
+		sb.WriteString(".dups")
+	}
+	sb.WriteString(".dfs")
+	// range bytes would go here: `.from.to`
+	suffix := strconv.FormatUint(xxhash.Sum64([]byte(sb.String())), 32)
+	fmt.Println("sb.String()", sb.String(), xxhash.Sum64([]byte(sb.String())))
+	return `"` + r.Cid.String() + ".car." + suffix + `"`
+}
+
 func ParseProtocolsString(v string) ([]multicodec.Code, error) {
 	vs := strings.Split(v, ",")
 	protocols := make([]multicodec.Code, 0, len(vs))

pkg/types/request_test.go

@@ -0,0 +1,134 @@
+package types_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/filecoin-project/lassie/pkg/types"
+	"github.com/ipfs/go-cid"
+)
+
+func TestEtag(t *testing.T) {
+	// To generate independent fixtures using Node.js, `npm install xxhash` then
+	// in a REPL:
+	//
+	//   xx = (s) => require('xxhash').hash64(Buffer.from(s), 0).readBigUInt64LE(0).toString(32)
+	//
+	// then generate the suffix with the expected construction:
+	//
+	//   xx('/ipfs/QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.dfs')
+
+	testCases := []struct {
+		cid      cid.Cid
+		path     string
+		scope    types.DagScope
+		dups     bool
+		expected string
+	}{
+		{
+			cid:      cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"),
+			scope:    types.DagScopeAll,
+			expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.58mf8vcmd2eo8"`,
+		},
+		{
+			cid:      cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"),
+			scope:    types.DagScopeEntity,
+			expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.3t6g88g8u04i6"`,
+		},
+		{
+			cid:      cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"),
+			scope:    types.DagScopeBlock,
+			expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.1fe71ua3km0b5"`,
+		},
+		{
+			cid:      cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"),
+			scope:    types.DagScopeAll,
+			dups:     true,
+			expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.4mglp6etuagob"`,
+		},
+		{
+			cid:      cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"),
+			scope:    types.DagScopeEntity,
+			dups:     true,
+			expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.fqhsp0g4l66m1"`,
+		},
+		{
+			cid:      cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"),
+			scope:    types.DagScopeBlock,
+			dups:     true,
+			expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.8u1ga109k62pp"`,
+		},
+		{
+			cid:      cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"),
+			scope:    types.DagScopeAll,
+			path:     "/some/path/to/thing",
+			expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.8q5lna3r43lgj"`,
+		},
+		{
+			cid:      cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"),
+			scope:    types.DagScopeEntity,
+			path:     "/some/path/to/thing",
+			expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.e4hni8qqgeove"`,
+		},
+		{
+			cid:      cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"),
+			scope:    types.DagScopeBlock,
+			path:     "/some/path/to/thing",
+			expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.7pdc786smhd1n"`,
+		},
+		{
+			cid:      cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"),
+			scope:    types.DagScopeAll,
+			path:     "/some/path/to/thing",
+			dups:     true,
+			expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.bdfv1q76a1oem"`,
+		},
+		{
+			cid:      cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"),
+			scope:    types.DagScopeEntity,
+			path:     "/some/path/to/thing",
+			dups:     true,
+			expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.790m13mh0recp"`,
+		},
+		{
+			cid:      cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"),
+			scope:    types.DagScopeBlock,
+			path:     "/some/path/to/thing",
+			dups:     true,
+			expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.972jmjvd3o3"`,
+		},
+		// path variations should be normalised
+		{
+			cid:      cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"),
+			scope:    types.DagScopeAll,
+			path:     "some/path/to/thing",
+			expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.8q5lna3r43lgj"`,
+		},
+		{
+			cid:      cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"),
+			scope:    types.DagScopeAll,
+			path:     "///some//path//to/thing/",
+			expected: `"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi.car.8q5lna3r43lgj"`,
+		},
+		{
+			cid:      cid.MustParse("bafyrgqhai26anf3i7pips7q22coa4sz2fr4gk4q4sqdtymvvjyginfzaqewveaeqdh524nsktaq43j65v22xxrybrtertmcfxufdam3da3hbk"),
+			scope:    types.DagScopeAll,
+			expected: `"bafyrgqhai26anf3i7pips7q22coa4sz2fr4gk4q4sqdtymvvjyginfzaqewveaeqdh524nsktaq43j65v22xxrybrtertmcfxufdam3da3hbk.car.9lumqv26cg30t"`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("%s:%s:%s:%v", tc.cid.String(), tc.path, tc.scope, tc.dups), func(t *testing.T) {
+			rr := types.RetrievalRequest{
+				Cid:        tc.cid,
+				Path:       tc.path,
+				Scope:      tc.scope,
+				Duplicates: tc.dups,
+			}
+			actual := rr.Etag()
+			if actual != tc.expected {
+				t.Errorf("expected %s, got %s", tc.expected, actual)
+			}
+		})
+	}
+}