chore: [email protected]

go.mod

@@ -22,11 +22,11 @@ require (
 	github.com/ipfs/go-ipld-format v0.6.0
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipfs/go-unixfsnode v1.8.0
-	github.com/ipld/go-car/v2 v2.11.0
+	github.com/ipld/go-car/v2 v2.13.1
 	github.com/ipld/go-codec-dagpb v1.6.0
 	github.com/ipld/go-ipld-prime v0.21.0
-	github.com/ipld/go-trustless-utils v0.0.0
-	github.com/ipld/ipld/specs v0.0.0-20230826120441-91918996e8eb
+	github.com/ipld/go-trustless-utils v0.1.1-0.20230907055204-83d1e838c1c1
+	github.com/ipld/ipld/specs v0.0.0-20230907004443-0e4ff95ff474
 	github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff
 	github.com/libp2p/go-libp2p v0.30.0
 	github.com/libp2p/go-libp2p-routing-helpers v0.7.0
@@ -90,7 +90,7 @@ require (
 	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
 	github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
 	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
-	github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
+	github.com/ipfs/go-ipld-cbor v0.1.0 // indirect
 	github.com/ipfs/go-log v1.0.5 // indirect
 	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
 	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
@@ -149,7 +149,7 @@ require (
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/warpfork/go-testmark v0.12.1 // indirect
 	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
-	github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
+	github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 // indirect
 	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	go.opentelemetry.io/otel/sdk v1.14.0 // indirect

go.sum

@@ -288,8 +288,9 @@ github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA
 github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4=
 github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4=
 github.com/ipfs/go-ipld-cbor v0.0.6-0.20211211231443-5d9b9e1f6fa8/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA=
-github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0=
 github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA=
+github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs=
+github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk=
 github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms=
 github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k=
 github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten1F5U=
@@ -317,17 +318,17 @@ github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU=
 github.com/ipfs/go-unixfsnode v1.8.0 h1:yCkakzuE365glu+YkgzZt6p38CSVEBPgngL9ZkfnyQU=
 github.com/ipfs/go-unixfsnode v1.8.0/go.mod h1:HxRu9HYHOjK6HUqFBAi++7DVoWAHn0o4v/nZ/VA+0g8=
 github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
-github.com/ipld/go-car/v2 v2.11.0 h1:lkAPwbbTFqbdfawgm+bfmFc8PjGC7D12VcaLXPCLNfM=
-github.com/ipld/go-car/v2 v2.11.0/go.mod h1:aDszqev0zjtU8l96g4lwXHaU9bzArj56Y7eEN0q/xqA=
+github.com/ipld/go-car/v2 v2.13.1 h1:KnlrKvEPEzr5IZHKTXLAEub+tPrzeAFQVRlSQvuxBO4=
+github.com/ipld/go-car/v2 v2.13.1/go.mod h1:QkdjjFNGit2GIkpQ953KBwowuoukoM75nP/JI1iDJdo=
 github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
 github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s=
 github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E=
 github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
-github.com/ipld/go-trustless-utils v0.0.0 h1:X+hY7lWZSd6kdBGfmGtEIjCUeqETaeahMKjont7+OGo=
-github.com/ipld/go-trustless-utils v0.0.0/go.mod h1:rqvDGdgk2acsKiGyV5mzCZZRK8JtnEZqvUBxLuYxR6A=
-github.com/ipld/ipld/specs v0.0.0-20230826120441-91918996e8eb h1:5ARxkQ9NqZq33RM7i/Eq3bvBj2RBxx2xE63hqfa+9KY=
-github.com/ipld/ipld/specs v0.0.0-20230826120441-91918996e8eb/go.mod h1:AfGlAr20WOjV5PyCowEnGY3pAm5x5i+o0R8IUeir6cs=
+github.com/ipld/go-trustless-utils v0.1.1-0.20230907055204-83d1e838c1c1 h1:E1YXkAnvFn0lXkiocREtp0pUBydIwMW94myLBLCvhsY=
+github.com/ipld/go-trustless-utils v0.1.1-0.20230907055204-83d1e838c1c1/go.mod h1:nbUA6YuKZqNHYd3mYKALxOQa5aC8fnr2w+OxirSSOqc=
+github.com/ipld/ipld/specs v0.0.0-20230907004443-0e4ff95ff474 h1:CiDtcUFyzRwhKyiS4Gn+fzdraMoNVtPHd+wVIekExPc=
+github.com/ipld/ipld/specs v0.0.0-20230907004443-0e4ff95ff474/go.mod h1:WcT0DfRe+e2QFY0kcbsOnuT6jL5Q0JNZ83I5DHIdStg=
 github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff h1:xbKrIvnpQkbF8iHPk/HGcegsypCDpcXWHhzBCLyCWf8=
 github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff/go.mod h1:paYP9U4N3/vOzGCuN9kU972vtvw9JUcQjOKyiCFGwRk=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
@@ -608,8 +609,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:f
 github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20210118024343-169e9d70c0c2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20210303213153-67a261a1d291/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
-github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o=
-github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
+github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 h1:yVYDLoN2gmB3OdBXFW8e1UwgVbmCvNlnAKhvHPaNARI=
+github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=

pkg/internal/itest/http_fetch_test.go

@@ -1143,7 +1143,7 @@ func TestHttpFetch(t *testing.T) {
 					req.Equal("public, max-age=29030400, immutable", resp.Header.Get("Cache-Control"))
 					ct := resp.Header.Get("Content-Type")
 					// TODO: check the noDups() case and assert accordingly
-					req.Equal("application/vnd.ipld.car; version=1; order=dfs; dups=", ct[:len(ct)-1]) // strip off y|n from the end
+					req.Equal("application/vnd.ipld.car;version=1;order=dfs;dups=", ct[:len(ct)-1]) // strip off y|n from the end
 					req.Equal("nosniff", resp.Header.Get("X-Content-Type-Options"))
 					etagStart := fmt.Sprintf(`"%s.car.`, srcData[i].Root.String())
 					etagGot := resp.Header.Get("ETag")

pkg/internal/itest/trustless_fetch_test.go

@@ -27,7 +27,7 @@ import (
 func TestTrustlessUnixfsFetch(t *testing.T) {
 	req := require.New(t)
 
-	testCases, err := trustlesspathing.Unixfs20mVarietyCases()
+	testCases, _, err := trustlesspathing.Unixfs20mVarietyCases()
 	req.NoError(err)
 	storage, closer, err := trustlesspathing.Unixfs20mVarietyReadableStorage()
 	req.NoError(err)
@@ -112,7 +112,7 @@ func TestTrustlessUnixfsFetch(t *testing.T) {
 				req.Equal(fmt.Sprintf(`attachment; filename="%s.car"`, tc.Root.String()), resp.Header.Get("Content-Disposition"))
 				req.Equal("none", resp.Header.Get("Accept-Ranges"))
 				req.Equal("public, max-age=29030400, immutable", resp.Header.Get("Cache-Control"))
-				req.Equal("application/vnd.ipld.car; version=1; order=dfs; dups=y", resp.Header.Get("Content-Type"))
+				req.Equal("application/vnd.ipld.car;version=1;order=dfs;dups=y", resp.Header.Get("Content-Type"))
 				req.Equal("nosniff", resp.Header.Get("X-Content-Type-Options"))
 				etagStart := fmt.Sprintf(`"%s.car.`, tc.Root.String())
 				etagGot := resp.Header.Get("ETag")

pkg/internal/testutil/mockroundtripper.go

@@ -109,7 +109,7 @@ func (mrt *MockRoundTripper) RoundTrip(req *http.Request) (*http.Response, error
 		expectedScope = scope
 	}
 	require.Equal(mrt.t, req.URL.RawQuery, fmt.Sprintf("dag-scope=%s", expectedScope))
-	require.Equal(mrt.t, []string{"application/vnd.ipld.car; version=1; order=dfs; dups=y"}, req.Header["Accept"])
+	require.Equal(mrt.t, []string{"application/vnd.ipld.car;version=1;order=dfs;dups=y"}, req.Header["Accept"])
 	reqId := req.Header["X-Request-Id"]
 	require.Len(mrt.t, reqId, 1)
 	_, err = uuid.Parse(reqId[0])

pkg/retriever/httpretriever.go

@@ -118,9 +118,10 @@ func (ph *ProtocolHttp) Retrieve(
 		return nil, ErrHttpRequestFailure{Code: resp.StatusCode}
 	}
 
-	// if Content-Type is not valid, it will default to
-	// trustlesshttp.DefaultIncludeDupes (true)
-	_, expectDuplicates := trustlesshttp.ParseContentType(resp.Header.Get("Content-Type"))
+	var expectDuplicates = trustlesshttp.DefaultIncludeDupes
+	if contentType, valid := trustlesshttp.ParseContentType(resp.Header.Get("Content-Type")); valid {
+		expectDuplicates = contentType.Duplicates
+	} // else be permissive and just expect duplicates (DefaultIncludeDupes)
 
 	var ttfb time.Duration
 	rdr := newTimeToFirstByteReader(resp.Body, func() {
@@ -188,7 +189,7 @@ func makeRequest(ctx context.Context, request types.RetrievalRequest, candidate
 		logger.Warnf("Couldn't construct a http request %s: %v", candidate.MinerPeer.ID, err)
 		return nil, fmt.Errorf("%w for peer %s: %v", ErrBadPathForRequest, candidate.MinerPeer.ID, err)
 	}
-	req.Header.Add("Accept", trustlesshttp.RequestAcceptHeader(true)) // prefer duplicates
+	req.Header.Add("Accept", trustlesshttp.DefaultContentType().String()) // prefer duplicates
 	req.Header.Add("X-Request-Id", request.RetrievalID.String())
 	req.Header.Add("User-Agent", build.UserAgent)
 

pkg/server/http/ipfs.go

@@ -82,7 +82,7 @@ func IpfsHandler(fetcher types.Fetcher, cfg HttpServerConfig) func(http.Response
 			res.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName))
 			res.Header().Set("Accept-Ranges", "none")
 			res.Header().Set("Cache-Control", trustlesshttp.ResponseCacheControlHeader)
-			res.Header().Set("Content-Type", trustlesshttp.ResponseContentTypeHeader(request.Duplicates))
+			res.Header().Set("Content-Type", trustlesshttp.DefaultContentType().WithDuplicates(request.Duplicates).String())
 			res.Header().Set("Etag", request.Etag())
 			res.Header().Set("X-Content-Type-Options", "nosniff")
 			res.Header().Set("X-Ipfs-Path", trustlessutils.PathEscape(req.URL.Path))
@@ -163,7 +163,7 @@ func decodeRequest(res http.ResponseWriter, req *http.Request, statusLogger *sta
 		return false, trustlessutils.Request{}
 	}
 
-	includeDupes, err := trustlesshttp.CheckFormat(req)
+	accept, err := trustlesshttp.CheckFormat(req)
 	if err != nil {
 		errorResponse(res, statusLogger, http.StatusBadRequest, err)
 		return false, trustlessutils.Request{}
@@ -186,7 +186,7 @@ func decodeRequest(res http.ResponseWriter, req *http.Request, statusLogger *sta
 		Path:       path.String(),
 		Scope:      dagScope,
 		Bytes:      byteRange,
-		Duplicates: includeDupes,
+		Duplicates: accept.Duplicates,
 	}
 }