diff --git a/go.mod b/go.mod index ea71e415..eea676da 100644 --- a/go.mod +++ b/go.mod @@ -22,11 +22,11 @@ require ( github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-unixfsnode v1.8.0 - github.com/ipld/go-car/v2 v2.11.0 + github.com/ipld/go-car/v2 v2.13.1 github.com/ipld/go-codec-dagpb v1.6.0 github.com/ipld/go-ipld-prime v0.21.0 - github.com/ipld/go-trustless-utils v0.0.0 - github.com/ipld/ipld/specs v0.0.0-20230826120441-91918996e8eb + github.com/ipld/go-trustless-utils v0.1.1-0.20230907055204-83d1e838c1c1 + github.com/ipld/ipld/specs v0.0.0-20230907004443-0e4ff95ff474 github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff github.com/libp2p/go-libp2p v0.30.0 github.com/libp2p/go-libp2p-routing-helpers v0.7.0 @@ -90,7 +90,7 @@ require ( github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect - github.com/ipfs/go-ipld-cbor v0.0.6 // indirect + github.com/ipfs/go-ipld-cbor v0.1.0 // indirect github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-peertaskqueue v0.8.1 // indirect @@ -149,7 +149,7 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/warpfork/go-testmark v0.12.1 // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect - github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect + github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.opentelemetry.io/otel/sdk v1.14.0 // indirect diff --git a/go.sum b/go.sum index 93ab667c..d8a96708 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,9 @@ github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-cbor v0.0.6-0.20211211231443-5d9b9e1f6fa8/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= -github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0= github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= +github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs= +github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten1F5U= @@ -317,17 +318,17 @@ github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= github.com/ipfs/go-unixfsnode v1.8.0 h1:yCkakzuE365glu+YkgzZt6p38CSVEBPgngL9ZkfnyQU= github.com/ipfs/go-unixfsnode v1.8.0/go.mod h1:HxRu9HYHOjK6HUqFBAi++7DVoWAHn0o4v/nZ/VA+0g8= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= -github.com/ipld/go-car/v2 v2.11.0 h1:lkAPwbbTFqbdfawgm+bfmFc8PjGC7D12VcaLXPCLNfM= -github.com/ipld/go-car/v2 v2.11.0/go.mod h1:aDszqev0zjtU8l96g4lwXHaU9bzArj56Y7eEN0q/xqA= +github.com/ipld/go-car/v2 v2.13.1 h1:KnlrKvEPEzr5IZHKTXLAEub+tPrzeAFQVRlSQvuxBO4= +github.com/ipld/go-car/v2 v2.13.1/go.mod h1:QkdjjFNGit2GIkpQ953KBwowuoukoM75nP/JI1iDJdo= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= -github.com/ipld/go-trustless-utils v0.0.0 h1:X+hY7lWZSd6kdBGfmGtEIjCUeqETaeahMKjont7+OGo= -github.com/ipld/go-trustless-utils v0.0.0/go.mod h1:rqvDGdgk2acsKiGyV5mzCZZRK8JtnEZqvUBxLuYxR6A= -github.com/ipld/ipld/specs v0.0.0-20230826120441-91918996e8eb h1:5ARxkQ9NqZq33RM7i/Eq3bvBj2RBxx2xE63hqfa+9KY= -github.com/ipld/ipld/specs v0.0.0-20230826120441-91918996e8eb/go.mod h1:AfGlAr20WOjV5PyCowEnGY3pAm5x5i+o0R8IUeir6cs= +github.com/ipld/go-trustless-utils v0.1.1-0.20230907055204-83d1e838c1c1 h1:E1YXkAnvFn0lXkiocREtp0pUBydIwMW94myLBLCvhsY= +github.com/ipld/go-trustless-utils v0.1.1-0.20230907055204-83d1e838c1c1/go.mod h1:nbUA6YuKZqNHYd3mYKALxOQa5aC8fnr2w+OxirSSOqc= +github.com/ipld/ipld/specs v0.0.0-20230907004443-0e4ff95ff474 h1:CiDtcUFyzRwhKyiS4Gn+fzdraMoNVtPHd+wVIekExPc= +github.com/ipld/ipld/specs v0.0.0-20230907004443-0e4ff95ff474/go.mod h1:WcT0DfRe+e2QFY0kcbsOnuT6jL5Q0JNZ83I5DHIdStg= github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff h1:xbKrIvnpQkbF8iHPk/HGcegsypCDpcXWHhzBCLyCWf8= github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff/go.mod h1:paYP9U4N3/vOzGCuN9kU972vtvw9JUcQjOKyiCFGwRk= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= @@ -608,8 +609,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:f github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20210118024343-169e9d70c0c2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20210303213153-67a261a1d291/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o= -github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 h1:yVYDLoN2gmB3OdBXFW8e1UwgVbmCvNlnAKhvHPaNARI= +github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= diff --git a/pkg/internal/itest/http_fetch_test.go b/pkg/internal/itest/http_fetch_test.go index 96a22e41..f1f34c11 100644 --- a/pkg/internal/itest/http_fetch_test.go +++ b/pkg/internal/itest/http_fetch_test.go @@ -1143,7 +1143,7 @@ func TestHttpFetch(t *testing.T) { req.Equal("public, max-age=29030400, immutable", resp.Header.Get("Cache-Control")) ct := resp.Header.Get("Content-Type") // TODO: check the noDups() case and assert accordingly - req.Equal("application/vnd.ipld.car; version=1; order=dfs; dups=", ct[:len(ct)-1]) // strip off y|n from the end + req.Equal("application/vnd.ipld.car;version=1;order=dfs;dups=", ct[:len(ct)-1]) // strip off y|n from the end req.Equal("nosniff", resp.Header.Get("X-Content-Type-Options")) etagStart := fmt.Sprintf(`"%s.car.`, srcData[i].Root.String()) etagGot := resp.Header.Get("ETag") diff --git a/pkg/internal/itest/trustless_fetch_test.go b/pkg/internal/itest/trustless_fetch_test.go index 7987c444..9b47cda4 100644 --- a/pkg/internal/itest/trustless_fetch_test.go +++ b/pkg/internal/itest/trustless_fetch_test.go @@ -27,7 +27,7 @@ import ( func TestTrustlessUnixfsFetch(t *testing.T) { req := require.New(t) - testCases, err := trustlesspathing.Unixfs20mVarietyCases() + testCases, _, err := trustlesspathing.Unixfs20mVarietyCases() req.NoError(err) storage, closer, err := trustlesspathing.Unixfs20mVarietyReadableStorage() req.NoError(err) @@ -112,7 +112,7 @@ func TestTrustlessUnixfsFetch(t *testing.T) { req.Equal(fmt.Sprintf(`attachment; filename="%s.car"`, tc.Root.String()), resp.Header.Get("Content-Disposition")) req.Equal("none", resp.Header.Get("Accept-Ranges")) req.Equal("public, max-age=29030400, immutable", resp.Header.Get("Cache-Control")) - req.Equal("application/vnd.ipld.car; version=1; order=dfs; dups=y", resp.Header.Get("Content-Type")) + req.Equal("application/vnd.ipld.car;version=1;order=dfs;dups=y", resp.Header.Get("Content-Type")) req.Equal("nosniff", resp.Header.Get("X-Content-Type-Options")) etagStart := fmt.Sprintf(`"%s.car.`, tc.Root.String()) etagGot := resp.Header.Get("ETag") diff --git a/pkg/internal/testutil/mockroundtripper.go b/pkg/internal/testutil/mockroundtripper.go index 68d7fd83..0c75ece6 100644 --- a/pkg/internal/testutil/mockroundtripper.go +++ b/pkg/internal/testutil/mockroundtripper.go @@ -109,7 +109,7 @@ func (mrt *MockRoundTripper) RoundTrip(req *http.Request) (*http.Response, error expectedScope = scope } require.Equal(mrt.t, req.URL.RawQuery, fmt.Sprintf("dag-scope=%s", expectedScope)) - require.Equal(mrt.t, []string{"application/vnd.ipld.car; version=1; order=dfs; dups=y"}, req.Header["Accept"]) + require.Equal(mrt.t, []string{"application/vnd.ipld.car;version=1;order=dfs;dups=y"}, req.Header["Accept"]) reqId := req.Header["X-Request-Id"] require.Len(mrt.t, reqId, 1) _, err = uuid.Parse(reqId[0]) diff --git a/pkg/retriever/httpretriever.go b/pkg/retriever/httpretriever.go index ef726cd1..c7f6221b 100644 --- a/pkg/retriever/httpretriever.go +++ b/pkg/retriever/httpretriever.go @@ -118,9 +118,10 @@ func (ph *ProtocolHttp) Retrieve( return nil, ErrHttpRequestFailure{Code: resp.StatusCode} } - // if Content-Type is not valid, it will default to - // trustlesshttp.DefaultIncludeDupes (true) - _, expectDuplicates := trustlesshttp.ParseContentType(resp.Header.Get("Content-Type")) + var expectDuplicates = trustlesshttp.DefaultIncludeDupes + if contentType, valid := trustlesshttp.ParseContentType(resp.Header.Get("Content-Type")); valid { + expectDuplicates = contentType.Duplicates + } // else be permissive and just expect duplicates (DefaultIncludeDupes) var ttfb time.Duration rdr := newTimeToFirstByteReader(resp.Body, func() { @@ -188,7 +189,7 @@ func makeRequest(ctx context.Context, request types.RetrievalRequest, candidate logger.Warnf("Couldn't construct a http request %s: %v", candidate.MinerPeer.ID, err) return nil, fmt.Errorf("%w for peer %s: %v", ErrBadPathForRequest, candidate.MinerPeer.ID, err) } - req.Header.Add("Accept", trustlesshttp.RequestAcceptHeader(true)) // prefer duplicates + req.Header.Add("Accept", trustlesshttp.DefaultContentType().String()) // prefer duplicates req.Header.Add("X-Request-Id", request.RetrievalID.String()) req.Header.Add("User-Agent", build.UserAgent) diff --git a/pkg/server/http/ipfs.go b/pkg/server/http/ipfs.go index f61ba434..bdcbc23b 100644 --- a/pkg/server/http/ipfs.go +++ b/pkg/server/http/ipfs.go @@ -82,7 +82,7 @@ func IpfsHandler(fetcher types.Fetcher, cfg HttpServerConfig) func(http.Response res.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName)) res.Header().Set("Accept-Ranges", "none") res.Header().Set("Cache-Control", trustlesshttp.ResponseCacheControlHeader) - res.Header().Set("Content-Type", trustlesshttp.ResponseContentTypeHeader(request.Duplicates)) + res.Header().Set("Content-Type", trustlesshttp.DefaultContentType().WithDuplicates(request.Duplicates).String()) res.Header().Set("Etag", request.Etag()) res.Header().Set("X-Content-Type-Options", "nosniff") res.Header().Set("X-Ipfs-Path", trustlessutils.PathEscape(req.URL.Path)) @@ -163,7 +163,7 @@ func decodeRequest(res http.ResponseWriter, req *http.Request, statusLogger *sta return false, trustlessutils.Request{} } - includeDupes, err := trustlesshttp.CheckFormat(req) + accept, err := trustlesshttp.CheckFormat(req) if err != nil { errorResponse(res, statusLogger, http.StatusBadRequest, err) return false, trustlessutils.Request{} @@ -186,7 +186,7 @@ func decodeRequest(res http.ResponseWriter, req *http.Request, statusLogger *sta Path: path.String(), Scope: dagScope, Bytes: byteRange, - Duplicates: includeDupes, + Duplicates: accept.Duplicates, } }