From 83eec5da3c569c7aa50a9690bb487624dcf13be9 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 9 Jun 2023 16:42:26 -0400 Subject: [PATCH 01/46] chore: update boxo --- lib/gateway_traversal.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lib/gateway_traversal.go diff --git a/lib/gateway_traversal.go b/lib/gateway_traversal.go new file mode 100644 index 0000000..e69de29 From ce24a3f161dab007639464849629b3036e732983 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 9 Jun 2023 20:13:52 -0400 Subject: [PATCH 02/46] feat: switch GetCAR to be backpressured --- lib/gateway_traversal.go | 312 +++++++++++++++++++++++++++++++++++++++ lib/graph_gateway.go | 96 ++++++++++-- 2 files changed, 396 insertions(+), 12 deletions(-) diff --git a/lib/gateway_traversal.go b/lib/gateway_traversal.go index e69de29..27ac8ab 100644 --- a/lib/gateway_traversal.go +++ b/lib/gateway_traversal.go @@ -0,0 +1,312 @@ +package lib + +import ( + "context" + "errors" + "fmt" + "io" + "sync" + "time" + + "github.com/ipfs/boxo/blockservice" + "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/exchange" + bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" + "github.com/ipfs/boxo/gateway" + ipfspath "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path/resolver" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-unixfsnode" + "github.com/ipfs/go-unixfsnode/data" + "github.com/ipld/go-car" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/schema" + "github.com/ipld/go-ipld-prime/traversal" + "github.com/ipld/go-ipld-prime/traversal/selector" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" +) + +type getBlock func(ctx context.Context, cid cid.Cid) (blocks.Block, error) + +type gbf struct { + fn getBlock +} + +func (g *gbf) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { + return g.fn(ctx, c) +} + +func (g *gbf) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { + ch := make(chan blocks.Block, len(cids)) + go func() { + defer close(ch) + for _, c := range cids { + blk, err := g.fn(ctx, c) + if err != nil { + return + } + select { + case ch <- blk: + case <-ctx.Done(): + return + } + } + }() + return ch, nil +} + +var _ exchange.Fetcher = (*gbf)(nil) + +func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *GraphGatewayMetrics) (getBlock, error) { + cr, err := car.NewCarReader(reader) + if err != nil { + return nil, err + } + + cbCtx, cncl := context.WithCancel(ctx) + defer cncl() + + type blockRead struct { + block blocks.Block + err error + } + + blkCh := make(chan blockRead, 1) + go func() { + defer close(blkCh) + for { + blk, rdErr := cr.Next() + select { + case blkCh <- blockRead{blk, rdErr}: + case <-cbCtx.Done(): + return + } + } + }() + + isFirstBlock := true + mx := sync.Mutex{} + + return func(ctx context.Context, c cid.Cid) (blocks.Block, error) { + mx.Lock() + defer mx.Unlock() + + // initially set a higher timeout here so that if there's an initial timeout error we get it from the car reader. + var t *time.Timer + if isFirstBlock { + t = time.NewTimer(GetBlockTimeout * 2) + } else { + t = time.NewTimer(GetBlockTimeout) + } + var blkRead blockRead + var ok bool + select { + case blkRead, ok = <-blkCh: + if !t.Stop() { + <-t.C + } + t.Reset(GetBlockTimeout) + case <-t.C: + return nil, gateway.ErrGatewayTimeout + } + if !ok || blkRead.err != nil { + if errors.Is(blkRead.err, io.EOF) { + return nil, io.EOF + } + return nil, blkRead.err + } + if blkRead.block != nil { + metrics.carBlocksFetchedMetric.Inc() + if !blkRead.block.Cid().Equals(c) { + return nil, fmt.Errorf("unexpected block received") + } + return blkRead.block, nil + } + return nil, fmt.Errorf("received a nil block with no error") + }, nil +} + +func getIPFSPathResolverAndLsysFromBlockReader(ctx context.Context, fn getBlock) (resolver.Resolver, *ipld.LinkSystem) { + fetcher := bsfetcher.NewFetcherConfig( + blockservice.New(blockstore.NewBlockstore(&datastore.NullDatastore{}), &blockFetcherExchWrapper{f: &gbf{fn}})) + fetcher.NodeReifier = unixfsnode.Reify + res := resolver.NewBasicResolver(fetcher) + lsys := cidlink.DefaultLinkSystem() + unixfsnode.AddUnixFSReificationToLinkSystem(&lsys) + return res, &lsys +} + +// walkGatewaySimpleSelector walks the subgraph described by the path and terminal element parameters +func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params gateway.CarParams, lsys *ipld.LinkSystem, pathResolver resolver.Resolver) error { + // First resolve the path since we always need to. + lastCid, remainder, err := pathResolver.ResolveToLastNode(ctx, p) + if err != nil { + return err + } + + lctx := ipld.LinkContext{Ctx: ctx} + pathTerminalCidLink := cidlink.Link{Cid: lastCid} + + // If the scope is the block, now we only need to retrieve the root block of the last element of the path. + if params.Scope == gateway.DagScopeBlock { + _, err = lsys.LoadRaw(lctx, pathTerminalCidLink) + return err + } + + // If we're asking for everything then give it + if params.Scope == gateway.DagScopeAll { + lastCidNode, err := lsys.Load(lctx, pathTerminalCidLink, basicnode.Prototype.Any) + if err != nil { + return err + } + + sel, err := selector.ParseSelector(selectorparse.CommonSelector_ExploreAllRecursively) + if err != nil { + return err + } + + progress := traversal.Progress{ + Cfg: &traversal.Config{ + Ctx: ctx, + LinkSystem: *lsys, + LinkTargetNodePrototypeChooser: bsfetcher.DefaultPrototypeChooser, + LinkVisitOnlyOnce: true, // This is safe for the "all" selector + }, + } + + if err := progress.WalkMatching(lastCidNode, sel, func(progress traversal.Progress, node datamodel.Node) error { + return nil + }); err != nil { + return err + } + return nil + } + + // From now on, dag-scope=entity! + // Since we need more of the graph load it to figure out what we have + // This includes determining if the terminal node is UnixFS or not + pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + + np, err := pc(pathTerminalCidLink, lctx) + if err != nil { + return err + } + + lastCidNode, err := lsys.Load(lctx, pathTerminalCidLink, np) + if err != nil { + return err + } + + if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { + // If it's not valid dag-pb then we're done + return nil + } else if len(remainder) > 0 { + // If we're trying to path into dag-pb node that's invalid and we're done + return nil + } else if !pbn.FieldData().Exists() { + // If it's not valid UnixFS then we're done + return nil + } else if unixfsFieldData, decodeErr := data.DecodeUnixFSData(pbn.Data.Must().Bytes()); decodeErr != nil { + // If it's not valid dag-pb and UnixFS then we're done + return nil + } else { + switch unixfsFieldData.FieldDataType().Int() { + case data.Data_Directory, data.Data_Symlink: + // These types are non-recursive so we're done + return nil + case data.Data_Raw, data.Data_Metadata: + // TODO: for now, we decided to return nil here. The different implementations are inconsistent + // and UnixFS is not properly specified: https://github.com/ipfs/specs/issues/316. + // - Is Data_Raw different from Data_File? + // - Data_Metadata is handled differently in boxo/ipld/unixfs and go-unixfsnode. + return nil + case data.Data_HAMTShard: + // Return all elements in the map + _, err := lsys.KnownReifiers["unixfs-preload"](lctx, lastCidNode, lsys) + if err != nil { + return err + } + return nil + case data.Data_File: + nd, err := unixfsnode.Reify(lctx, lastCidNode, lsys) + if err != nil { + return err + } + + fnd, ok := nd.(datamodel.LargeBytesNode) + if !ok { + return fmt.Errorf("could not process file since it did not present as large bytes") + } + f, err := fnd.AsLargeBytes() + if err != nil { + return err + } + + // Get the entity range. If it's empty, assume the defaults (whole file). + entityRange := params.Range + if entityRange == nil { + entityRange = &gateway.DagByteRange{ + From: 0, + } + } + + from := entityRange.From + + // If we're starting to read based on the end of the file, find out where that is. + var fileLength int64 + foundFileLength := false + if entityRange.From < 0 { + fileLength, err = f.Seek(0, io.SeekEnd) + if err != nil { + return err + } + from = fileLength + entityRange.From + foundFileLength = true + } + + // If we're reading until the end of the file then do it + if entityRange.To == nil { + if _, err := f.Seek(from, io.SeekStart); err != nil { + return err + } + _, err = io.Copy(io.Discard, f) + return err + } + + to := *entityRange.To + if (*entityRange.To) < 0 && !foundFileLength { + fileLength, err = f.Seek(0, io.SeekEnd) + if err != nil { + return err + } + to = fileLength + *entityRange.To + foundFileLength = true + } + + numToRead := 1 + to - from + if numToRead < 0 { + return fmt.Errorf("tried to read less than zero bytes") + } + + if _, err := f.Seek(from, io.SeekStart); err != nil { + return err + } + _, err = io.CopyN(io.Discard, f, numToRead) + return err + default: + // Not a supported type, so we're done + return nil + } + } +} diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 21fc287..d010c99 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -29,6 +29,8 @@ import ( format "github.com/ipfs/go-ipld-format" golog "github.com/ipfs/go-log/v2" "github.com/ipld/go-car" + carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/storage" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" @@ -471,11 +473,11 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by rangeCount := len(byteRanges) api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "entity", "entityRanges": strconv.Itoa(rangeCount)}).Inc() - // IPIP-402 - carParams := "?format=car&dag-scope=entity" + // TODO: remove &bytes= &depth= and &car-scope from all CAR request after transition is done: + // https://github.com/ipfs/bifrost-gateway/issues/80 + carParams := "?format=car&dag-scope=entity&car-scope=file&depth=1" - // If request was HTTP Range Request fetch CAR with entity-bytes=from:to to - // get minimal set of blocks + // fetch CAR with &bytes= to get minimal set of blocks for the request // Note: majority of requests have 0 or max 1 ranges. if there are more ranges than one, // that is a niche edge cache we don't prefetch as CAR and use fallback blockstore instead. if rangeCount > 0 { @@ -497,7 +499,17 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by } else { bytesBuilder.WriteString("*") } + + // TODO: &bytes= is present only for transition reasons, remove below block after https://github.com/ipfs/bifrost-gateway/issues/80 is done + if strings.HasSuffix(carParams, "&depth=1") { + parts := strings.Split(bytesBuilder.String(), "=") + value := parts[1] + bytesBuilder.WriteString("&bytes=") + bytesBuilder.WriteString(value) + } + carParams += bytesBuilder.String() + } blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+carParams) @@ -516,7 +528,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "all", "entityRanges": "0"}).Inc() - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=all") + blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=all&car-scope=all&depth=all") if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -534,7 +546,7 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "block", "entityRanges": "0"}).Inc() // TODO: if path is `/ipfs/cid`, we should use ?format=raw - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=block") + blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=block&car-scope=block&depth=0") if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -556,7 +568,7 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) ( api.metrics.bytesRangeStartMetric.Observe(0) api.metrics.bytesRangeSizeMetric.Observe(1024) - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=entity&entity-bytes=0:1023") + blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=entity&entity-bytes=0:1023&car-scope=file&depth=1&bytes=0:1023") if err != nil { return gateway.ContentPathMetadata{}, nil, err @@ -574,7 +586,7 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) ( func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "block", "entityRanges": "0"}).Inc() - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=block") + blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=block&car-scope=block&depth=0") if err != nil { return gateway.ContentPathMetadata{}, err } @@ -583,13 +595,73 @@ func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.Immutable } func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams) (gateway.ContentPathMetadata, io.ReadCloser, error) { - api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "all", "entityRanges": "0"}).Inc() - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=all") + api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": string(params.Scope), "entityRanges": "0"}).Inc() + rootCid, err := getRootCid(path) if err != nil { return gateway.ContentPathMetadata{}, nil, err } - defer closeFn() - return blkgw.GetCAR(ctx, path, params) + + r, w := io.Pipe() + go func() { + cw, err := storage.NewWritable(w, []cid.Cid{cid.MustParse("bafkqaaa")}, carv2.WriteAsCarV1(true)) + if err != nil { + // io.PipeWriter.CloseWithError always returns nil. + _ = w.CloseWithError(err) + return + } + numBlocksSent := 0 + err = api.fetcher.Fetch(ctx, path.String()+"?format=car&dag-scope=all&car-scope=all&depth=all", func(resource string, reader io.Reader) error { + numBlocksThisCall := 0 + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + teeBlock := func(ctx context.Context, c cid.Cid) (blocks.Block, error) { + blk, err := gb(ctx, c) + if err != nil { + return nil, err + } + if numBlocksThisCall >= numBlocksSent { + err = cw.Put(ctx, blk.Cid().KeyString(), blk.RawData()) + if err != nil { + return nil, fmt.Errorf("error writing car block: %w", err) + } + numBlocksSent++ + } + numBlocksThisCall++ + return blk, nil + } + r, l := getIPFSPathResolverAndLsysFromBlockReader(ctx, teeBlock) + err = walkGatewaySimpleSelector(ctx, ipfspath.FromString(path.String()), params, l, r) + if err != nil { + return err + } + return nil + }) + // io.PipeWriter.CloseWithError always returns nil. + _ = w.CloseWithError(err) + }() + + return gateway.ContentPathMetadata{ + PathSegmentRoots: []cid.Cid{rootCid}, + LastSegment: nil, + ContentType: "", + }, r, nil +} + +func getRootCid(imPath gateway.ImmutablePath) (cid.Cid, error) { + imPathStr := imPath.String() + if !strings.HasPrefix(imPathStr, "/ipfs/") { + return cid.Undef, fmt.Errorf("path does not have /ipfs/ prefix") + } + + firstSegment, _, _ := strings.Cut(imPathStr[6:], "/") + rootCid, err := cid.Decode(firstSegment) + if err != nil { + return cid.Undef, err + } + + return rootCid, nil } func (api *GraphGateway) IsCached(ctx context.Context, path ifacepath.Path) bool { From 7ec385aa623d4da865260c590a62ddbee5bb721d Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 12 Jun 2023 12:24:43 -0400 Subject: [PATCH 03/46] feat: switch ResolvePath, Head and GetBlock to be backpressured --- lib/graph_gateway.go | 127 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 111 insertions(+), 16 deletions(-) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index d010c99..bafe012 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + gopath "path" "runtime" "runtime/debug" "strconv" @@ -31,6 +32,8 @@ import ( "github.com/ipld/go-car" carv2 "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/storage" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" @@ -546,18 +549,49 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "block", "entityRanges": "0"}).Inc() // TODO: if path is `/ipfs/cid`, we should use ?format=raw - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=block&car-scope=block&depth=0") - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - md, f, err := blkgw.GetBlock(ctx, path) + rootCid, err := getRootCid(path) if err != nil { return gateway.ContentPathMetadata{}, nil, err } - f, err = wrapNodeWithClose(f, closeFn) + p := ipfspath.FromString(path.String()) + + var md gateway.ContentPathMetadata + var f files.File + // TODO: fallback to dynamic fetches in case we haven't requested enough data + err = api.fetcher.Fetch(ctx, path.String()+"?format=car&dag-scope=block", func(resource string, reader io.Reader) error { + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + r, lsys := getIPFSPathResolverAndLsysFromBlockReader(ctx, gb) + + // First resolve the path since we always need to. + lastCid, remainder, err := r.ResolveToLastNode(ctx, p) + if err != nil { + return err + } + + md = gateway.ContentPathMetadata{ + PathSegmentRoots: []cid.Cid{rootCid}, + LastSegment: ifacepath.NewResolvedPath(p, lastCid, rootCid, gopath.Join(remainder...)), + } + + lctx := ipld.LinkContext{Ctx: ctx} + pathTerminalCidLink := cidlink.Link{Cid: lastCid} + + data, err := lsys.LoadRaw(lctx, pathTerminalCidLink) + if err != nil { + return err + } + + f = files.NewBytesFile(data) + return nil + }) + if err != nil { return gateway.ContentPathMetadata{}, nil, err } + return md, f, nil } @@ -568,30 +602,91 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) ( api.metrics.bytesRangeStartMetric.Observe(0) api.metrics.bytesRangeSizeMetric.Observe(1024) - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=entity&entity-bytes=0:1023&car-scope=file&depth=1&bytes=0:1023") - - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - md, f, err := blkgw.Head(ctx, path) + rootCid, err := getRootCid(path) if err != nil { return gateway.ContentPathMetadata{}, nil, err } - f, err = wrapNodeWithClose(f, closeFn) + p := ipfspath.FromString(path.String()) + + var md gateway.ContentPathMetadata + var f files.File + // TODO: fallback to dynamic fetches in case we haven't requested enough data + err = api.fetcher.Fetch(ctx, path.String()+"?format=car&dag-scope=entity&entity-bytes=0:3071", func(resource string, reader io.Reader) error { + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + r, lsys := getIPFSPathResolverAndLsysFromBlockReader(ctx, gb) + + // First resolve the path since we always need to. + lastCid, remainder, err := r.ResolveToLastNode(ctx, p) + if err != nil { + return err + } + + md = gateway.ContentPathMetadata{ + PathSegmentRoots: []cid.Cid{rootCid}, + LastSegment: ifacepath.NewResolvedPath(p, lastCid, rootCid, gopath.Join(remainder...)), + } + + // It's not UnixFS if there is a remainder + if len(remainder) > 0 { + lctx := ipld.LinkContext{Ctx: ctx} + pathTerminalCidLink := cidlink.Link{Cid: lastCid} + + data, err := lsys.LoadRaw(lctx, pathTerminalCidLink) + if err != nil { + return err + } + + f = files.NewBytesFile(data) + return nil + } + + return nil + }) + if err != nil { return gateway.ContentPathMetadata{}, nil, err } + return md, f, nil } func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "block", "entityRanges": "0"}).Inc() - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=block&car-scope=block&depth=0") + rootCid, err := getRootCid(path) if err != nil { return gateway.ContentPathMetadata{}, err } - defer closeFn() - return blkgw.ResolvePath(ctx, path) + + var md gateway.ContentPathMetadata + err = api.fetcher.Fetch(ctx, path.String()+"?format=car&dag-scope=block", func(resource string, reader io.Reader) error { + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + r, _ := getIPFSPathResolverAndLsysFromBlockReader(ctx, gb) + + // First resolve the path since we always need to. + lastCid, remainder, err := r.ResolveToLastNode(ctx, ipfspath.FromString(path.String())) + if err != nil { + return err + } + + md = gateway.ContentPathMetadata{ + PathSegmentRoots: []cid.Cid{rootCid}, + LastSegment: ifacepath.NewResolvedPath(ipfspath.FromString(path.String()), lastCid, rootCid, gopath.Join(remainder...)), + } + + return nil + }) + + if err != nil { + return gateway.ContentPathMetadata{}, err + } + + return md, nil } func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams) (gateway.ContentPathMetadata, io.ReadCloser, error) { From 468796ad0797b1861251f45fd57a69ec5169e46a Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 12 Jun 2023 16:42:21 -0400 Subject: [PATCH 04/46] feat: switch Get to be backpressured (not fully implemented) --- go.mod | 4 +- lib/block_fetchers.go | 194 +++++++++ lib/gateway_traversal.go | 38 +- lib/graph_gateway.go | 907 +++++++++++++++++++++++++++------------ 4 files changed, 843 insertions(+), 300 deletions(-) create mode 100644 lib/block_fetchers.go diff --git a/go.mod b/go.mod index f7c9f89..a090e11 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/ipfs/boxo v0.11.0 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 + github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ipld-format v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-unixfsnode v1.7.1 @@ -77,7 +78,6 @@ require ( github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-blockservice v0.5.0 // indirect - github.com/ipfs/go-datastore v0.6.0 // indirect github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect @@ -186,3 +186,5 @@ require ( lukechampine.com/blake3 v1.1.7 // indirect nhooyr.io/websocket v1.8.7 // indirect ) + +replace github.com/ipfs/boxo => ../boxo diff --git a/lib/block_fetchers.go b/lib/block_fetchers.go new file mode 100644 index 0000000..72ec622 --- /dev/null +++ b/lib/block_fetchers.go @@ -0,0 +1,194 @@ +package lib + +import ( + "context" + "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/exchange" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + format "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + "go.uber.org/multierr" +) + +type inboundBlockExchange struct { + ps BlockPubSub +} + +func newInboundBlockExchange() *inboundBlockExchange { + return &inboundBlockExchange{ + ps: NewBlockPubSub(), + } +} + +func (i *inboundBlockExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { + blk, more := <-i.ps.Subscribe(ctx, c.Hash()) + if err := ctx.Err(); err != nil { + return nil, err + } + if !more { + return nil, format.ErrNotFound{Cid: c} + } + return blk, nil +} + +func (i *inboundBlockExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { + mhMap := make(map[string]struct{}) + for _, c := range cids { + mhMap[string(c.Hash())] = struct{}{} + } + mhs := make([]multihash.Multihash, 0, len(mhMap)) + for k := range mhMap { + mhs = append(mhs, multihash.Multihash(k)) + } + return i.ps.Subscribe(ctx, mhs...), nil +} + +func (i *inboundBlockExchange) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { + // TODO: handle context cancellation and/or blockage here + i.ps.Publish(blocks...) + return nil +} + +func (i *inboundBlockExchange) Close() error { + i.ps.Shutdown() + return nil +} + +var _ exchange.Interface = (*inboundBlockExchange)(nil) + +type handoffExchange struct { + startingExchange, followupExchange exchange.Interface + bstore blockstore.Blockstore + handoffCh <-chan struct{} + metrics *GraphGatewayMetrics +} + +func (f *handoffExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { + blkCh, err := f.startingExchange.GetBlocks(ctx, []cid.Cid{c}) + if err != nil { + return nil, err + } + blk, ok := <-blkCh + if ok { + return blk, nil + } + + select { + case <-f.handoffCh: + graphLog.Debugw("switching to backup block fetcher", "cid", c) + f.metrics.blockRecoveryAttemptMetric.Inc() + return f.followupExchange.GetBlock(ctx, c) + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (f *handoffExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { + blkCh, err := f.startingExchange.GetBlocks(ctx, cids) + if err != nil { + return nil, err + } + + retCh := make(chan blocks.Block) + + go func() { + cs := cid.NewSet() + for cs.Len() < len(cids) { + blk, ok := <-blkCh + if !ok { + break + } + select { + case retCh <- blk: + cs.Add(blk.Cid()) + case <-ctx.Done(): + } + } + + for cs.Len() < len(cids) { + select { + case <-ctx.Done(): + return + case <-f.handoffCh: + var newCidArr []cid.Cid + for _, c := range cids { + if !cs.Has(c) { + blk, _ := f.bstore.Get(ctx, c) + if blk != nil { + select { + case retCh <- blk: + cs.Add(blk.Cid()) + case <-ctx.Done(): + return + } + } else { + newCidArr = append(newCidArr, c) + } + } + } + + if len(newCidArr) == 0 { + return + } + + graphLog.Debugw("needed to use use a backup fetcher for cids", "cids", newCidArr) + f.metrics.blockRecoveryAttemptMetric.Add(float64(len(newCidArr))) + fch, err := f.followupExchange.GetBlocks(ctx, newCidArr) + if err != nil { + graphLog.Errorw("error getting blocks from followupExchange", "error", err) + return + } + for cs.Len() < len(cids) { + blk, ok := <-fch + if !ok { + return + } + select { + case retCh <- blk: + cs.Add(blk.Cid()) + case <-ctx.Done(): + return + } + } + } + } + }() + return retCh, nil +} + +func (f *handoffExchange) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { + err1 := f.startingExchange.NotifyNewBlocks(ctx, blocks...) + err2 := f.followupExchange.NotifyNewBlocks(ctx, blocks...) + return multierr.Combine(err1, err2) +} + +func (f *handoffExchange) Close() error { + err1 := f.startingExchange.Close() + err2 := f.followupExchange.Close() + return multierr.Combine(err1, err2) +} + +var _ exchange.Interface = (*handoffExchange)(nil) + +type blockFetcherExchWrapper struct { + f exchange.Fetcher +} + +func (b *blockFetcherExchWrapper) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { + return b.f.GetBlock(ctx, c) +} + +func (b *blockFetcherExchWrapper) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { + return b.f.GetBlocks(ctx, cids) +} + +func (b *blockFetcherExchWrapper) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { + return nil +} + +func (b *blockFetcherExchWrapper) Close() error { + return nil +} + +var _ exchange.Interface = (*blockFetcherExchWrapper)(nil) diff --git a/lib/gateway_traversal.go b/lib/gateway_traversal.go index 27ac8ab..d031aa4 100644 --- a/lib/gateway_traversal.go +++ b/lib/gateway_traversal.go @@ -1,6 +1,7 @@ package lib import ( + "bytes" "context" "errors" "fmt" @@ -8,12 +9,13 @@ import ( "sync" "time" + "github.com/ipld/go-ipld-prime/linking" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/exchange" bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/gateway" - ipfspath "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/path/resolver" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -70,7 +72,6 @@ func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *Grap } cbCtx, cncl := context.WithCancel(ctx) - defer cncl() type blockRead struct { block blocks.Block @@ -79,11 +80,15 @@ func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *Grap blkCh := make(chan blockRead, 1) go func() { + defer cncl() defer close(blkCh) for { blk, rdErr := cr.Next() select { case blkCh <- blockRead{blk, rdErr}: + if rdErr != nil { + cncl() + } case <-cbCtx.Done(): return } @@ -116,7 +121,7 @@ func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *Grap return nil, gateway.ErrGatewayTimeout } if !ok || blkRead.err != nil { - if errors.Is(blkRead.err, io.EOF) { + if !ok || errors.Is(blkRead.err, io.EOF) { return nil, io.EOF } return nil, blkRead.err @@ -136,22 +141,32 @@ func getIPFSPathResolverAndLsysFromBlockReader(ctx context.Context, fn getBlock) fetcher := bsfetcher.NewFetcherConfig( blockservice.New(blockstore.NewBlockstore(&datastore.NullDatastore{}), &blockFetcherExchWrapper{f: &gbf{fn}})) fetcher.NodeReifier = unixfsnode.Reify + fetcher.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) res := resolver.NewBasicResolver(fetcher) lsys := cidlink.DefaultLinkSystem() + lsys.StorageReadOpener = func(linkContext linking.LinkContext, link datamodel.Link) (io.Reader, error) { + c := link.(cidlink.Link).Cid + blk, err := fn(linkContext.Ctx, c) + if err != nil { + return nil, err + } + return bytes.NewReader(blk.RawData()), nil + } + lsys.TrustedStorage = true unixfsnode.AddUnixFSReificationToLinkSystem(&lsys) return res, &lsys } // walkGatewaySimpleSelector walks the subgraph described by the path and terminal element parameters -func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params gateway.CarParams, lsys *ipld.LinkSystem, pathResolver resolver.Resolver) error { - // First resolve the path since we always need to. - lastCid, remainder, err := pathResolver.ResolveToLastNode(ctx, p) - if err != nil { - return err - } - +func walkGatewaySimpleSelector(ctx context.Context, lastCid cid.Cid, params gateway.CarParams, lsys *ipld.LinkSystem, pathResolver resolver.Resolver) error { lctx := ipld.LinkContext{Ctx: ctx} pathTerminalCidLink := cidlink.Link{Cid: lastCid} + var err error // If the scope is the block, now we only need to retrieve the root block of the last element of the path. if params.Scope == gateway.DagScopeBlock { @@ -211,9 +226,6 @@ func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params gate if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { // If it's not valid dag-pb then we're done return nil - } else if len(remainder) > 0 { - // If we're trying to path into dag-pb node that's invalid and we're done - return nil } else if !pbn.FieldData().Exists() { // If it's not valid UnixFS then we're done return nil diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index bafe012..643be65 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -1,19 +1,21 @@ package lib import ( + "bytes" "context" "errors" "fmt" "io" "net/http" gopath "path" - "runtime" "runtime/debug" "strconv" "strings" "sync" "time" + "github.com/ipfs/boxo/path/resolver" + "github.com/filecoin-saturn/caboose" "github.com/ipfs/boxo/blockservice" blockstore "github.com/ipfs/boxo/blockstore" @@ -22,6 +24,8 @@ import ( exchange "github.com/ipfs/boxo/exchange" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/gateway" + "github.com/ipfs/boxo/ipld/merkledag" + "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/boxo/namesys" "github.com/ipfs/boxo/namesys/resolve" ipfspath "github.com/ipfs/boxo/path" @@ -29,18 +33,22 @@ import ( "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" golog "github.com/ipfs/go-log/v2" + "github.com/ipfs/go-unixfsnode" + ufsData "github.com/ipfs/go-unixfsnode/data" "github.com/ipld/go-car" carv2 "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/storage" + dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/schema" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multicodec" - "github.com/multiformats/go-multihash" "github.com/prometheus/client_golang/prometheus" - "go.uber.org/multierr" ) var graphLog = golog.Logger("backend/graph") @@ -478,7 +486,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by // TODO: remove &bytes= &depth= and &car-scope from all CAR request after transition is done: // https://github.com/ipfs/bifrost-gateway/issues/80 - carParams := "?format=car&dag-scope=entity&car-scope=file&depth=1" + carParams := "?format=car&dag-scope=entity" // fetch CAR with &bytes= to get minimal set of blocks for the request // Note: majority of requests have 0 or max 1 ranges. if there are more ranges than one, @@ -503,35 +511,353 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by bytesBuilder.WriteString("*") } - // TODO: &bytes= is present only for transition reasons, remove below block after https://github.com/ipfs/bifrost-gateway/issues/80 is done - if strings.HasSuffix(carParams, "&depth=1") { - parts := strings.Split(bytesBuilder.String(), "=") - value := parts[1] - bytesBuilder.WriteString("&bytes=") - bytesBuilder.WriteString(value) - } - carParams += bytesBuilder.String() - } - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+carParams) + rootCid, err := getRootCid(path) if err != nil { return gateway.ContentPathMetadata{}, nil, err } - md, gr, err := blkgw.Get(ctx, path, byteRanges...) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err + p := ipfspath.FromString(path.String()) + + type terminalPathType struct { + resp *gateway.GetResponse + err error + lastCid cid.Cid + } + + terminalPathElementCh := make(chan terminalPathType, 1) + errNotUnixFS := fmt.Errorf("data was not unixfs") + + var terminalFile *multiReadCloser + var terminalDir chan unixfs.LinkResult + lastDirLinkNum := 0 + go func() { + cctx, cancel := context.WithCancel(ctx) + defer cancel() + err = api.fetcher.Fetch(cctx, path.String()+"?format=car&dag-scope=all", func(resource string, reader io.Reader) error { + gb, err := carToLinearBlockGetter(cctx, reader, api.metrics) + if err != nil { + return err + } + r, lsys := getIPFSPathResolverAndLsysFromBlockReader(cctx, gb) + // First resolve the path since we always need to. + lastCid, remainder, err := r.ResolveToLastNode(cctx, p) + if err != nil { + return err + } + + lctx := ipld.LinkContext{Ctx: cctx} + pathTerminalCidLink := cidlink.Link{Cid: lastCid} + + if len(remainder) > 0 { + terminalPathElementCh <- terminalPathType{err: errNotUnixFS} + return nil + } + + // From now on, dag-scope=entity! + // Since we need more of the graph load it to figure out what we have + // This includes determining if the terminal node is UnixFS or not + pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + + np, err := pc(pathTerminalCidLink, lctx) + if err != nil { + return err + } + + lastCidNode, nodeBytes, err := lsys.LoadPlusRaw(lctx, pathTerminalCidLink, np) + if err != nil { + return err + } + + if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { + // If it's not valid dag-pb then we're done + return nil + } else if len(remainder) > 0 { + // If we're trying to path into dag-pb node that's invalid and we're done + return nil + } else if !pbn.FieldData().Exists() { + // If it's not valid UnixFS then we're done + return nil + } else if unixfsFieldData, decodeErr := ufsData.DecodeUnixFSData(pbn.Data.Must().Bytes()); decodeErr != nil { + // If it's not valid dag-pb and UnixFS then we're done + return nil + } else { + switch fieldNum := unixfsFieldData.FieldDataType().Int(); fieldNum { + case ufsData.Data_Symlink: + fd := unixfsFieldData.FieldData() + if fd.Exists() { + lnkTarget := string(fd.Must().Bytes()) + f := files.NewLinkFile(lnkTarget, nil) + s, ok := f.(*files.Symlink) + if !ok { + terminalPathElementCh <- terminalPathType{err: fmt.Errorf("should be unreachable: symlink does not have a symlink type")} + return nil + } + terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromSymlink(s, int64(len(lnkTarget))), lastCid: lastCid} + return nil + } + terminalPathElementCh <- terminalPathType{err: fmt.Errorf("UnixFS Symlink does not contain target")} + return nil + case ufsData.Data_Metadata: + terminalPathElementCh <- terminalPathType{err: fmt.Errorf("UnixFS Metadata unsupported")} + return nil + case ufsData.Data_HAMTShard, ufsData.Data_Directory: + blk, err := blocks.NewBlockWithCid(nodeBytes, lastCid) + if err != nil { + terminalPathElementCh <- terminalPathType{err: fmt.Errorf("could not create block: %w", err)} + return nil + } + dirRootNd, err := merkledag.ProtoNodeConverter(blk, lastCidNode) + if err != nil { + terminalPathElementCh <- terminalPathType{err: fmt.Errorf("could not create dag-pb universal block from UnixFS directory root: %w", err)} + return nil + } + pn, ok := dirRootNd.(*merkledag.ProtoNode) + if !ok { + terminalPathElementCh <- terminalPathType{err: fmt.Errorf("could not create dag-pb node from UnixFS directory root: %w", err)} + return nil + } + + sz, err := pn.Size() + if err != nil { + terminalPathElementCh <- terminalPathType{err: fmt.Errorf("could not get cumulative size from dag-pb node: %w", err)} + return nil + } + + lnks := make(chan unixfs.LinkResult) + defer close(lnks) + if terminalDir == nil { + terminalDir = lnks + } + terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromDirectoryListing(sz, lnks, nil)} + + dirLinkNum := 0 + switch fieldNum { + case ufsData.Data_Directory: + iter := pbn.Links.Iterator() + for !iter.Done() { + _, v := iter.Next() + c := v.Hash.Link().(cidlink.Link).Cid + var name string + var size int64 + if v.Name.Exists() { + name = v.Name.Must().String() + } + if v.Tsize.Exists() { + size = v.Tsize.Must().Int() + } + lnk := unixfs.LinkResult{Link: &format.Link{ + Name: name, + Size: uint64(size), + Cid: c, + }} + + dirLinkNum++ + if dirLinkNum-1 <= lastDirLinkNum { + continue + } + + select { + case lnks <- lnk: + lastDirLinkNum++ + case <-cctx.Done(): + // TODO: what here, send an error with another select? + return nil + } + } + case ufsData.Data_HAMTShard: + // Note: we are making up the entries + dirNd, err := unixfsnode.Reify(lctx, pbn, lsys) + if err != nil { + select { + case lnks <- unixfs.LinkResult{Err: fmt.Errorf("could not reify sharded directory: %w", err)}: + case <-cctx.Done(): + // TODO: what here? + } + return nil + } + + mi := dirNd.MapIterator() + for !mi.Done() { + k, v, err := mi.Next() + if err != nil { + return err + } + keyStr, err := k.AsString() + if err != nil { + select { + case lnks <- unixfs.LinkResult{Err: fmt.Errorf("could not interpret directory key as string: %w", err)}: + case <-cctx.Done(): + // TODO: what here? + } + return nil + } + valLink, err := v.AsLink() + if err != nil { + select { + case lnks <- unixfs.LinkResult{Err: fmt.Errorf("could not interpret directory value as link: %w", err)}: + case <-cctx.Done(): + // TODO: what here? + } + return nil + } + valCid := valLink.(cidlink.Link).Cid + lnk := unixfs.LinkResult{Link: &format.Link{ + Name: keyStr, + Size: uint64(0), + Cid: valCid, + }} + + dirLinkNum++ + if dirLinkNum-1 <= lastDirLinkNum { + continue + } + + select { + case lnks <- lnk: + lastDirLinkNum++ + case <-cctx.Done(): + // TODO: what here? + } + } + } + return nil + case ufsData.Data_Raw, ufsData.Data_File: + nd, err := unixfsnode.Reify(lctx, lastCidNode, lsys) + if err != nil { + return err + } + + fnd, ok := nd.(datamodel.LargeBytesNode) + if !ok { + return fmt.Errorf("could not process file since it did not present as large bytes") + } + f, err := fnd.AsLargeBytes() + if err != nil { + return err + } + + fileSize, err := f.Seek(0, io.SeekEnd) + if err != nil { + terminalPathElementCh <- terminalPathType{err: fmt.Errorf("unable to get UnixFS file size: %w", err)} + return nil + } + _, err = f.Seek(0, io.SeekStart) + if err != nil { + terminalPathElementCh <- terminalPathType{err: fmt.Errorf("unable to get reset UnixFS file reader: %w", err)} + return nil + } + + if terminalFile == nil { + mrc := &multiReadCloser{ + closeFn: nil, + mr: f, + closed: make(chan struct{}), + newReader: make(chan io.Reader), + retErr: nil, + isClosed: false, + } + terminalFile = mrc + terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromReader(files.NewReaderFile(mrc), fileSize), lastCid: lastCid} + } else { + select { + case terminalFile.newReader <- f: + case <-cctx.Done(): + // TODO: what here? + return nil + } + } + + _, err = io.Copy(io.Discard, terminalFile) + if err != nil { + return err + } + return nil + default: + terminalPathElementCh <- terminalPathType{err: fmt.Errorf("unknown UnixFS field type")} + return nil + } + } + }) + }() + + select { + case t := <-terminalPathElementCh: + if t.err != nil { + return gateway.ContentPathMetadata{}, nil, t.err + } + + md := gateway.ContentPathMetadata{ + PathSegmentRoots: []cid.Cid{rootCid}, + LastSegment: ifacepath.NewResolvedPath(p, t.lastCid, rootCid, ""), + } + return md, t.resp, nil + case <-ctx.Done(): + return gateway.ContentPathMetadata{}, nil, ctx.Err() } +} - //TODO: interfaces here aren't good enough so we're getting around the problem this way - runtime.SetFinalizer(gr, func(_ *gateway.GetResponse) { closeFn() }) - return md, gr, nil +type multiReadCloser struct { + closeFn func() error + mr io.Reader + closed chan struct{} + newReader chan io.Reader + retErr error + isClosed bool } +func (r *multiReadCloser) Read(p []byte) (n int, err error) { + if r.retErr == nil { + n, err = r.mr.Read(p) + if err == nil || err == io.EOF { + return n, err + } + + if n > 0 { + r.retErr = err + return n, nil + } + } else { + err = r.retErr + } + + select { + case <-r.closed: + return n, err + case newReader, ok := <-r.newReader: + if ok { + r.mr = io.MultiReader(r.mr, newReader) + return r.Read(p) + } + return n, err + } +} + +func (r *multiReadCloser) Close() error { + if r.isClosed { + return nil + } + + close(r.newReader) + close(r.closed) + + if r.closeFn == nil { + return nil + } + return r.closeFn() +} + +var _ io.ReadCloser = (*multiReadCloser)(nil) + func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "all", "entityRanges": "0"}).Inc() - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=all&car-scope=all&depth=all") + blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=all") if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -557,37 +883,43 @@ func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePat var md gateway.ContentPathMetadata var f files.File + var ipldError error // TODO: fallback to dynamic fetches in case we haven't requested enough data err = api.fetcher.Fetch(ctx, path.String()+"?format=car&dag-scope=block", func(resource string, reader io.Reader) error { - gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) - if err != nil { - return err - } - r, lsys := getIPFSPathResolverAndLsysFromBlockReader(ctx, gb) + return checkRetryableError(&ipldError, func() error { + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + r, lsys := getIPFSPathResolverAndLsysFromBlockReader(ctx, gb) - // First resolve the path since we always need to. - lastCid, remainder, err := r.ResolveToLastNode(ctx, p) - if err != nil { - return err - } + // First resolve the path since we always need to. + lastCid, remainder, err := r.ResolveToLastNode(ctx, p) + if err != nil { + return err + } - md = gateway.ContentPathMetadata{ - PathSegmentRoots: []cid.Cid{rootCid}, - LastSegment: ifacepath.NewResolvedPath(p, lastCid, rootCid, gopath.Join(remainder...)), - } + md = gateway.ContentPathMetadata{ + PathSegmentRoots: []cid.Cid{rootCid}, + LastSegment: ifacepath.NewResolvedPath(p, lastCid, rootCid, gopath.Join(remainder...)), + } - lctx := ipld.LinkContext{Ctx: ctx} - pathTerminalCidLink := cidlink.Link{Cid: lastCid} + lctx := ipld.LinkContext{Ctx: ctx} + pathTerminalCidLink := cidlink.Link{Cid: lastCid} - data, err := lsys.LoadRaw(lctx, pathTerminalCidLink) - if err != nil { - return err - } + data, err := lsys.LoadRaw(lctx, pathTerminalCidLink) + if err != nil { + return err + } - f = files.NewBytesFile(data) - return nil + f = files.NewBytesFile(data) + return nil + }) }) + if ipldError != nil { + err = ipldError + } if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -595,12 +927,12 @@ func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePat return md, f, nil } -func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { +func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, *gateway.HeadResponse, error) { api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "entity", "entityRanges": "1"}).Inc() // TODO: we probably want to move this either to boxo, or at least to loadRequestIntoSharedBlockstoreAndBlocksGateway api.metrics.bytesRangeStartMetric.Observe(0) - api.metrics.bytesRangeSizeMetric.Observe(1024) + api.metrics.bytesRangeSizeMetric.Observe(3071) rootCid, err := getRootCid(path) if err != nil { @@ -609,48 +941,157 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) ( p := ipfspath.FromString(path.String()) var md gateway.ContentPathMetadata - var f files.File + var n *gateway.HeadResponse + var ipldError error // TODO: fallback to dynamic fetches in case we haven't requested enough data err = api.fetcher.Fetch(ctx, path.String()+"?format=car&dag-scope=entity&entity-bytes=0:3071", func(resource string, reader io.Reader) error { - gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) - if err != nil { - return err - } - r, lsys := getIPFSPathResolverAndLsysFromBlockReader(ctx, gb) + return checkRetryableError(&ipldError, func() error { + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + r, lsys := getIPFSPathResolverAndLsysFromBlockReader(ctx, gb) - // First resolve the path since we always need to. - lastCid, remainder, err := r.ResolveToLastNode(ctx, p) - if err != nil { - return err - } + // First resolve the path since we always need to. + lastCid, remainder, err := r.ResolveToLastNode(ctx, p) + if err != nil { + return err + } - md = gateway.ContentPathMetadata{ - PathSegmentRoots: []cid.Cid{rootCid}, - LastSegment: ifacepath.NewResolvedPath(p, lastCid, rootCid, gopath.Join(remainder...)), - } + md = gateway.ContentPathMetadata{ + PathSegmentRoots: []cid.Cid{rootCid}, + LastSegment: ifacepath.NewResolvedPath(p, lastCid, rootCid, gopath.Join(remainder...)), + } - // It's not UnixFS if there is a remainder - if len(remainder) > 0 { lctx := ipld.LinkContext{Ctx: ctx} pathTerminalCidLink := cidlink.Link{Cid: lastCid} - data, err := lsys.LoadRaw(lctx, pathTerminalCidLink) + // Load the block at the root of the terminal path element + dataBytes, err := lsys.LoadRaw(lctx, pathTerminalCidLink) if err != nil { return err } - f = files.NewBytesFile(data) - return nil - } + // It's not UnixFS if there is a remainder or it's not dag-pb + if len(remainder) > 0 || lastCid.Type() != uint64(multicodec.DagPb) { + n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + return nil + } - return nil + // Let's figure out if the terminal element is valid UnixFS and if so what kind + pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + + np, err := pc(pathTerminalCidLink, lctx) + if err != nil { + return err + } + + nodeDecoder, err := lsys.DecoderChooser(pathTerminalCidLink) + if err != nil { + return err + } + + nb := np.NewBuilder() + err = nodeDecoder(nb, bytes.NewReader(dataBytes)) + if err != nil { + return err + } + lastCidNode := nb.Build() + + if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { + // This shouldn't be possible since we already checked for dag-pb usage + return fmt.Errorf("node was not go-codec-dagpb node") + } else if !pbn.FieldData().Exists() { + // If it's not valid UnixFS then just return the block bytes + n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + return nil + } else if unixfsFieldData, decodeErr := ufsData.DecodeUnixFSData(pbn.Data.Must().Bytes()); decodeErr != nil { + // If it's not valid UnixFS then just return the block bytes + n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + return nil + } else { + switch fieldNum := unixfsFieldData.FieldDataType().Int(); fieldNum { + case ufsData.Data_Directory, ufsData.Data_HAMTShard: + blk, err := blocks.NewBlockWithCid(dataBytes, lastCid) + if err != nil { + return fmt.Errorf("could not create block: %w", err) + } + dirRootNd, err := merkledag.ProtoNodeConverter(blk, lastCidNode) + if err != nil { + return fmt.Errorf("could not create dag-pb universal block from UnixFS directory root: %w", err) + } + pn, ok := dirRootNd.(*merkledag.ProtoNode) + if !ok { + return fmt.Errorf("could not create dag-pb node from UnixFS directory root: %w", err) + } + + sz, err := pn.Size() + if err != nil { + return fmt.Errorf("could not get cumulative size from dag-pb node: %w", err) + } + + n = gateway.NewHeadResponseForDirectory(int64(sz)) + return nil + case ufsData.Data_Symlink: + fd := unixfsFieldData.FieldData() + if fd.Exists() { + n = gateway.NewHeadResponseForSymlink(int64(len(fd.Must().Bytes()))) + return nil + } + // If there is no target then it's invalid so just return the block + gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + return nil + case ufsData.Data_Metadata: + n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + return nil + case ufsData.Data_Raw, ufsData.Data_File: + ufsNode, err := unixfsnode.Reify(lctx, pbn, lsys) + if err != nil { + return err + } + fileNode, ok := ufsNode.(datamodel.LargeBytesNode) + if !ok { + return fmt.Errorf("data not a large bytes node despite being UnixFS bytes") + } + f, err := fileNode.AsLargeBytes() + if err != nil { + return err + } + + fileSize, err := f.Seek(0, io.SeekEnd) + if err != nil { + return fmt.Errorf("unable to get UnixFS file size: %w", err) + } + _, err = f.Seek(0, io.SeekStart) + if err != nil { + return fmt.Errorf("unable to get reset UnixFS file reader: %w", err) + } + + out, err := io.ReadAll(io.LimitReader(f, 3072)) + if errors.Is(err, io.EOF) { + n = gateway.NewHeadResponseForFile(files.NewBytesFile(out), fileSize) + return nil + } + return err + } + } + return nil + }) }) + if ipldError != nil { + err = ipldError + } if err != nil { return gateway.ContentPathMetadata{}, nil, err } - return md, f, nil + return md, n, nil } func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { @@ -661,27 +1102,33 @@ func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.Immutable } var md gateway.ContentPathMetadata + var ipldError error err = api.fetcher.Fetch(ctx, path.String()+"?format=car&dag-scope=block", func(resource string, reader io.Reader) error { - gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) - if err != nil { - return err - } - r, _ := getIPFSPathResolverAndLsysFromBlockReader(ctx, gb) + return checkRetryableError(&ipldError, func() error { + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + r, _ := getIPFSPathResolverAndLsysFromBlockReader(ctx, gb) - // First resolve the path since we always need to. - lastCid, remainder, err := r.ResolveToLastNode(ctx, ipfspath.FromString(path.String())) - if err != nil { - return err - } + // First resolve the path since we always need to. + lastCid, remainder, err := r.ResolveToLastNode(ctx, ipfspath.FromString(path.String())) + if err != nil { + return err + } - md = gateway.ContentPathMetadata{ - PathSegmentRoots: []cid.Cid{rootCid}, - LastSegment: ifacepath.NewResolvedPath(ipfspath.FromString(path.String()), lastCid, rootCid, gopath.Join(remainder...)), - } + md = gateway.ContentPathMetadata{ + PathSegmentRoots: []cid.Cid{rootCid}, + LastSegment: ifacepath.NewResolvedPath(ipfspath.FromString(path.String()), lastCid, rootCid, gopath.Join(remainder...)), + } - return nil + return nil + }) }) + if ipldError != nil { + err = ipldError + } if err != nil { return gateway.ContentPathMetadata{}, err } @@ -695,51 +1142,86 @@ func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath, if err != nil { return gateway.ContentPathMetadata{}, nil, err } + p := ipfspath.FromString(path.String()) r, w := io.Pipe() go func() { - cw, err := storage.NewWritable(w, []cid.Cid{cid.MustParse("bafkqaaa")}, carv2.WriteAsCarV1(true)) - if err != nil { - // io.PipeWriter.CloseWithError always returns nil. - _ = w.CloseWithError(err) - return - } numBlocksSent := 0 - err = api.fetcher.Fetch(ctx, path.String()+"?format=car&dag-scope=all&car-scope=all&depth=all", func(resource string, reader io.Reader) error { - numBlocksThisCall := 0 - gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) - if err != nil { - return err - } - teeBlock := func(ctx context.Context, c cid.Cid) (blocks.Block, error) { - blk, err := gb(ctx, c) + var cw storage.WritableCar + var blockBuffer []blocks.Block + var ipldError error + err = api.fetcher.Fetch(ctx, path.String()+"?format=car&dag-scope=all", func(resource string, reader io.Reader) error { + return checkRetryableError(&ipldError, func() error { + numBlocksThisCall := 0 + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + teeBlock := func(ctx context.Context, c cid.Cid) (blocks.Block, error) { + blk, err := gb(ctx, c) + if err != nil { + return nil, err + } + if numBlocksThisCall >= numBlocksSent { + if cw == nil { + blockBuffer = append(blockBuffer, blk) + } else { + err = cw.Put(ctx, blk.Cid().KeyString(), blk.RawData()) + if err != nil { + return nil, fmt.Errorf("error writing car block: %w", err) + } + } + numBlocksSent++ + } + numBlocksThisCall++ + return blk, nil + } + r, l := getIPFSPathResolverAndLsysFromBlockReader(ctx, teeBlock) + + // First resolve the path since we always need to. + lastCid, remainder, err := r.ResolveToLastNode(ctx, p) + if err != nil { + return err + } + if len(remainder) > 0 { + return nil + } + + cw, err = storage.NewWritable(w, []cid.Cid{lastCid}, carv2.WriteAsCarV1(true)) if err != nil { - return nil, err + // io.PipeWriter.CloseWithError always returns nil. + _ = w.CloseWithError(err) + return nil } - if numBlocksThisCall >= numBlocksSent { + for _, blk := range blockBuffer { err = cw.Put(ctx, blk.Cid().KeyString(), blk.RawData()) if err != nil { - return nil, fmt.Errorf("error writing car block: %w", err) + _ = w.CloseWithError(fmt.Errorf("error writing car block: %w", err)) + return nil } - numBlocksSent++ } - numBlocksThisCall++ - return blk, nil - } - r, l := getIPFSPathResolverAndLsysFromBlockReader(ctx, teeBlock) - err = walkGatewaySimpleSelector(ctx, ipfspath.FromString(path.String()), params, l, r) - if err != nil { - return err - } - return nil + blockBuffer = nil + + err = walkGatewaySimpleSelector(ctx, lastCid, params, l, r) + if err != nil { + return err + } + return nil + }) }) - // io.PipeWriter.CloseWithError always returns nil. - _ = w.CloseWithError(err) + + if ipldError != nil { + err = ipldError + } + if err != nil { + // io.PipeWriter.CloseWithError always returns nil. + _ = w.CloseWithError(err) + } }() return gateway.ContentPathMetadata{ PathSegmentRoots: []cid.Cid{rootCid}, - LastSegment: nil, + LastSegment: ifacepath.NewResolvedPath(p, rootCid, rootCid, ""), ContentType: "", }, r, nil } @@ -829,184 +1311,37 @@ func (api *GraphGateway) GetDNSLinkRecord(ctx context.Context, hostname string) var _ gateway.IPFSBackend = (*GraphGateway)(nil) -type inboundBlockExchange struct { - ps BlockPubSub -} - -func newInboundBlockExchange() *inboundBlockExchange { - return &inboundBlockExchange{ - ps: NewBlockPubSub(), +func checkRetryableError(e *error, fn func() error) error { + err := fn() + if format.IsNotFound(err) { + return err } -} - -func (i *inboundBlockExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - blk, more := <-i.ps.Subscribe(ctx, c.Hash()) - if err := ctx.Err(); err != nil { - return nil, err - } - if !more { - return nil, format.ErrNotFound{Cid: c} - } - return blk, nil -} + initialErr := err -func (i *inboundBlockExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { - mhMap := make(map[string]struct{}) - for _, c := range cids { - mhMap[string(c.Hash())] = struct{}{} - } - mhs := make([]multihash.Multihash, 0, len(mhMap)) - for k := range mhMap { - mhs = append(mhs, multihash.Multihash(k)) - } - return i.ps.Subscribe(ctx, mhs...), nil -} - -func (i *inboundBlockExchange) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { - // TODO: handle context cancellation and/or blockage here - i.ps.Publish(blocks...) - return nil -} - -func (i *inboundBlockExchange) Close() error { - i.ps.Shutdown() - return nil -} - -var _ exchange.Interface = (*inboundBlockExchange)(nil) - -type handoffExchange struct { - startingExchange, followupExchange exchange.Interface - bstore blockstore.Blockstore - handoffCh <-chan struct{} - metrics *GraphGatewayMetrics -} - -func (f *handoffExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - blkCh, err := f.startingExchange.GetBlocks(ctx, []cid.Cid{c}) - if err != nil { - return nil, err - } - blk, ok := <-blkCh - if ok { - return blk, nil - } - - select { - case <-f.handoffCh: - graphLog.Debugw("switching to backup block fetcher", "cid", c) - f.metrics.blockRecoveryAttemptMetric.Inc() - return f.followupExchange.GetBlock(ctx, c) - case <-ctx.Done(): - return nil, ctx.Err() - } -} - -func (f *handoffExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { - blkCh, err := f.startingExchange.GetBlocks(ctx, cids) - if err != nil { - return nil, err - } - - retCh := make(chan blocks.Block) - - go func() { - cs := cid.NewSet() - for cs.Len() < len(cids) { - blk, ok := <-blkCh - if !ok { - break - } - select { - case retCh <- blk: - cs.Add(blk.Cid()) - case <-ctx.Done(): - } + // Checks if err is of a type that does not implement the .Is interface and + // cannot be directly compared to. Therefore, errors.Is cannot be used. + for { + _, ok := err.(resolver.ErrNoLink) + if ok { + *e = err + return nil } - for cs.Len() < len(cids) { - select { - case <-ctx.Done(): - return - case <-f.handoffCh: - var newCidArr []cid.Cid - for _, c := range cids { - if !cs.Has(c) { - blk, _ := f.bstore.Get(ctx, c) - if blk != nil { - select { - case retCh <- blk: - cs.Add(blk.Cid()) - case <-ctx.Done(): - return - } - } else { - newCidArr = append(newCidArr, c) - } - } - } - - if len(newCidArr) == 0 { - return - } - - graphLog.Debugw("needed to use use a backup fetcher for cids", "cids", newCidArr) - f.metrics.blockRecoveryAttemptMetric.Add(float64(len(newCidArr))) - fch, err := f.followupExchange.GetBlocks(ctx, newCidArr) - if err != nil { - graphLog.Errorw("error getting blocks from followupExchange", "error", err) - return - } - for cs.Len() < len(cids) { - blk, ok := <-fch - if !ok { - return - } - select { - case retCh <- blk: - cs.Add(blk.Cid()) - case <-ctx.Done(): - return - } - } - } + _, ok = err.(datamodel.ErrWrongKind) + if ok { + *e = err + return nil } - }() - return retCh, nil -} - -func (f *handoffExchange) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { - err1 := f.startingExchange.NotifyNewBlocks(ctx, blocks...) - err2 := f.followupExchange.NotifyNewBlocks(ctx, blocks...) - return multierr.Combine(err1, err2) -} - -func (f *handoffExchange) Close() error { - err1 := f.startingExchange.Close() - err2 := f.followupExchange.Close() - return multierr.Combine(err1, err2) -} - -var _ exchange.Interface = (*handoffExchange)(nil) - -type blockFetcherExchWrapper struct { - f exchange.Fetcher -} - -func (b *blockFetcherExchWrapper) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - return b.f.GetBlock(ctx, c) -} -func (b *blockFetcherExchWrapper) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { - return b.f.GetBlocks(ctx, cids) -} - -func (b *blockFetcherExchWrapper) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { - return nil -} + _, ok = err.(datamodel.ErrNotExists) + if ok { + *e = err + return nil + } -func (b *blockFetcherExchWrapper) Close() error { - return nil + err = errors.Unwrap(err) + if err == nil { + return initialErr + } + } } - -var _ exchange.Interface = (*blockFetcherExchWrapper)(nil) From db4ecfed49694482075939eddc6609c932b7cf75 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 29 Jun 2023 11:13:31 -0400 Subject: [PATCH 05/46] switch fetch to do escaping --- blockstore_proxy.go | 8 +- go.mod | 2 +- go.sum | 2 - lib/graph_gateway.go | 276 +++++++++++-------------------------------- 4 files changed, 75 insertions(+), 213 deletions(-) diff --git a/blockstore_proxy.go b/blockstore_proxy.go index 2b44ef4..4f30499 100644 --- a/blockstore_proxy.go +++ b/blockstore_proxy.go @@ -50,7 +50,13 @@ func (ps *proxyBlockStore) Fetch(ctx context.Context, path string, cb lib.DataCa } if resp.StatusCode != http.StatusOK { - return fmt.Errorf("http error from car gateway: %s", resp.Status) + errData, err := io.ReadAll(resp.Body) + if err != nil { + err = fmt.Errorf("could not read error message: %w", err) + } else { + err = fmt.Errorf("%q", string(errData)) + } + return fmt.Errorf("http error from car gateway: %s: %w", resp.Status, err) } err = cb(path, resp.Body) diff --git a/go.mod b/go.mod index a090e11..51e821e 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/cskr/pubsub v1.0.2 github.com/filecoin-saturn/caboose v0.0.3 github.com/hashicorp/golang-lru/v2 v2.0.1 - github.com/ipfs/boxo v0.11.0 + github.com/ipfs/boxo v0.11.1-0.20230803093408-3b83506a2ea5 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index 643ffbf..fa1c7e3 100644 --- a/go.sum +++ b/go.sum @@ -272,8 +272,6 @@ github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7P github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.11.0 h1:urMxhZ3xoF4HssJVD3+0ssGT9pptEfHfbL8DYdoWFlg= -github.com/ipfs/boxo v0.11.0/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 643be65..c52acb0 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -7,8 +7,8 @@ import ( "fmt" "io" "net/http" + "net/url" gopath "path" - "runtime/debug" "strconv" "strings" "sync" @@ -17,7 +17,6 @@ import ( "github.com/ipfs/boxo/path/resolver" "github.com/filecoin-saturn/caboose" - "github.com/ipfs/boxo/blockservice" blockstore "github.com/ipfs/boxo/blockstore" nsopts "github.com/ipfs/boxo/coreiface/options/namesys" ifacepath "github.com/ipfs/boxo/coreiface/path" @@ -278,169 +277,6 @@ func (api *GraphGateway) getRootOfPath(path string) string { } } -/* -Implementation iteration plan: - -1. Fetch CAR into per-request memory blockstore and serve response -2. Fetch CAR into shared memory blockstore and serve response along with a blockservice that does block requests for missing data -3. Start doing the walk locally and then if a path segment is incomplete send a request for blocks and upon every received block try to continue -4. Start doing the walk locally and keep a list of "plausible" blocks, if after issuing a request we get a non-plausible block then report them and attempt to recover by redoing the last segment -5. Don't redo the last segment fully if it's part of a UnixFS file and we can do range requests -*/ - -func (api *GraphGateway) loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx context.Context, path string) (gateway.IPFSBackend, func(), error) { - bstore := api.bstore - carFetchingExch := newInboundBlockExchange() - doneWithFetcher := make(chan struct{}, 1) - exch := &handoffExchange{ - startingExchange: carFetchingExch, - followupExchange: &blockFetcherExchWrapper{api.blockFetcher}, - bstore: bstore, - handoffCh: doneWithFetcher, - metrics: api.metrics, - } - - notifierKey := api.getRootOfPath(path) - var notifier *notifiersForRootCid - for { - notifiers, _ := api.notifiers.LoadOrStore(notifierKey, ¬ifiersForRootCid{notifiers: []Notifier{}}) - if n, ok := notifiers.(*notifiersForRootCid); ok { - n.lk.Lock() - // could have been deleted after our load. try again. - if n.deleted != 0 { - n.lk.Unlock() - continue - } - notifier = n - n.notifiers = append(n.notifiers, exch) - n.lk.Unlock() - break - } else { - return nil, nil, errors.New("failed to get notifier") - } - } - - go func(metrics *GraphGatewayMetrics) { - defer func() { - if r := recover(); r != nil { - // TODO: move to Debugw? - graphLog.Errorw("Recovered fetcher error", "path", path, "error", r, "stacktrace", string(debug.Stack())) - } - }() - metrics.carFetchAttemptMetric.Inc() - - if ce := ctx.Err(); ce != nil && errors.Is(ce, context.Canceled) { - metrics.contextAlreadyCancelledMetric.Inc() - } - - cctx, cncl := context.WithCancel(ctx) - defer cncl() - err := api.fetcher.Fetch(cctx, path, func(resource string, reader io.Reader) error { - cr, err := car.NewCarReader(reader) - if err != nil { - return err - } - - cbCtx, cncl := context.WithCancel(cctx) - defer cncl() - - type blockRead struct { - block blocks.Block - err error - } - - blkCh := make(chan blockRead, 1) - go func() { - defer close(blkCh) - for { - blk, rdErr := cr.Next() - select { - case blkCh <- blockRead{blk, rdErr}: - case <-cbCtx.Done(): - return - } - } - }() - - // initially set a higher timeout here so that if there's an initial timeout error we get it from the car reader. - t := time.NewTimer(GetBlockTimeout * 2) - for { - var blkRead blockRead - var ok bool - select { - case blkRead, ok = <-blkCh: - if !t.Stop() { - <-t.C - } - t.Reset(GetBlockTimeout) - case <-t.C: - return gateway.ErrGatewayTimeout - } - if !ok || blkRead.err != nil { - if errors.Is(blkRead.err, io.EOF) { - return nil - } - return blkRead.err - } - if blkRead.block != nil { - if err := bstore.Put(ctx, blkRead.block); err != nil { - return err - } - metrics.carBlocksFetchedMetric.Inc() - api.notifyOngoingRequests(ctx, notifierKey, blkRead.block) - } - } - }) - if err != nil { - graphLog.Infow("car Fetch failed", "path", path, "error", err) - } - if err := carFetchingExch.Close(); err != nil { - graphLog.Errorw("carFetchingExch.Close()", "error", err) - } - doneWithFetcher <- struct{}{} - close(doneWithFetcher) - }(api.metrics) - - bserv := blockservice.New(bstore, exch) - blkgw, err := gateway.NewBlocksBackend(bserv) - if err != nil { - return nil, nil, err - } - - return blkgw, func() { - notifier.lk.Lock() - for i, e := range notifier.notifiers { - if e == exch { - notifier.notifiers = append(notifier.notifiers[0:i], notifier.notifiers[i+1:]...) - break - } - } - if len(notifier.notifiers) == 0 { - notifier.deleted = 1 - api.notifiers.Delete(notifierKey) - } - notifier.lk.Unlock() - }, nil -} - -func (api *GraphGateway) notifyOngoingRequests(ctx context.Context, key string, blks ...blocks.Block) { - if notifiers, ok := api.notifiers.Load(key); ok { - notifier, ok := notifiers.(*notifiersForRootCid) - if !ok { - graphLog.Errorw("notifyOngoingRequests failed", "key", key, "error", "could not get notifiersForRootCid") - return - } - notifier.lk.RLock() - for _, n := range notifier.notifiers { - err := n.NotifyNewBlocks(ctx, blks...) - if err != nil { - graphLog.Errorw("notifyOngoingRequests failed", "key", key, "error", err) - } - } - notifier.lk.RUnlock() - } -} - type fileCloseWrapper struct { files.File closeFn func() @@ -480,38 +316,49 @@ func wrapNodeWithClose[T files.Node](node T, closeFn func()) (T, error) { } } +func (api *GraphGateway) fetchCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams, cb DataCallback) error { + escapedPath := url.PathEscape(path.String()[1:]) + paramsBuilder := strings.Builder{} + paramsBuilder.WriteString("dag-scope=") + paramsBuilder.WriteString(string(params.Scope)) + if params.Range != nil { + paramsBuilder.WriteString("&entity-bytes=") + paramsBuilder.WriteString(strconv.FormatInt(params.Range.From, 10)) + paramsBuilder.WriteString(":") + if params.Range.To != nil { + paramsBuilder.WriteString(strconv.FormatInt(*params.Range.To, 10)) + } else { + paramsBuilder.WriteString("*") + } + } + urlWithoutHost := fmt.Sprintf("/%s?%s", escapedPath, paramsBuilder.String()) + return api.fetcher.Fetch(ctx, urlWithoutHost, cb) +} + func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, byteRanges ...gateway.ByteRange) (gateway.ContentPathMetadata, *gateway.GetResponse, error) { rangeCount := len(byteRanges) api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "entity", "entityRanges": strconv.Itoa(rangeCount)}).Inc() - // TODO: remove &bytes= &depth= and &car-scope from all CAR request after transition is done: - // https://github.com/ipfs/bifrost-gateway/issues/80 - carParams := "?format=car&dag-scope=entity" + carParams := gateway.CarParams{Scope: gateway.DagScopeEntity} // fetch CAR with &bytes= to get minimal set of blocks for the request // Note: majority of requests have 0 or max 1 ranges. if there are more ranges than one, // that is a niche edge cache we don't prefetch as CAR and use fallback blockstore instead. if rangeCount > 0 { - bytesBuilder := strings.Builder{} - bytesBuilder.WriteString("&entity-bytes=") r := byteRanges[0] - - bytesBuilder.WriteString(strconv.FormatUint(r.From, 10)) - bytesBuilder.WriteString(":") + carParams.Range = &gateway.DagByteRange{ + From: int64(r.From), + } // TODO: move to boxo or to loadRequestIntoSharedBlockstoreAndBlocksGateway after we pass params in a humane way api.metrics.bytesRangeStartMetric.Observe(float64(r.From)) if r.To != nil { - bytesBuilder.WriteString(strconv.FormatInt(*r.To, 10)) + carParams.Range.To = r.To // TODO: move to boxo or to loadRequestIntoSharedBlockstoreAndBlocksGateway after we pass params in a humane way api.metrics.bytesRangeSizeMetric.Observe(float64(*r.To) - float64(r.From) + 1) - } else { - bytesBuilder.WriteString("*") } - - carParams += bytesBuilder.String() } rootCid, err := getRootCid(path) @@ -532,10 +379,11 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by var terminalFile *multiReadCloser var terminalDir chan unixfs.LinkResult lastDirLinkNum := 0 + hasSentAsyncData := false go func() { cctx, cancel := context.WithCancel(ctx) defer cancel() - err = api.fetcher.Fetch(cctx, path.String()+"?format=car&dag-scope=all", func(resource string, reader io.Reader) error { + err = api.fetchCAR(cctx, path, carParams, func(resource string, reader io.Reader) error { gb, err := carToLinearBlockGetter(cctx, reader, api.metrics) if err != nil { return err @@ -577,15 +425,19 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { // If it's not valid dag-pb then we're done + terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromReader(files.NewBytesFile(nodeBytes), int64(len(nodeBytes))), lastCid: lastCid} return nil } else if len(remainder) > 0 { // If we're trying to path into dag-pb node that's invalid and we're done + terminalPathElementCh <- terminalPathType{err: fmt.Errorf("cannot path into non-UnixFS dagpb")} return nil } else if !pbn.FieldData().Exists() { // If it's not valid UnixFS then we're done + terminalPathElementCh <- terminalPathType{err: fmt.Errorf("not valid UnixFS")} return nil } else if unixfsFieldData, decodeErr := ufsData.DecodeUnixFSData(pbn.Data.Must().Bytes()); decodeErr != nil { // If it's not valid dag-pb and UnixFS then we're done + terminalPathElementCh <- terminalPathType{err: fmt.Errorf("not valid UnixFS")} return nil } else { switch fieldNum := unixfsFieldData.FieldDataType().Int(); fieldNum { @@ -630,12 +482,10 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by return nil } - lnks := make(chan unixfs.LinkResult) - defer close(lnks) if terminalDir == nil { - terminalDir = lnks + terminalDir = make(chan unixfs.LinkResult) + terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromDirectoryListing(sz, terminalDir, nil)} } - terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromDirectoryListing(sz, lnks, nil)} dirLinkNum := 0 switch fieldNum { @@ -664,7 +514,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by } select { - case lnks <- lnk: + case terminalDir <- lnk: lastDirLinkNum++ case <-cctx.Done(): // TODO: what here, send an error with another select? @@ -672,11 +522,12 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by } } case ufsData.Data_HAMTShard: + hasSentAsyncData = true // Note: we are making up the entries dirNd, err := unixfsnode.Reify(lctx, pbn, lsys) if err != nil { select { - case lnks <- unixfs.LinkResult{Err: fmt.Errorf("could not reify sharded directory: %w", err)}: + case terminalDir <- unixfs.LinkResult{Err: fmt.Errorf("could not reify sharded directory: %w", err)}: case <-cctx.Done(): // TODO: what here? } @@ -692,7 +543,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by keyStr, err := k.AsString() if err != nil { select { - case lnks <- unixfs.LinkResult{Err: fmt.Errorf("could not interpret directory key as string: %w", err)}: + case terminalDir <- unixfs.LinkResult{Err: fmt.Errorf("could not interpret directory key as string: %w", err)}: case <-cctx.Done(): // TODO: what here? } @@ -701,7 +552,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by valLink, err := v.AsLink() if err != nil { select { - case lnks <- unixfs.LinkResult{Err: fmt.Errorf("could not interpret directory value as link: %w", err)}: + case terminalDir <- unixfs.LinkResult{Err: fmt.Errorf("could not interpret directory value as link: %w", err)}: case <-cctx.Done(): // TODO: what here? } @@ -720,7 +571,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by } select { - case lnks <- lnk: + case terminalDir <- lnk: lastDirLinkNum++ case <-cctx.Done(): // TODO: what here? @@ -765,6 +616,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by } terminalFile = mrc terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromReader(files.NewReaderFile(mrc), fileSize), lastCid: lastCid} + hasSentAsyncData = true } else { select { case terminalFile.newReader <- f: @@ -774,9 +626,11 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by } } - _, err = io.Copy(io.Discard, terminalFile) - if err != nil { - return err + select { + case <-terminalFile.closed: + return nil + case <-cctx.Done(): + // TODO: what here? } return nil default: @@ -785,6 +639,22 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by } } }) + if err != nil { + if !hasSentAsyncData { + terminalPathElementCh <- terminalPathType{err: fmt.Errorf("failed fetch: %w", err)} + } else if terminalFile != nil { + // TODO: this error should surface through the io.Reader + } else if terminalDir != nil { + select { + case terminalDir <- unixfs.LinkResult{Err: err}: + case <-cctx.Done(): + // TODO: what here? + } + } + } + if terminalDir != nil { + close(terminalDir) + } }() select { @@ -856,20 +726,7 @@ func (r *multiReadCloser) Close() error { var _ io.ReadCloser = (*multiReadCloser)(nil) func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { - api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "all", "entityRanges": "0"}).Inc() - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=all") - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - md, f, err := blkgw.GetAll(ctx, path) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - f, err = wrapNodeWithClose(f, closeFn) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - return md, f, nil + panic("not implemented") } func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { @@ -885,7 +742,7 @@ func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePat var f files.File var ipldError error // TODO: fallback to dynamic fetches in case we haven't requested enough data - err = api.fetcher.Fetch(ctx, path.String()+"?format=car&dag-scope=block", func(resource string, reader io.Reader) error { + err = api.fetchCAR(ctx, path, gateway.CarParams{Scope: gateway.DagScopeBlock}, func(resource string, reader io.Reader) error { return checkRetryableError(&ipldError, func() error { gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) if err != nil { @@ -944,7 +801,8 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) ( var n *gateway.HeadResponse var ipldError error // TODO: fallback to dynamic fetches in case we haven't requested enough data - err = api.fetcher.Fetch(ctx, path.String()+"?format=car&dag-scope=entity&entity-bytes=0:3071", func(resource string, reader io.Reader) error { + rangeTo := int64(3071) + err = api.fetchCAR(ctx, path, gateway.CarParams{Scope: gateway.DagScopeEntity, Range: &gateway.DagByteRange{From: 0, To: &rangeTo}}, func(resource string, reader io.Reader) error { return checkRetryableError(&ipldError, func() error { gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) if err != nil { @@ -1103,7 +961,7 @@ func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.Immutable var md gateway.ContentPathMetadata var ipldError error - err = api.fetcher.Fetch(ctx, path.String()+"?format=car&dag-scope=block", func(resource string, reader io.Reader) error { + err = api.fetchCAR(ctx, path, gateway.CarParams{Scope: gateway.DagScopeBlock}, func(resource string, reader io.Reader) error { return checkRetryableError(&ipldError, func() error { gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) if err != nil { @@ -1150,7 +1008,7 @@ func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath, var cw storage.WritableCar var blockBuffer []blocks.Block var ipldError error - err = api.fetcher.Fetch(ctx, path.String()+"?format=car&dag-scope=all", func(resource string, reader io.Reader) error { + err = api.fetchCAR(ctx, path, gateway.CarParams{Scope: gateway.DagScopeAll}, func(resource string, reader io.Reader) error { return checkRetryableError(&ipldError, func() error { numBlocksThisCall := 0 gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) From 2458e3eec197e941603684c5c36456b4eb248960 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 6 Jul 2023 11:42:32 -0400 Subject: [PATCH 06/46] fix detection of go-ipld-prime schema errors and path resolution --- lib/gateway_traversal.go | 75 ++--- lib/graph_gateway.go | 635 ++++++++++++++++++++++----------------- 2 files changed, 392 insertions(+), 318 deletions(-) diff --git a/lib/gateway_traversal.go b/lib/gateway_traversal.go index d031aa4..854a3ef 100644 --- a/lib/gateway_traversal.go +++ b/lib/gateway_traversal.go @@ -11,15 +11,11 @@ import ( "github.com/ipld/go-ipld-prime/linking" - "github.com/ipfs/boxo/blockservice" - "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/exchange" bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/gateway" - "github.com/ipfs/boxo/path/resolver" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" "github.com/ipfs/go-unixfsnode" "github.com/ipfs/go-unixfsnode/data" "github.com/ipld/go-car" @@ -137,17 +133,7 @@ func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *Grap }, nil } -func getIPFSPathResolverAndLsysFromBlockReader(ctx context.Context, fn getBlock) (resolver.Resolver, *ipld.LinkSystem) { - fetcher := bsfetcher.NewFetcherConfig( - blockservice.New(blockstore.NewBlockstore(&datastore.NullDatastore{}), &blockFetcherExchWrapper{f: &gbf{fn}})) - fetcher.NodeReifier = unixfsnode.Reify - fetcher.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - res := resolver.NewBasicResolver(fetcher) +func getLinksystem(fn getBlock) *ipld.LinkSystem { lsys := cidlink.DefaultLinkSystem() lsys.StorageReadOpener = func(linkContext linking.LinkContext, link datamodel.Link) (io.Reader, error) { c := link.(cidlink.Link).Cid @@ -159,28 +145,50 @@ func getIPFSPathResolverAndLsysFromBlockReader(ctx context.Context, fn getBlock) } lsys.TrustedStorage = true unixfsnode.AddUnixFSReificationToLinkSystem(&lsys) - return res, &lsys + return &lsys } // walkGatewaySimpleSelector walks the subgraph described by the path and terminal element parameters -func walkGatewaySimpleSelector(ctx context.Context, lastCid cid.Cid, params gateway.CarParams, lsys *ipld.LinkSystem, pathResolver resolver.Resolver) error { +func walkGatewaySimpleSelector(ctx context.Context, terminalBlk blocks.Block, params gateway.CarParams, lsys *ipld.LinkSystem) error { lctx := ipld.LinkContext{Ctx: ctx} - pathTerminalCidLink := cidlink.Link{Cid: lastCid} var err error - // If the scope is the block, now we only need to retrieve the root block of the last element of the path. + // If the scope is the block, we only need the root block of the last element of the path, which we have. if params.Scope == gateway.DagScopeBlock { - _, err = lsys.LoadRaw(lctx, pathTerminalCidLink) + return nil + } + + // decode the terminal block into a node + pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + + pathTerminalCidLink := cidlink.Link{Cid: terminalBlk.Cid()} + np, err := pc(pathTerminalCidLink, lctx) + if err != nil { + return err + } + + decoder, err := lsys.DecoderChooser(pathTerminalCidLink) + if err != nil { + return err + } + nb := np.NewBuilder() + blockData := terminalBlk.RawData() + if err := decoder(nb, bytes.NewReader(blockData)); err != nil { return err } + lastCidNode := nb.Build() + + // TODO: Evaluate: + // Does it matter that we're ignoring the "remainder" portion of the traversal in GetCAR? + // Does it matter that we're using a linksystem with the UnixFS reifier for dagscope=all? // If we're asking for everything then give it if params.Scope == gateway.DagScopeAll { - lastCidNode, err := lsys.Load(lctx, pathTerminalCidLink, basicnode.Prototype.Any) - if err != nil { - return err - } - sel, err := selector.ParseSelector(selectorparse.CommonSelector_ExploreAllRecursively) if err != nil { return err @@ -206,23 +214,6 @@ func walkGatewaySimpleSelector(ctx context.Context, lastCid cid.Cid, params gate // From now on, dag-scope=entity! // Since we need more of the graph load it to figure out what we have // This includes determining if the terminal node is UnixFS or not - pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - - np, err := pc(pathTerminalCidLink, lctx) - if err != nil { - return err - } - - lastCidNode, err := lsys.Load(lctx, pathTerminalCidLink, np) - if err != nil { - return err - } - if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { // If it's not valid dag-pb then we're done return nil diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index c52acb0..4a09103 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -14,7 +14,7 @@ import ( "sync" "time" - "github.com/ipfs/boxo/path/resolver" + "github.com/ipld/go-ipld-prime/traversal" "github.com/filecoin-saturn/caboose" blockstore "github.com/ipfs/boxo/blockstore" @@ -28,13 +28,13 @@ import ( "github.com/ipfs/boxo/namesys" "github.com/ipfs/boxo/namesys/resolve" ipfspath "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path/resolver" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" golog "github.com/ipfs/go-log/v2" "github.com/ipfs/go-unixfsnode" ufsData "github.com/ipfs/go-unixfsnode/data" - "github.com/ipld/go-car" carv2 "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/storage" dagpb "github.com/ipld/go-codec-dagpb" @@ -114,6 +114,8 @@ type GraphGateway struct { namesys namesys.NameSystem bstore blockstore.Blockstore + pc traversal.LinkTargetNodePrototypeChooser + notifiers sync.Map // cid -> notifiersForRootCid metrics *GraphGatewayMetrics } @@ -177,6 +179,12 @@ func NewGraphGatewayBackend(f CarFetcher, blockFetcher exchange.Fetcher, opts .. bstore: bs, notifiers: sync.Map{}, metrics: registerGraphGatewayMetrics(), + pc: dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }), }, nil } @@ -332,7 +340,137 @@ func (api *GraphGateway) fetchCAR(ctx context.Context, path gateway.ImmutablePat } } urlWithoutHost := fmt.Sprintf("/%s?%s", escapedPath, paramsBuilder.String()) - return api.fetcher.Fetch(ctx, urlWithoutHost, cb) + + var ipldError error + fetchErr := api.fetcher.Fetch(ctx, urlWithoutHost, func(resource string, reader io.Reader) error { + return checkRetryableError(&ipldError, func() error { + return cb(resource, reader) + }) + }) + + if ipldError != nil { + fetchErr = ipldError + } + + return fetchErr +} + +// resolvePathWithRootsAndBlock takes a path and linksystem and returns the set of non-terminal cids, the terminal cid, the remainder, and the block corresponding to the terminal cid +func (api *GraphGateway) resolvePathWithRootsAndBlock(ctx context.Context, fpath ipfspath.Path, unixFSLsys *ipld.LinkSystem) ([]cid.Cid, cid.Cid, []string, blocks.Block, error) { + pathRootCids, terminalCid, remainder, terminalBlk, err := api.resolvePathToLastWithRoots(ctx, fpath, unixFSLsys) + if err != nil { + return nil, cid.Undef, nil, nil, err + } + + if terminalBlk == nil { + lctx := ipld.LinkContext{Ctx: ctx} + lnk := cidlink.Link{Cid: terminalCid} + blockData, err := unixFSLsys.LoadRaw(lctx, lnk) + if err != nil { + return nil, cid.Undef, nil, nil, err + } + terminalBlk, err = blocks.NewBlockWithCid(blockData, terminalCid) + if err != nil { + return nil, cid.Undef, nil, nil, err + } + } + + return pathRootCids, terminalCid, remainder, terminalBlk, err +} + +// resolvePathToLastWithRoots takes a path and linksystem and returns the set of non-terminal cids, the terminal cid, +// the remainder pathing, the last block loaded, and the last node loaded. +// +// Note: the block returned will be nil if the terminal element is a link or the path is just a CID +func (api *GraphGateway) resolvePathToLastWithRoots(ctx context.Context, fpath ipfspath.Path, unixFSLsys *ipld.LinkSystem) ([]cid.Cid, cid.Cid, []string, blocks.Block, error) { + c, p, err := ipfspath.SplitAbsPath(fpath) + if err != nil { + return nil, cid.Undef, nil, nil, err + } + + if len(p) == 0 { + return nil, c, nil, nil, nil + } + + unixFSLsys.NodeReifier = unixfsnode.Reify + defer func() { unixFSLsys.NodeReifier = nil }() + + var cids []cid.Cid + cids = append(cids, c) + + loadNode := func(ctx context.Context, c cid.Cid) (blocks.Block, ipld.Node, error) { + lctx := ipld.LinkContext{Ctx: ctx} + rootLnk := cidlink.Link{Cid: c} + np, err := api.pc(rootLnk, lctx) + if err != nil { + return nil, nil, err + } + nd, blockData, err := unixFSLsys.LoadPlusRaw(lctx, rootLnk, np) + if err != nil { + return nil, nil, err + } + blk, err := blocks.NewBlockWithCid(blockData, c) + if err != nil { + return nil, nil, err + } + return blk, nd, nil + } + + nextBlk, nextNd, err := loadNode(ctx, c) + if err != nil { + return nil, cid.Undef, nil, nil, err + } + + depth := 0 + for i, elem := range p { + nextNd, err = nextNd.LookupBySegment(ipld.ParsePathSegment(elem)) + if err != nil { + return nil, cid.Undef, nil, nil, err + } + if nextNd.Kind() == ipld.Kind_Link { + depth = 0 + lnk, err := nextNd.AsLink() + if err != nil { + return nil, cid.Undef, nil, nil, err + } + cidLnk, ok := lnk.(cidlink.Link) + if !ok { + return nil, cid.Undef, nil, nil, fmt.Errorf("link is not a cidlink: %v", cidLnk) + } + cids = append(cids, cidLnk.Cid) + + if i < len(p)-1 { + nextBlk, nextNd, err = loadNode(ctx, cidLnk.Cid) + if err != nil { + return nil, cid.Undef, nil, nil, err + } + } + } else { + depth++ + } + } + + // if last node is not a link, just return it's cid, add path to remainder and return + if nextNd.Kind() != ipld.Kind_Link { + // return the cid and the remainder of the path + return cids[:len(cids)-1], cids[len(cids)-1], p[len(p)-depth:], nextBlk, nil + } + + return cids[:len(cids)-1], cids[len(cids)-1], nil, nil, nil +} + +func contentMetadataFromRootsAndRemainder(p ipfspath.Path, pathRoots []cid.Cid, terminalCid cid.Cid, remainder []string) gateway.ContentPathMetadata { + var rootCid cid.Cid + if len(pathRoots) > 0 { + rootCid = pathRoots[0] + } else { + rootCid = terminalCid + } + md := gateway.ContentPathMetadata{ + PathSegmentRoots: pathRoots, + LastSegment: ifacepath.NewResolvedPath(p, terminalCid, rootCid, gopath.Join(remainder...)), + } + return md } func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, byteRanges ...gateway.ByteRange) (gateway.ContentPathMetadata, *gateway.GetResponse, error) { @@ -361,16 +499,12 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by } } - rootCid, err := getRootCid(path) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } p := ipfspath.FromString(path.String()) type terminalPathType struct { - resp *gateway.GetResponse - err error - lastCid cid.Cid + resp *gateway.GetResponse + err error + md gateway.ContentPathMetadata } terminalPathElementCh := make(chan terminalPathType, 1) @@ -383,20 +517,22 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by go func() { cctx, cancel := context.WithCancel(ctx) defer cancel() - err = api.fetchCAR(cctx, path, carParams, func(resource string, reader io.Reader) error { + err := api.fetchCAR(cctx, path, carParams, func(resource string, reader io.Reader) error { gb, err := carToLinearBlockGetter(cctx, reader, api.metrics) if err != nil { return err } - r, lsys := getIPFSPathResolverAndLsysFromBlockReader(cctx, gb) + + lsys := getLinksystem(gb) // First resolve the path since we always need to. - lastCid, remainder, err := r.ResolveToLastNode(cctx, p) + pathRootCids, terminalCid, remainder, terminalBlk, err := api.resolvePathWithRootsAndBlock(cctx, p, lsys) if err != nil { return err } + md := contentMetadataFromRootsAndRemainder(p, pathRootCids, terminalCid, remainder) lctx := ipld.LinkContext{Ctx: cctx} - pathTerminalCidLink := cidlink.Link{Cid: lastCid} + pathTerminalCidLink := cidlink.Link{Cid: terminalCid} if len(remainder) > 0 { terminalPathElementCh <- terminalPathType{err: errNotUnixFS} @@ -406,26 +542,25 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by // From now on, dag-scope=entity! // Since we need more of the graph load it to figure out what we have // This includes determining if the terminal node is UnixFS or not - pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - - np, err := pc(pathTerminalCidLink, lctx) + np, err := api.pc(pathTerminalCidLink, lctx) if err != nil { return err } - lastCidNode, nodeBytes, err := lsys.LoadPlusRaw(lctx, pathTerminalCidLink, np) + decoder, err := lsys.DecoderChooser(pathTerminalCidLink) if err != nil { return err } + nb := np.NewBuilder() + blockData := terminalBlk.RawData() + if err := decoder(nb, bytes.NewReader(blockData)); err != nil { + return err + } + lastCidNode := nb.Build() if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { // If it's not valid dag-pb then we're done - terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromReader(files.NewBytesFile(nodeBytes), int64(len(nodeBytes))), lastCid: lastCid} + terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromReader(files.NewBytesFile(blockData), int64(len(blockData))), md: md} return nil } else if len(remainder) > 0 { // If we're trying to path into dag-pb node that's invalid and we're done @@ -451,7 +586,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by terminalPathElementCh <- terminalPathType{err: fmt.Errorf("should be unreachable: symlink does not have a symlink type")} return nil } - terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromSymlink(s, int64(len(lnkTarget))), lastCid: lastCid} + terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromSymlink(s, int64(len(lnkTarget))), md: md} return nil } terminalPathElementCh <- terminalPathType{err: fmt.Errorf("UnixFS Symlink does not contain target")} @@ -460,7 +595,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by terminalPathElementCh <- terminalPathType{err: fmt.Errorf("UnixFS Metadata unsupported")} return nil case ufsData.Data_HAMTShard, ufsData.Data_Directory: - blk, err := blocks.NewBlockWithCid(nodeBytes, lastCid) + blk, err := blocks.NewBlockWithCid(blockData, pathTerminalCidLink.Cid) if err != nil { terminalPathElementCh <- terminalPathType{err: fmt.Errorf("could not create block: %w", err)} return nil @@ -484,7 +619,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by if terminalDir == nil { terminalDir = make(chan unixfs.LinkResult) - terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromDirectoryListing(sz, terminalDir, nil)} + terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromDirectoryListing(sz, terminalDir, nil), md: md} } dirLinkNum := 0 @@ -509,7 +644,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by }} dirLinkNum++ - if dirLinkNum-1 <= lastDirLinkNum { + if dirLinkNum-1 < lastDirLinkNum { continue } @@ -615,7 +750,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by isClosed: false, } terminalFile = mrc - terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromReader(files.NewReaderFile(mrc), fileSize), lastCid: lastCid} + terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromReader(files.NewReaderFile(mrc), fileSize), md: md} hasSentAsyncData = true } else { select { @@ -663,11 +798,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by return gateway.ContentPathMetadata{}, nil, t.err } - md := gateway.ContentPathMetadata{ - PathSegmentRoots: []cid.Cid{rootCid}, - LastSegment: ifacepath.NewResolvedPath(p, t.lastCid, rootCid, ""), - } - return md, t.resp, nil + return t.md, t.resp, nil case <-ctx.Done(): return gateway.ContentPathMetadata{}, nil, ctx.Err() } @@ -731,52 +862,42 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "block", "entityRanges": "0"}).Inc() - // TODO: if path is `/ipfs/cid`, we should use ?format=raw - rootCid, err := getRootCid(path) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } p := ipfspath.FromString(path.String()) var md gateway.ContentPathMetadata var f files.File - var ipldError error - // TODO: fallback to dynamic fetches in case we haven't requested enough data - err = api.fetchCAR(ctx, path, gateway.CarParams{Scope: gateway.DagScopeBlock}, func(resource string, reader io.Reader) error { - return checkRetryableError(&ipldError, func() error { - gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) - if err != nil { - return err - } - r, lsys := getIPFSPathResolverAndLsysFromBlockReader(ctx, gb) - - // First resolve the path since we always need to. - lastCid, remainder, err := r.ResolveToLastNode(ctx, p) - if err != nil { - return err - } + // TODO: if path is `/ipfs/cid`, we should use ?format=raw + err := api.fetchCAR(ctx, path, gateway.CarParams{Scope: gateway.DagScopeBlock}, func(resource string, reader io.Reader) error { + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + lsys := getLinksystem(gb) - md = gateway.ContentPathMetadata{ - PathSegmentRoots: []cid.Cid{rootCid}, - LastSegment: ifacepath.NewResolvedPath(p, lastCid, rootCid, gopath.Join(remainder...)), - } + // First resolve the path since we always need to. + pathRoots, terminalCid, remainder, terminalBlk, err := api.resolvePathToLastWithRoots(ctx, p, lsys) + if err != nil { + return err + } + var blockData []byte + if terminalBlk != nil { + blockData = terminalBlk.RawData() + } else { lctx := ipld.LinkContext{Ctx: ctx} - pathTerminalCidLink := cidlink.Link{Cid: lastCid} - - data, err := lsys.LoadRaw(lctx, pathTerminalCidLink) + lnk := cidlink.Link{Cid: terminalCid} + blockData, err = lsys.LoadRaw(lctx, lnk) if err != nil { return err } + } - f = files.NewBytesFile(data) - return nil - }) + md = contentMetadataFromRootsAndRemainder(p, pathRoots, terminalCid, remainder) + + f = files.NewBytesFile(blockData) + return nil }) - if ipldError != nil { - err = ipldError - } if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -791,160 +912,133 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) ( api.metrics.bytesRangeStartMetric.Observe(0) api.metrics.bytesRangeSizeMetric.Observe(3071) - rootCid, err := getRootCid(path) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } p := ipfspath.FromString(path.String()) var md gateway.ContentPathMetadata var n *gateway.HeadResponse - var ipldError error // TODO: fallback to dynamic fetches in case we haven't requested enough data rangeTo := int64(3071) - err = api.fetchCAR(ctx, path, gateway.CarParams{Scope: gateway.DagScopeEntity, Range: &gateway.DagByteRange{From: 0, To: &rangeTo}}, func(resource string, reader io.Reader) error { - return checkRetryableError(&ipldError, func() error { - gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) - if err != nil { - return err - } - r, lsys := getIPFSPathResolverAndLsysFromBlockReader(ctx, gb) + err := api.fetchCAR(ctx, path, gateway.CarParams{Scope: gateway.DagScopeEntity, Range: &gateway.DagByteRange{From: 0, To: &rangeTo}}, func(resource string, reader io.Reader) error { + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + lsys := getLinksystem(gb) - // First resolve the path since we always need to. - lastCid, remainder, err := r.ResolveToLastNode(ctx, p) - if err != nil { - return err - } + // First resolve the path since we always need to. + pathRoots, terminalCid, remainder, terminalBlk, err := api.resolvePathWithRootsAndBlock(ctx, p, lsys) + if err != nil { + return err + } - md = gateway.ContentPathMetadata{ - PathSegmentRoots: []cid.Cid{rootCid}, - LastSegment: ifacepath.NewResolvedPath(p, lastCid, rootCid, gopath.Join(remainder...)), - } + md = contentMetadataFromRootsAndRemainder(p, pathRoots, terminalCid, remainder) - lctx := ipld.LinkContext{Ctx: ctx} - pathTerminalCidLink := cidlink.Link{Cid: lastCid} + lctx := ipld.LinkContext{Ctx: ctx} + pathTerminalCidLink := cidlink.Link{Cid: terminalCid} - // Load the block at the root of the terminal path element - dataBytes, err := lsys.LoadRaw(lctx, pathTerminalCidLink) - if err != nil { - return err - } + // Load the block at the root of the terminal path element + dataBytes := terminalBlk.RawData() - // It's not UnixFS if there is a remainder or it's not dag-pb - if len(remainder) > 0 || lastCid.Type() != uint64(multicodec.DagPb) { - n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) - return nil - } + // It's not UnixFS if there is a remainder or it's not dag-pb + if len(remainder) > 0 || terminalCid.Type() != uint64(multicodec.DagPb) { + n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + return nil + } - // Let's figure out if the terminal element is valid UnixFS and if so what kind - pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) + // Let's figure out if the terminal element is valid UnixFS and if so what kind + np, err := api.pc(pathTerminalCidLink, lctx) + if err != nil { + return err + } - np, err := pc(pathTerminalCidLink, lctx) - if err != nil { - return err - } + nodeDecoder, err := lsys.DecoderChooser(pathTerminalCidLink) + if err != nil { + return err + } - nodeDecoder, err := lsys.DecoderChooser(pathTerminalCidLink) - if err != nil { - return err - } + nb := np.NewBuilder() + err = nodeDecoder(nb, bytes.NewReader(dataBytes)) + if err != nil { + return err + } + lastCidNode := nb.Build() + + if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { + // This shouldn't be possible since we already checked for dag-pb usage + return fmt.Errorf("node was not go-codec-dagpb node") + } else if !pbn.FieldData().Exists() { + // If it's not valid UnixFS then just return the block bytes + n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + return nil + } else if unixfsFieldData, decodeErr := ufsData.DecodeUnixFSData(pbn.Data.Must().Bytes()); decodeErr != nil { + // If it's not valid UnixFS then just return the block bytes + n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + return nil + } else { + switch fieldNum := unixfsFieldData.FieldDataType().Int(); fieldNum { + case ufsData.Data_Directory, ufsData.Data_HAMTShard: + dirRootNd, err := merkledag.ProtoNodeConverter(terminalBlk, lastCidNode) + if err != nil { + return fmt.Errorf("could not create dag-pb universal block from UnixFS directory root: %w", err) + } + pn, ok := dirRootNd.(*merkledag.ProtoNode) + if !ok { + return fmt.Errorf("could not create dag-pb node from UnixFS directory root: %w", err) + } - nb := np.NewBuilder() - err = nodeDecoder(nb, bytes.NewReader(dataBytes)) - if err != nil { - return err - } - lastCidNode := nb.Build() + sz, err := pn.Size() + if err != nil { + return fmt.Errorf("could not get cumulative size from dag-pb node: %w", err) + } - if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { - // This shouldn't be possible since we already checked for dag-pb usage - return fmt.Errorf("node was not go-codec-dagpb node") - } else if !pbn.FieldData().Exists() { - // If it's not valid UnixFS then just return the block bytes - n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + n = gateway.NewHeadResponseForDirectory(int64(sz)) return nil - } else if unixfsFieldData, decodeErr := ufsData.DecodeUnixFSData(pbn.Data.Must().Bytes()); decodeErr != nil { - // If it's not valid UnixFS then just return the block bytes + case ufsData.Data_Symlink: + fd := unixfsFieldData.FieldData() + if fd.Exists() { + n = gateway.NewHeadResponseForSymlink(int64(len(fd.Must().Bytes()))) + return nil + } + // If there is no target then it's invalid so just return the block + gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + return nil + case ufsData.Data_Metadata: n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) return nil - } else { - switch fieldNum := unixfsFieldData.FieldDataType().Int(); fieldNum { - case ufsData.Data_Directory, ufsData.Data_HAMTShard: - blk, err := blocks.NewBlockWithCid(dataBytes, lastCid) - if err != nil { - return fmt.Errorf("could not create block: %w", err) - } - dirRootNd, err := merkledag.ProtoNodeConverter(blk, lastCidNode) - if err != nil { - return fmt.Errorf("could not create dag-pb universal block from UnixFS directory root: %w", err) - } - pn, ok := dirRootNd.(*merkledag.ProtoNode) - if !ok { - return fmt.Errorf("could not create dag-pb node from UnixFS directory root: %w", err) - } + case ufsData.Data_Raw, ufsData.Data_File: + ufsNode, err := unixfsnode.Reify(lctx, pbn, lsys) + if err != nil { + return err + } + fileNode, ok := ufsNode.(datamodel.LargeBytesNode) + if !ok { + return fmt.Errorf("data not a large bytes node despite being UnixFS bytes") + } + f, err := fileNode.AsLargeBytes() + if err != nil { + return err + } - sz, err := pn.Size() - if err != nil { - return fmt.Errorf("could not get cumulative size from dag-pb node: %w", err) - } + fileSize, err := f.Seek(0, io.SeekEnd) + if err != nil { + return fmt.Errorf("unable to get UnixFS file size: %w", err) + } + _, err = f.Seek(0, io.SeekStart) + if err != nil { + return fmt.Errorf("unable to get reset UnixFS file reader: %w", err) + } - n = gateway.NewHeadResponseForDirectory(int64(sz)) + out, err := io.ReadAll(io.LimitReader(f, 3072)) + if errors.Is(err, io.EOF) { + n = gateway.NewHeadResponseForFile(files.NewBytesFile(out), fileSize) return nil - case ufsData.Data_Symlink: - fd := unixfsFieldData.FieldData() - if fd.Exists() { - n = gateway.NewHeadResponseForSymlink(int64(len(fd.Must().Bytes()))) - return nil - } - // If there is no target then it's invalid so just return the block - gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) - return nil - case ufsData.Data_Metadata: - n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) - return nil - case ufsData.Data_Raw, ufsData.Data_File: - ufsNode, err := unixfsnode.Reify(lctx, pbn, lsys) - if err != nil { - return err - } - fileNode, ok := ufsNode.(datamodel.LargeBytesNode) - if !ok { - return fmt.Errorf("data not a large bytes node despite being UnixFS bytes") - } - f, err := fileNode.AsLargeBytes() - if err != nil { - return err - } - - fileSize, err := f.Seek(0, io.SeekEnd) - if err != nil { - return fmt.Errorf("unable to get UnixFS file size: %w", err) - } - _, err = f.Seek(0, io.SeekStart) - if err != nil { - return fmt.Errorf("unable to get reset UnixFS file reader: %w", err) - } - - out, err := io.ReadAll(io.LimitReader(f, 3072)) - if errors.Is(err, io.EOF) { - n = gateway.NewHeadResponseForFile(files.NewBytesFile(out), fileSize) - return nil - } - return err } + return err } - return nil - }) + } + return nil }) - if ipldError != nil { - err = ipldError - } if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -954,39 +1048,27 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) ( func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "block", "entityRanges": "0"}).Inc() - rootCid, err := getRootCid(path) - if err != nil { - return gateway.ContentPathMetadata{}, err - } var md gateway.ContentPathMetadata - var ipldError error - err = api.fetchCAR(ctx, path, gateway.CarParams{Scope: gateway.DagScopeBlock}, func(resource string, reader io.Reader) error { - return checkRetryableError(&ipldError, func() error { - gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) - if err != nil { - return err - } - r, _ := getIPFSPathResolverAndLsysFromBlockReader(ctx, gb) + err := api.fetchCAR(ctx, path, gateway.CarParams{Scope: gateway.DagScopeBlock}, func(resource string, reader io.Reader) error { + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + lsys := getLinksystem(gb) - // First resolve the path since we always need to. - lastCid, remainder, err := r.ResolveToLastNode(ctx, ipfspath.FromString(path.String())) - if err != nil { - return err - } + // First resolve the path since we always need to. + p := ipfspath.FromString(path.String()) + pathRoots, terminalCid, remainder, _, err := api.resolvePathToLastWithRoots(ctx, p, lsys) + if err != nil { + return err + } - md = gateway.ContentPathMetadata{ - PathSegmentRoots: []cid.Cid{rootCid}, - LastSegment: ifacepath.NewResolvedPath(ipfspath.FromString(path.String()), lastCid, rootCid, gopath.Join(remainder...)), - } + md = contentMetadataFromRootsAndRemainder(p, pathRoots, terminalCid, remainder) - return nil - }) + return nil }) - if ipldError != nil { - err = ipldError - } if err != nil { return gateway.ContentPathMetadata{}, err } @@ -1007,74 +1089,65 @@ func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath, numBlocksSent := 0 var cw storage.WritableCar var blockBuffer []blocks.Block - var ipldError error - err = api.fetchCAR(ctx, path, gateway.CarParams{Scope: gateway.DagScopeAll}, func(resource string, reader io.Reader) error { - return checkRetryableError(&ipldError, func() error { - numBlocksThisCall := 0 - gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + err = api.fetchCAR(ctx, path, gateway.CarParams{Scope: params.Scope}, func(resource string, reader io.Reader) error { + numBlocksThisCall := 0 + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + teeBlock := func(ctx context.Context, c cid.Cid) (blocks.Block, error) { + blk, err := gb(ctx, c) if err != nil { - return err + return nil, err } - teeBlock := func(ctx context.Context, c cid.Cid) (blocks.Block, error) { - blk, err := gb(ctx, c) - if err != nil { - return nil, err - } - if numBlocksThisCall >= numBlocksSent { - if cw == nil { - blockBuffer = append(blockBuffer, blk) - } else { - err = cw.Put(ctx, blk.Cid().KeyString(), blk.RawData()) - if err != nil { - return nil, fmt.Errorf("error writing car block: %w", err) - } + if numBlocksThisCall >= numBlocksSent { + if cw == nil { + blockBuffer = append(blockBuffer, blk) + } else { + err = cw.Put(ctx, blk.Cid().KeyString(), blk.RawData()) + if err != nil { + return nil, fmt.Errorf("error writing car block: %w", err) } - numBlocksSent++ } - numBlocksThisCall++ - return blk, nil + numBlocksSent++ } - r, l := getIPFSPathResolverAndLsysFromBlockReader(ctx, teeBlock) + numBlocksThisCall++ + return blk, nil + } + l := getLinksystem(teeBlock) - // First resolve the path since we always need to. - lastCid, remainder, err := r.ResolveToLastNode(ctx, p) - if err != nil { - return err - } - if len(remainder) > 0 { - return nil - } + // First resolve the path since we always need to. + _, terminalCid, remainder, terminalBlk, err := api.resolvePathWithRootsAndBlock(ctx, p, l) + if err != nil { + return err + } + if len(remainder) > 0 { + return nil + } - cw, err = storage.NewWritable(w, []cid.Cid{lastCid}, carv2.WriteAsCarV1(true)) + cw, err = storage.NewWritable(w, []cid.Cid{terminalCid}, carv2.WriteAsCarV1(true)) + if err != nil { + // io.PipeWriter.CloseWithError always returns nil. + _ = w.CloseWithError(err) + return nil + } + for _, blk := range blockBuffer { + err = cw.Put(ctx, blk.Cid().KeyString(), blk.RawData()) if err != nil { - // io.PipeWriter.CloseWithError always returns nil. - _ = w.CloseWithError(err) + _ = w.CloseWithError(fmt.Errorf("error writing car block: %w", err)) return nil } - for _, blk := range blockBuffer { - err = cw.Put(ctx, blk.Cid().KeyString(), blk.RawData()) - if err != nil { - _ = w.CloseWithError(fmt.Errorf("error writing car block: %w", err)) - return nil - } - } - blockBuffer = nil + } + blockBuffer = nil - err = walkGatewaySimpleSelector(ctx, lastCid, params, l, r) - if err != nil { - return err - } - return nil - }) + err = walkGatewaySimpleSelector(ctx, terminalBlk, params, l) + if err != nil { + return err + } + return nil }) - if ipldError != nil { - err = ipldError - } - if err != nil { - // io.PipeWriter.CloseWithError always returns nil. - _ = w.CloseWithError(err) - } + _ = w.CloseWithError(err) }() return gateway.ContentPathMetadata{ @@ -1197,6 +1270,16 @@ func checkRetryableError(e *error, fn func() error) error { return nil } + errNoSuchField, ok := err.(schema.ErrNoSuchField) + if ok { + // Convert into a more general error type so the gateway code can know what this means + // TODO: Have either a more generally usable error type system for IPLD errors (e.g. a base type indicating that data cannot exist) + // or at least have one that is specific to the gateway consumer and part of the Backend contract instead of this being implicit + err = datamodel.ErrNotExists{Segment: errNoSuchField.Field} + *e = err + return nil + } + err = errors.Unwrap(err) if err == nil { return initialErr From 8b2cdc797fd4b495aa38b49dc3180d07820117dd Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 6 Jul 2023 11:55:27 -0400 Subject: [PATCH 07/46] fix boxo go.mod --- go.mod | 4 +--- go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 51e821e..8d9c079 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/ipfs/boxo v0.11.1-0.20230803093408-3b83506a2ea5 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 - github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ipld-format v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-unixfsnode v1.7.1 @@ -78,6 +77,7 @@ require ( github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-blockservice v0.5.0 // indirect + github.com/ipfs/go-datastore v0.6.0 // indirect github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect @@ -186,5 +186,3 @@ require ( lukechampine.com/blake3 v1.1.7 // indirect nhooyr.io/websocket v1.8.7 // indirect ) - -replace github.com/ipfs/boxo => ../boxo diff --git a/go.sum b/go.sum index fa1c7e3..c839483 100644 --- a/go.sum +++ b/go.sum @@ -272,6 +272,8 @@ github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7P github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/boxo v0.11.1-0.20230803093408-3b83506a2ea5 h1:noxhUh8BG2Yxsq2+2+bIFtZeeUTLEKwD5yCKc0Fg+WI= +github.com/ipfs/boxo v0.11.1-0.20230803093408-3b83506a2ea5/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= From 56fce88b4bd8a4ae4e6d6dc4076c355259b1cab2 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 6 Jul 2023 13:40:48 -0400 Subject: [PATCH 08/46] fix passing range bytes in CAR requests, and reporting entity ranges --- lib/graph_gateway.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 4a09103..534605e 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -1077,7 +1077,11 @@ func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.Immutable } func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams) (gateway.ContentPathMetadata, io.ReadCloser, error) { - api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": string(params.Scope), "entityRanges": "0"}).Inc() + numRanges := "0" + if params.Range != nil { + numRanges = "1" + } + api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": string(params.Scope), "entityRanges": numRanges}).Inc() rootCid, err := getRootCid(path) if err != nil { return gateway.ContentPathMetadata{}, nil, err @@ -1089,7 +1093,7 @@ func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath, numBlocksSent := 0 var cw storage.WritableCar var blockBuffer []blocks.Block - err = api.fetchCAR(ctx, path, gateway.CarParams{Scope: params.Scope}, func(resource string, reader io.Reader) error { + err = api.fetchCAR(ctx, path, params, func(resource string, reader io.Reader) error { numBlocksThisCall := 0 gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) if err != nil { From 965e29db78645aee5a60279d2ab1bba6e61402c6 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 6 Jul 2023 14:21:22 -0400 Subject: [PATCH 09/46] chore: remove unused code --- go.mod | 2 +- lib/block_fetchers.go | 194 ------------------------------------------ lib/graph_gateway.go | 66 -------------- 3 files changed, 1 insertion(+), 261 deletions(-) delete mode 100644 lib/block_fetchers.go diff --git a/go.mod b/go.mod index 8d9c079..5790895 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,6 @@ require ( go.opentelemetry.io/otel/sdk v1.14.0 go.opentelemetry.io/otel/trace v1.14.0 go.uber.org/atomic v1.10.0 - go.uber.org/multierr v1.9.0 go.uber.org/zap v1.24.0 ) @@ -169,6 +168,7 @@ require ( go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/dig v1.15.0 // indirect go.uber.org/fx v1.18.2 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.6.0 // indirect golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb // indirect golang.org/x/mod v0.7.0 // indirect diff --git a/lib/block_fetchers.go b/lib/block_fetchers.go deleted file mode 100644 index 72ec622..0000000 --- a/lib/block_fetchers.go +++ /dev/null @@ -1,194 +0,0 @@ -package lib - -import ( - "context" - "github.com/ipfs/boxo/blockstore" - "github.com/ipfs/boxo/exchange" - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - format "github.com/ipfs/go-ipld-format" - "github.com/multiformats/go-multihash" - "go.uber.org/multierr" -) - -type inboundBlockExchange struct { - ps BlockPubSub -} - -func newInboundBlockExchange() *inboundBlockExchange { - return &inboundBlockExchange{ - ps: NewBlockPubSub(), - } -} - -func (i *inboundBlockExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - blk, more := <-i.ps.Subscribe(ctx, c.Hash()) - if err := ctx.Err(); err != nil { - return nil, err - } - if !more { - return nil, format.ErrNotFound{Cid: c} - } - return blk, nil -} - -func (i *inboundBlockExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { - mhMap := make(map[string]struct{}) - for _, c := range cids { - mhMap[string(c.Hash())] = struct{}{} - } - mhs := make([]multihash.Multihash, 0, len(mhMap)) - for k := range mhMap { - mhs = append(mhs, multihash.Multihash(k)) - } - return i.ps.Subscribe(ctx, mhs...), nil -} - -func (i *inboundBlockExchange) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { - // TODO: handle context cancellation and/or blockage here - i.ps.Publish(blocks...) - return nil -} - -func (i *inboundBlockExchange) Close() error { - i.ps.Shutdown() - return nil -} - -var _ exchange.Interface = (*inboundBlockExchange)(nil) - -type handoffExchange struct { - startingExchange, followupExchange exchange.Interface - bstore blockstore.Blockstore - handoffCh <-chan struct{} - metrics *GraphGatewayMetrics -} - -func (f *handoffExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - blkCh, err := f.startingExchange.GetBlocks(ctx, []cid.Cid{c}) - if err != nil { - return nil, err - } - blk, ok := <-blkCh - if ok { - return blk, nil - } - - select { - case <-f.handoffCh: - graphLog.Debugw("switching to backup block fetcher", "cid", c) - f.metrics.blockRecoveryAttemptMetric.Inc() - return f.followupExchange.GetBlock(ctx, c) - case <-ctx.Done(): - return nil, ctx.Err() - } -} - -func (f *handoffExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { - blkCh, err := f.startingExchange.GetBlocks(ctx, cids) - if err != nil { - return nil, err - } - - retCh := make(chan blocks.Block) - - go func() { - cs := cid.NewSet() - for cs.Len() < len(cids) { - blk, ok := <-blkCh - if !ok { - break - } - select { - case retCh <- blk: - cs.Add(blk.Cid()) - case <-ctx.Done(): - } - } - - for cs.Len() < len(cids) { - select { - case <-ctx.Done(): - return - case <-f.handoffCh: - var newCidArr []cid.Cid - for _, c := range cids { - if !cs.Has(c) { - blk, _ := f.bstore.Get(ctx, c) - if blk != nil { - select { - case retCh <- blk: - cs.Add(blk.Cid()) - case <-ctx.Done(): - return - } - } else { - newCidArr = append(newCidArr, c) - } - } - } - - if len(newCidArr) == 0 { - return - } - - graphLog.Debugw("needed to use use a backup fetcher for cids", "cids", newCidArr) - f.metrics.blockRecoveryAttemptMetric.Add(float64(len(newCidArr))) - fch, err := f.followupExchange.GetBlocks(ctx, newCidArr) - if err != nil { - graphLog.Errorw("error getting blocks from followupExchange", "error", err) - return - } - for cs.Len() < len(cids) { - blk, ok := <-fch - if !ok { - return - } - select { - case retCh <- blk: - cs.Add(blk.Cid()) - case <-ctx.Done(): - return - } - } - } - } - }() - return retCh, nil -} - -func (f *handoffExchange) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { - err1 := f.startingExchange.NotifyNewBlocks(ctx, blocks...) - err2 := f.followupExchange.NotifyNewBlocks(ctx, blocks...) - return multierr.Combine(err1, err2) -} - -func (f *handoffExchange) Close() error { - err1 := f.startingExchange.Close() - err2 := f.followupExchange.Close() - return multierr.Combine(err1, err2) -} - -var _ exchange.Interface = (*handoffExchange)(nil) - -type blockFetcherExchWrapper struct { - f exchange.Fetcher -} - -func (b *blockFetcherExchWrapper) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - return b.f.GetBlock(ctx, c) -} - -func (b *blockFetcherExchWrapper) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { - return b.f.GetBlocks(ctx, cids) -} - -func (b *blockFetcherExchWrapper) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { - return nil -} - -func (b *blockFetcherExchWrapper) Close() error { - return nil -} - -var _ exchange.Interface = (*blockFetcherExchWrapper)(nil) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 534605e..e1a8e84 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -32,7 +32,6 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" - golog "github.com/ipfs/go-log/v2" "github.com/ipfs/go-unixfsnode" ufsData "github.com/ipfs/go-unixfsnode/data" carv2 "github.com/ipld/go-car/v2" @@ -50,8 +49,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -var graphLog = golog.Logger("backend/graph") - const GetBlockTimeout = time.Second * 60 // type DataCallback = func(resource string, reader io.Reader) error @@ -95,18 +92,6 @@ func WithBlockstore(bs blockstore.Blockstore) GraphGatewayOption { type GraphGatewayOption func(gwOptions *gwOptions) error -type Notifier interface { - NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error -} - -// notifiersForRootCid is used for reducing lock contention by only notifying -// exchanges related to the same content root CID -type notifiersForRootCid struct { - lk sync.RWMutex - deleted int8 - notifiers []Notifier -} - type GraphGateway struct { fetcher CarFetcher blockFetcher exchange.Fetcher @@ -273,57 +258,6 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { } } -func (api *GraphGateway) getRootOfPath(path string) string { - pth, err := ipfspath.ParsePath(path) - if err != nil { - return path - } - if pth.IsJustAKey() { - return pth.Segments()[0] - } else { - return pth.Segments()[1] - } -} - -type fileCloseWrapper struct { - files.File - closeFn func() -} - -func (w *fileCloseWrapper) Close() error { - w.closeFn() - return w.File.Close() -} - -type dirCloseWrapper struct { - files.Directory - closeFn func() -} - -func (w *dirCloseWrapper) Close() error { - w.closeFn() - return w.Directory.Close() -} - -func wrapNodeWithClose[T files.Node](node T, closeFn func()) (T, error) { - var genericNode files.Node = node - switch n := genericNode.(type) { - case *files.Symlink: - closeFn() - return node, nil - case files.File: - var f files.File = &fileCloseWrapper{n, closeFn} - return f.(T), nil - case files.Directory: - var d files.Directory = &dirCloseWrapper{n, closeFn} - return d.(T), nil - default: - closeFn() - var zeroType T - return zeroType, fmt.Errorf("unsupported node type") - } -} - func (api *GraphGateway) fetchCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams, cb DataCallback) error { escapedPath := url.PathEscape(path.String()[1:]) paramsBuilder := strings.Builder{} From c2fb61a52e73fa3ae85152962df61abd5eccabac Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 12 Jul 2023 22:51:50 -0400 Subject: [PATCH 10/46] feat: add support for backpressured tar files --- lib/files.go | 526 +++++++++++++++++++++++++++++++++++++++++++ lib/graph_gateway.go | 177 +++++++++++++-- 2 files changed, 688 insertions(+), 15 deletions(-) create mode 100644 lib/files.go diff --git a/lib/files.go b/lib/files.go new file mode 100644 index 0000000..ffac463 --- /dev/null +++ b/lib/files.go @@ -0,0 +1,526 @@ +package lib + +import ( + "bytes" + "context" + "fmt" + "github.com/ipfs/boxo/files" + "github.com/ipfs/boxo/gateway" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-unixfsnode" + ufsData "github.com/ipfs/go-unixfsnode/data" + "github.com/ipfs/go-unixfsnode/hamt" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/schema" + "github.com/multiformats/go-multicodec" + "io" +) + +type AwaitCloser interface { + AwaitClose() <-chan error +} + +type backpressuredFile struct { + size int64 + f io.ReadSeeker + getLsys lsysGetter + + ctx context.Context + fileCid cid.Cid + byteRange gateway.DagByteRange + retErr error + + closed chan error +} + +func (b *backpressuredFile) AwaitClose() <-chan error { + return b.closed +} + +func (b *backpressuredFile) Close() error { + close(b.closed) + return nil +} + +func (b *backpressuredFile) Size() (int64, error) { + return b.size, nil +} + +func (b *backpressuredFile) Read(p []byte) (n int, err error) { + if b.retErr == nil { + n, err = b.f.Read(p) + if err == nil || err == io.EOF { + return n, err + } + + if n > 0 { + b.retErr = err + return n, nil + } + } else { + err = b.retErr + } + + from, err := b.f.Seek(io.SeekCurrent, 0) + if err != nil { + return 0, err + } + nd, err := tv(b.ctx, b.fileCid, nil, nil, gateway.CarParams{Scope: gateway.DagScopeEntity, Range: &gateway.DagByteRange{From: from, To: b.byteRange.To}}, b.getLsys) + if err != nil { + return 0, err + } + + f, ok := nd.(files.File) + if !ok { + return 0, fmt.Errorf("not a file, should be unreachable") + } + + b.f = f + return b.Read(p) +} + +func (b *backpressuredFile) Seek(offset int64, whence int) (int64, error) { + //TODO implement me + panic("implement me") +} + +var _ files.File = (*backpressuredFile)(nil) +var _ AwaitCloser = (*backpressuredFile)(nil) + +type singleUseDirectory struct { + dirIter files.DirIterator + closed chan error +} + +func (b *singleUseDirectory) AwaitClose() <-chan error { + return b.closed +} + +func (b *singleUseDirectory) Close() error { + close(b.closed) + return nil +} + +func (b *singleUseDirectory) Size() (int64, error) { + //TODO implement me + panic("implement me") +} + +func (b *singleUseDirectory) Entries() files.DirIterator { + return b.dirIter +} + +var _ files.Directory = (*singleUseDirectory)(nil) +var _ AwaitCloser = (*singleUseDirectory)(nil) + +type backpressuredFlatDirIter struct { + linksItr *dagpb.PBLinks__Itr + lsys *ipld.LinkSystem + getLsys lsysGetter + ctx context.Context + + curName string + curFile files.Node + + err error +} + +func (it *backpressuredFlatDirIter) Name() string { + return it.curName +} + +func (it *backpressuredFlatDirIter) Node() files.Node { + return it.curFile +} + +func (it *backpressuredFlatDirIter) Next() bool { + if it.err != nil { + return false + } + + iter := it.linksItr + if iter.Done() { + return false + } + + _, v := iter.Next() + c := v.Hash.Link().(cidlink.Link).Cid + var name string + if v.Name.Exists() { + name = v.Name.Must().String() + } + + var nd files.Node + var err error + params := gateway.CarParams{Scope: gateway.DagScopeAll} + for { + if it.ctx.Err() != nil { + it.err = it.ctx.Err() + return false + } + if err != nil { + it.lsys, err = it.getLsys(it.ctx, c, params) + continue + } + nd, err = tv(it.ctx, c, nil, it.lsys, params, it.getLsys) + if err != nil { + if err := it.ctx.Err(); err == nil { + retry, processedErr := isRetryableError(err) + if retry { + err = processedErr + continue + } + it.err = processedErr + return false + } + } + break + } + + it.curName = name + it.curFile = nd + return true + + /* + Load link, figure out what it is: + 1. If it's a symlink just stash it + 2. If it's a raw block stash it? + 3. If it's metadata then skip it + 4. If it's a file return a wrapper around multireadcloser that waits for new lsys' to show up. There's got to be a way to feedback through the top requests for new lsys' along with the request we're waiting on for the retry + 5. If it's a flat directory stash it (will need to carry the feedback layer for its children) + 6. If it's a HAMT return an iterator that for "next" sends a request and waits for an lsys if there's a problem. Create the next object and return it passing through the request + response mechanisms for continuations + */ +} + +func (it *backpressuredFlatDirIter) Err() error { + return it.err +} + +var _ files.DirIterator = (*backpressuredFlatDirIter)(nil) + +type backpressuredHAMTDirIter struct { + linksItr ipld.MapIterator + dirCid cid.Cid + + lsys *ipld.LinkSystem + getLsys lsysGetter + ctx context.Context + + curName string + curFile files.Node + curProcessed int + + err error +} + +func (it *backpressuredHAMTDirIter) Name() string { + return it.curName +} + +func (it *backpressuredHAMTDirIter) Node() files.Node { + return it.curFile +} + +func (it *backpressuredHAMTDirIter) Next() bool { + if it.err != nil { + return false + } + + iter := it.linksItr + if iter.Done() { + return false + } + + /* + Since there is no way to make a graph request for part of a HAMT during errors we can either fill in the HAMT with + block requests, or we can re-request the HAMT and skip over the parts we already have. + + Here we choose the latter, however in the event of a re-request we request the entity rather than the entire DAG as + a compromise between more requests and over-fetching data. + */ + + var err error + for { + if it.ctx.Err() != nil { + it.err = it.ctx.Err() + return false + } + + retry, processedErr := isRetryableError(err) + if !retry { + it.err = processedErr + return false + } + + var nd ipld.Node + if err != nil { + var lsys *ipld.LinkSystem + lsys, err = it.getLsys(it.ctx, it.dirCid, gateway.CarParams{Scope: gateway.DagScopeEntity}) + if err != nil { + continue + } + + _, pbn, fieldData, _, _, ufsBaseErr := loadUnixFSBase(it.ctx, it.dirCid, nil, lsys) + if err != nil { + err = ufsBaseErr + continue + } + + nd, err = hamt.NewUnixFSHAMTShard(it.ctx, pbn, fieldData, lsys) + if err != nil { + err = fmt.Errorf("could not reify sharded directory: %w", err) + continue + } + + iter = nd.MapIterator() + for i := 0; i < it.curProcessed; i++ { + _, _, err = iter.Next() + if err != nil { + continue + } + } + + it.linksItr = nd.MapIterator() + iter = it.linksItr + } + + var k, v ipld.Node + k, v, err = iter.Next() + if err != nil { + retry, processedErr = isRetryableError(err) + if retry { + err = processedErr + continue + } + it.err = processedErr + return false + } + + var name string + name, err = k.AsString() + if err != nil { + it.err = err + return false + } + var lnk ipld.Link + lnk, err = v.AsLink() + if err != nil { + it.err = err + return false + } + + cl, ok := lnk.(cidlink.Link) + if !ok { + it.err = fmt.Errorf("link not a cidlink") + return false + } + + c := cl.Cid + params := gateway.CarParams{Scope: gateway.DagScopeAll} + var childNd files.Node + for { + if it.ctx.Err() != nil { + it.err = it.ctx.Err() + return false + } + + if err != nil { + retry, processedErr = isRetryableError(err) + if !retry { + it.err = processedErr + return false + } + + it.lsys, err = it.getLsys(it.ctx, c, params) + continue + } + + childNd, err = tv(it.ctx, c, nil, it.lsys, params, it.getLsys) + if err != nil { + continue + } + break + } + + it.curName = name + it.curFile = childNd + it.curProcessed++ + break + } + + return true + + /* + Load link, figure out what it is: + 1. If it's a symlink just stash it + 2. If it's a raw block stash it? + 3. If it's metadata then skip it + 4. If it's a file return a wrapper around multireadcloser that waits for new lsys' to show up. There's got to be a way to feedback through the top requests for new lsys' along with the request we're waiting on for the retry + 5. If it's a flat directory stash it (will need to carry the feedback layer for its children) + 6. If it's a HAMT return an iterator that for "next" sends a request and waits for an lsys if there's a problem. Create the next object and return it passing through the request + response mechanisms for continuations + */ +} + +func (it *backpressuredHAMTDirIter) Err() error { + return it.err +} + +var _ files.DirIterator = (*backpressuredHAMTDirIter)(nil) + +/* +1. Run traversal to get the top-level response +2. Response can do a callback for another response +*/ + +type lsysGetter = func(ctx context.Context, c cid.Cid, params gateway.CarParams) (*ipld.LinkSystem, error) + +func loadUnixFSBase(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld.LinkSystem) ([]byte, dagpb.PBNode, ufsData.UnixFSData, int64, []byte, error) { + lctx := ipld.LinkContext{Ctx: ctx} + pathTerminalCidLink := cidlink.Link{Cid: c} + + var blockData []byte + var err error + + if blk != nil { + blockData = blk.RawData() + } else { + blockData, err = lsys.LoadRaw(lctx, pathTerminalCidLink) + if err != nil { + return nil, nil, nil, 0, nil, err + } + } + + if c.Type() == uint64(multicodec.Raw) { + return blockData, nil, nil, 0, nil, nil + } + + // decode the terminal block into a node + pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + + np, err := pc(pathTerminalCidLink, lctx) + if err != nil { + return nil, nil, nil, 0, nil, err + } + + decoder, err := lsys.DecoderChooser(pathTerminalCidLink) + if err != nil { + return nil, nil, nil, 0, nil, err + } + nb := np.NewBuilder() + if err := decoder(nb, bytes.NewReader(blockData)); err != nil { + return nil, nil, nil, 0, nil, err + } + lastCidNode := nb.Build() + + if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { + // If it's not valid dag-pb then we're done + return nil, nil, nil, 0, nil, errNotUnixFS + } else if !pbn.FieldData().Exists() { + // If it's not valid UnixFS then we're done + return nil, nil, nil, 0, nil, errNotUnixFS + } else if unixfsFieldData, decodeErr := ufsData.DecodeUnixFSData(pbn.Data.Must().Bytes()); decodeErr != nil { + return nil, nil, nil, 0, nil, errNotUnixFS + } else { + switch fieldNum := unixfsFieldData.FieldDataType().Int(); fieldNum { + case ufsData.Data_Symlink, ufsData.Data_Metadata, ufsData.Data_Raw, ufsData.Data_File, ufsData.Data_Directory, ufsData.Data_HAMTShard: + return nil, pbn, unixfsFieldData, fieldNum, pbn.FieldData().Must().Bytes(), nil + default: + return nil, nil, nil, 0, nil, errNotUnixFS + } + } +} + +func tv(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld.LinkSystem, params gateway.CarParams, getLsys lsysGetter) (files.Node, error) { + var err error + if lsys == nil { + lsys, err = getLsys(ctx, c, params) + if err != nil { + return nil, err + } + } + + lctx := ipld.LinkContext{Ctx: ctx} + blockData, pbn, _, fieldNum, fieldDataBytes, err := loadUnixFSBase(ctx, c, blk, lsys) + if err != nil { + return nil, err + } + + if c.Type() == uint64(multicodec.Raw) { + return files.NewBytesFile(blockData), nil + } + + switch fieldNum { + case ufsData.Data_Symlink: + lnkTarget := string(fieldDataBytes) + f := files.NewLinkFile(lnkTarget, nil) + return f, nil + case ufsData.Data_Metadata: + return nil, fmt.Errorf("UnixFS Metadata unsupported") + case ufsData.Data_HAMTShard, ufsData.Data_Directory: + switch fieldNum { + case ufsData.Data_Directory: + d := &singleUseDirectory{&backpressuredFlatDirIter{ + ctx: ctx, + linksItr: pbn.Links.Iterator(), + lsys: lsys, + getLsys: getLsys, + }, make(chan error)} + return d, nil + case ufsData.Data_HAMTShard: + dirNd, err := unixfsnode.Reify(lctx, pbn, lsys) + if err != nil { + return nil, fmt.Errorf("could not reify sharded directory: %w", err) + } + + d := &singleUseDirectory{ + &backpressuredHAMTDirIter{ + linksItr: dirNd.MapIterator(), + dirCid: c, + lsys: lsys, + getLsys: getLsys, + ctx: ctx, + }, make(chan error), + } + return d, nil + default: + return nil, fmt.Errorf("not a basic or HAMT directory: should be unreachable") + } + case ufsData.Data_Raw, ufsData.Data_File: + nd, err := unixfsnode.Reify(lctx, pbn, lsys) + if err != nil { + return nil, err + } + + fnd, ok := nd.(datamodel.LargeBytesNode) + if !ok { + return nil, fmt.Errorf("could not process file since it did not present as large bytes") + } + f, err := fnd.AsLargeBytes() + if err != nil { + return nil, err + } + + fileSize, err := f.Seek(0, io.SeekEnd) + if err != nil { + return nil, fmt.Errorf("unable to get UnixFS file size: %w", err) + } + _, err = f.Seek(0, io.SeekStart) + if err != nil { + return nil, fmt.Errorf("unable to get reset UnixFS file reader: %w", err) + } + + return &backpressuredFile{ctx: ctx, size: fileSize, f: f, getLsys: getLsys, closed: make(chan error)}, nil + default: + return nil, fmt.Errorf("unknown UnixFS field type") + } +} diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index e1a8e84..e47fb49 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -55,6 +55,9 @@ const GetBlockTimeout = time.Second * 60 // TODO: Don't use a caboose type, perhaps ask them to use a type alias instead of a type type DataCallback = caboose.DataCallback +// TODO: Don't use a caboose type +type ErrPartialResponse = caboose.ErrPartialResponse + type CarFetcher interface { Fetch(ctx context.Context, path string, cb DataCallback) error } @@ -258,8 +261,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { } } -func (api *GraphGateway) fetchCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams, cb DataCallback) error { - escapedPath := url.PathEscape(path.String()[1:]) +func paramsToString(params gateway.CarParams) string { paramsBuilder := strings.Builder{} paramsBuilder.WriteString("dag-scope=") paramsBuilder.WriteString(string(params.Scope)) @@ -273,7 +275,13 @@ func (api *GraphGateway) fetchCAR(ctx context.Context, path gateway.ImmutablePat paramsBuilder.WriteString("*") } } - urlWithoutHost := fmt.Sprintf("/%s?%s", escapedPath, paramsBuilder.String()) + return paramsBuilder.String() +} + +func (api *GraphGateway) fetchCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams, cb DataCallback) error { + escapedPath := url.PathEscape(path.String()[1:]) + paramsStr := paramsToString(params) + urlWithoutHost := fmt.Sprintf("/%s?%s", escapedPath, paramsStr) var ipldError error fetchErr := api.fetcher.Fetch(ctx, urlWithoutHost, func(resource string, reader io.Reader) error { @@ -407,6 +415,8 @@ func contentMetadataFromRootsAndRemainder(p ipfspath.Path, pathRoots []cid.Cid, return md } +var errNotUnixFS = fmt.Errorf("data was not unixfs") + func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, byteRanges ...gateway.ByteRange) (gateway.ContentPathMetadata, *gateway.GetResponse, error) { rangeCount := len(byteRanges) api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "entity", "entityRanges": strconv.Itoa(rangeCount)}).Inc() @@ -442,7 +452,6 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by } terminalPathElementCh := make(chan terminalPathType, 1) - errNotUnixFS := fmt.Errorf("data was not unixfs") var terminalFile *multiReadCloser var terminalDir chan unixfs.LinkResult @@ -791,7 +800,140 @@ func (r *multiReadCloser) Close() error { var _ io.ReadCloser = (*multiReadCloser)(nil) func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { - panic("not implemented") + api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "all", "entityRanges": "0"}).Inc() + p := ipfspath.FromString(path.String()) + + type terminalPathType struct { + resp files.Node + err error + md gateway.ContentPathMetadata + } + + terminalPathElementCh := make(chan terminalPathType, 1) + + go func() { + cctx, cancel := context.WithCancel(ctx) + defer cancel() + + hasSentAsyncData := false + var closeCh <-chan error + + type nextReq struct { + c cid.Cid + params gateway.CarParams + } + + sendRequest := make(chan nextReq, 1) + sendResponse := make(chan *ipld.LinkSystem, 1) + getLsys := func(ctx context.Context, c cid.Cid, params gateway.CarParams) (*ipld.LinkSystem, error) { + select { + case sendRequest <- nextReq{c: c, params: params}: + case <-ctx.Done(): + return nil, ctx.Err() + } + + select { + case lsys := <-sendResponse: + return lsys, nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + + err := api.fetchCAR(cctx, path, gateway.CarParams{Scope: gateway.DagScopeAll}, func(resource string, reader io.Reader) error { + gb, err := carToLinearBlockGetter(cctx, reader, api.metrics) + if err != nil { + return err + } + + lsys := getLinksystem(gb) + // First resolve the path since we always need to. + pathRootCids, terminalCid, remainder, terminalBlk, err := api.resolvePathWithRootsAndBlock(cctx, p, lsys) + if err != nil { + return err + } + md := contentMetadataFromRootsAndRemainder(p, pathRootCids, terminalCid, remainder) + + if len(remainder) > 0 { + terminalPathElementCh <- terminalPathType{err: errNotUnixFS} + return nil + } + + if hasSentAsyncData { + select { + case sendResponse <- lsys: + case <-ctx.Done(): + return ctx.Err() + } + } + + nd, err := tv(cctx, terminalCid, terminalBlk, lsys, gateway.CarParams{Scope: gateway.DagScopeAll}, getLsys) + if err != nil { + return err + } + + ndAc, ok := nd.(AwaitCloser) + if !ok { + terminalPathElementCh <- terminalPathType{ + resp: nd, + md: md, + } + return nil + } + + hasSentAsyncData = true + terminalPathElementCh <- terminalPathType{ + resp: nd, + md: md, + } + + closeCh = ndAc.AwaitClose() + select { + case closeErr := <-closeCh: + return closeErr + case req := <-sendRequest: + requestStr := fmt.Sprintf("/ipfs/%s?%s", req.c.String(), paramsToString(req.params)) + return &ErrPartialResponse{StillNeed: []string{requestStr}} + case <-cctx.Done(): + return cctx.Err() + } + }) + + if !hasSentAsyncData && err != nil { + terminalPathElementCh <- terminalPathType{err: err} + return + } + + if err != nil { + for { + select { + case <-closeCh: + return + case <-sendRequest: + lsys := getLinksystem(func(ctx context.Context, cid cid.Cid) (blocks.Block, error) { + return nil, fmt.Errorf("done trying to fetch CAR data: %w", format.ErrNotFound{Cid: cid}) + }) + select { + case sendResponse <- lsys: + case <-cctx.Done(): + return + } + case <-cctx.Done(): + return + } + } + } + }() + + select { + case t := <-terminalPathElementCh: + if t.err != nil { + return gateway.ContentPathMetadata{}, nil, t.err + } + return t.md, t.resp, nil + case <-ctx.Done(): + return gateway.ContentPathMetadata{}, nil, ctx.Err() + } } func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { @@ -1182,8 +1324,17 @@ var _ gateway.IPFSBackend = (*GraphGateway)(nil) func checkRetryableError(e *error, fn func() error) error { err := fn() + retry, processedErr := isRetryableError(err) + if retry { + return processedErr + } + *e = processedErr + return nil +} + +func isRetryableError(err error) (bool, error) { if format.IsNotFound(err) { - return err + return true, err } initialErr := err @@ -1192,20 +1343,17 @@ func checkRetryableError(e *error, fn func() error) error { for { _, ok := err.(resolver.ErrNoLink) if ok { - *e = err - return nil + return false, err } _, ok = err.(datamodel.ErrWrongKind) if ok { - *e = err - return nil + return false, err } _, ok = err.(datamodel.ErrNotExists) if ok { - *e = err - return nil + return false, err } errNoSuchField, ok := err.(schema.ErrNoSuchField) @@ -1214,13 +1362,12 @@ func checkRetryableError(e *error, fn func() error) error { // TODO: Have either a more generally usable error type system for IPLD errors (e.g. a base type indicating that data cannot exist) // or at least have one that is specific to the gateway consumer and part of the Backend contract instead of this being implicit err = datamodel.ErrNotExists{Segment: errNoSuchField.Field} - *e = err - return nil + return false, err } err = errors.Unwrap(err) if err == nil { - return initialErr + return true, initialErr } } } From 0691af94085c2499389128da368ced0187fe1d4a Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 12 Jul 2023 23:26:05 -0400 Subject: [PATCH 11/46] chore: removed unused testing tools --- test-backend/handlers.go | 627 --------------------------------------- test-backend/main.go | 147 --------- test-backend/version.go | 39 --- 3 files changed, 813 deletions(-) delete mode 100644 test-backend/handlers.go delete mode 100644 test-backend/main.go delete mode 100644 test-backend/version.go diff --git a/test-backend/handlers.go b/test-backend/handlers.go deleted file mode 100644 index 231583e..0000000 --- a/test-backend/handlers.go +++ /dev/null @@ -1,627 +0,0 @@ -package main - -import ( - "bytes" - "context" - "fmt" - "io" - "net/http" - "net/url" - "runtime/debug" - "strconv" - "strings" - "sync" - "time" - - "github.com/ipfs/boxo/fetcher" - "github.com/ipfs/boxo/files" - "github.com/ipfs/boxo/gateway" - "github.com/ipfs/boxo/ipld/merkledag" - unixfile "github.com/ipfs/boxo/ipld/unixfs/file" - "github.com/ipfs/boxo/path" - "github.com/ipfs/boxo/path/resolver" - blocks "github.com/ipfs/go-block-format" - format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-unixfsnode" - "github.com/ipld/go-car" - "github.com/ipld/go-car/util" - dagpb "github.com/ipld/go-codec-dagpb" - "github.com/ipld/go-ipld-prime" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/node/basicnode" - "github.com/ipld/go-ipld-prime/schema" - "github.com/ipld/go-ipld-prime/traversal" - "github.com/ipld/go-ipld-prime/traversal/selector" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - - "github.com/ipfs/boxo/blockservice" - ipath "github.com/ipfs/boxo/coreiface/path" - "github.com/ipfs/go-cid" -) - -func makeMetricsHandler(port int) (*http.Server, error) { - mux := http.NewServeMux() - - gatherers := prometheus.Gatherers{ - prometheus.DefaultGatherer, - } - options := promhttp.HandlerOpts{} - mux.Handle("/debug/metrics/prometheus", promhttp.HandlerFor(gatherers, options)) - - return &http.Server{ - Handler: mux, - Addr: ":" + strconv.Itoa(port), - }, nil -} - -func withRequestLogger(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - goLog.Infow(r.Method, "url", r.URL, "host", r.Host) - // TODO: if debug is enabled, show more? goLog.Infow("request received", "url", r.URL, "host", r.Host, "method", r.Method, "ua", r.UserAgent(), "referer", r.Referer()) - next.ServeHTTP(w, r) - }) -} - -var noModtime = time.Unix(0, 0) - -func makeGatewayCARHandler(bsrv blockservice.BlockService, port int) (*http.Server, error) { - mux := http.NewServeMux() - mux.HandleFunc("/ipfs/", func(w http.ResponseWriter, r *http.Request) { - // the hour is a hard fallback, we don't expect it to happen, but just in case - ctx, cancel := context.WithTimeout(r.Context(), time.Hour) - defer cancel() - r = r.WithContext(ctx) - - defer func() { - if r := recover(); r != nil { - goLog.Error("A panic occurred in the gateway handler!") - goLog.Error(r) - debug.PrintStack() - } - }() - - if r.Method != http.MethodGet { - w.Header().Add("Allow", http.MethodGet) - - errmsg := "Method " + r.Method + " not allowed" - http.Error(w, errmsg, http.StatusMethodNotAllowed) - return - } - - isCar := false - if formatParam := r.URL.Query().Get("format"); formatParam != "" { - isCar = formatParam == "car" - if !isCar { - http.Error(w, "only car format supported", http.StatusBadRequest) - return - } - } else { - for _, header := range r.Header.Values("Accept") { - for _, value := range strings.Split(header, ",") { - accept := strings.TrimSpace(value) - if strings.HasPrefix(accept, "application/vnd.ipld.car") { - isCar = true - break - } - } - } - } - if !isCar { - http.Error(w, "only car format supported", http.StatusBadRequest) - return - } - - contentPath := ipath.New(r.URL.Path) - if contentPath.Mutable() { - http.Error(w, "only immutable block requests supported", http.StatusBadRequest) - return - } else if contentPath.Namespace() != "ipfs" { - http.Error(w, "only the ipfs names is supported", http.StatusBadRequest) - return - } - - carStream, err := simpleSelectorToCar(ctx, bsrv, contentPath.String(), r.URL.Query()) - if err != nil { - http.Error(w, "only the ipfs names is supported", http.StatusBadRequest) - return - } - - const immutableCacheControl = "public, max-age=29030400, immutable" - // immutable! CACHE ALL THE THINGS, FOREVER! wolololol - w.Header().Set("Cache-Control", immutableCacheControl) - w.Header().Set("Server", userAgent) - - // Set modtime to 'zero time' to disable Last-Modified header (superseded by Cache-Control) - - io.Copy(w, carStream) - }) - - // Creates metrics handler for total response size. Matches the same metrics - // from Kubo: - // https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152 - sum := prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Namespace: "ipfs", - Subsystem: "http", - Name: "response_size_bytes", - Help: "The HTTP response sizes in bytes.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, nil) - err := prometheus.Register(sum) - if err != nil { - return nil, err - } - - // Construct the HTTP handler for the gateway. - handler := promhttp.InstrumentHandlerResponseSize(sum, mux) - - // Add logging - handler = withRequestLogger(handler) - - return &http.Server{ - Handler: handler, - Addr: ":" + strconv.Itoa(port), - }, nil -} - -func simpleSelectorToCar(ctx context.Context, bsrv blockservice.BlockService, p string, params url.Values) (io.ReadCloser, error) { - pathSegs := strings.Split(p, "/") - if len(pathSegs) < 3 || !(pathSegs[0] == "" && pathSegs[1] == "ipfs") { - return nil, fmt.Errorf("invalid path") - } - pathSegs = pathSegs[2:] - rootCidStr := pathSegs[0] - rootCid, err := cid.Decode(rootCidStr) - if err != nil { - return nil, err - } - - ipfspath, err := path.ParsePath(p) - if err != nil { - return nil, err - } - - r, w := io.Pipe() - - rangeStr, hasRange := params.Get("bytes"), params.Has("bytes") - depthStr, hasDepth := params.Get("depth"), params.Has("depth") - - if hasDepth && !(depthStr == "0" || depthStr == "1" || depthStr == "all") { - return nil, fmt.Errorf("depth type: %s not supported", depthStr) - } - var getRange *gateway.ByteRange - if hasRange { - getRange, err = rangeStrToByteRange(rangeStr) - if err != nil { - return nil, err - } - } - - go func() { - defer w.Close() - - // Setup header for the output car - err = car.WriteHeader(&car.CarHeader{ - Roots: []cid.Cid{rootCid}, - Version: 1, - }, w) - if err != nil { - goLog.Error(fmt.Errorf("writing car header: %w", err)) - } - - blockGetter := merkledag.NewDAGService(bsrv).Session(ctx) - blockGetter = &nodeGetterToCarExporer{ - ng: blockGetter, - w: w, - mhSet: make(map[string]struct{}), - } - dsrv := merkledag.NewReadOnlyDagService(blockGetter) - - // Setup the UnixFS resolver. - f := newNodeGetterFetcherSingleUseFactory(ctx, blockGetter) - pathResolver := resolver.NewBasicResolver(f) - - lastCid, remainder, err := pathResolver.ResolveToLastNode(ctx, ipfspath) - if err != nil { - goLog.Error(err) - return - } - - if hasDepth && depthStr == "0" { - return - } - - lastCidNode, err := dsrv.Get(ctx, lastCid) - if err != nil { - goLog.Error(err) - return - } - - ufsNode, err := unixfile.NewUnixfsFile(ctx, dsrv, lastCidNode) - if err != nil { - // It's not UnixFS - - // If it's all fetch the graph recursively - if depthStr == "all" { - if err := merkledag.FetchGraph(ctx, lastCid, dsrv); err != nil { - goLog.Error(err) - } - return - } - - //if not then either this is an error (which we can't report) or this is the last block for us to return - return - } - if f, ok := ufsNode.(files.File); ok { - if len(remainder) > 0 { - // this is an error, so we're done - return - } - - if !hasRange { - nw, err := io.Copy(io.Discard, f) - goLog.Debugf("nwritten %d", nw) - if err != nil { - goLog.Error(err) - } - return - } - - // TODO: testing + check off by one errors - var numToRead int64 - if *getRange.To < 0 { - size, err := f.Seek(0, io.SeekEnd) - if err != nil { - return - } - numToRead = (size - *getRange.To) - int64(getRange.From) - } else { - numToRead = int64(getRange.From) - *getRange.To - } - - if _, err := f.Seek(int64(getRange.From), io.SeekStart); err != nil { - return - } - _, _ = io.CopyN(io.Discard, f, numToRead) - return - } else if d, ok := ufsNode.(files.Directory); ok { - if depthStr == "1" { - iter := d.Entries() - for iter.Next() { - } - return - } - if depthStr == "all" { - // TODO: being lazy here - w, err := files.NewTarWriter(io.Discard) - if err != nil { - goLog.Error(fmt.Errorf("could not create tar write %w", err)) - return - } - if err := w.WriteFile(d, "tmp"); err != nil { - goLog.Error(err) - return - } - return - } - } else { - return - } - }() - return r, nil -} - -type nodeGetterToCarExporer struct { - ng format.NodeGetter - w io.Writer - - lk sync.RWMutex - mhSet map[string]struct{} -} - -func (n *nodeGetterToCarExporer) Get(ctx context.Context, c cid.Cid) (format.Node, error) { - nd, err := n.ng.Get(ctx, c) - if err != nil { - return nil, err - } - - if err := n.trySendBlock(nd); err != nil { - return nil, err - } - - return nd, nil -} - -func (n *nodeGetterToCarExporer) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption { - ndCh := n.ng.GetMany(ctx, cids) - outCh := make(chan *format.NodeOption) - go func() { - defer close(outCh) - for nd := range ndCh { - if nd.Err == nil { - if err := n.trySendBlock(nd.Node); err != nil { - select { - case outCh <- &format.NodeOption{Err: err}: - case <-ctx.Done(): - } - return - } - select { - case outCh <- nd: - case <-ctx.Done(): - } - } - } - }() - return outCh -} - -func (n *nodeGetterToCarExporer) trySendBlock(block blocks.Block) error { - h := string(block.Cid().Hash()) - n.lk.RLock() - _, found := n.mhSet[h] - n.lk.RUnlock() - if !found { - doSend := false - n.lk.Lock() - _, found := n.mhSet[h] - if !found { - doSend = true - n.mhSet[h] = struct{}{} - } - n.lk.Unlock() - if doSend { - err := util.LdWrite(n.w, block.Cid().Bytes(), block.RawData()) // write to the output car - if err != nil { - return fmt.Errorf("writing to output car: %w", err) - } - } - } - return nil -} - -var _ format.NodeGetter = (*nodeGetterToCarExporer)(nil) - -type nodeGetterFetcherSingleUseFactory struct { - linkSystem ipld.LinkSystem - protoChooser traversal.LinkTargetNodePrototypeChooser -} - -func newNodeGetterFetcherSingleUseFactory(ctx context.Context, ng format.NodeGetter) *nodeGetterFetcherSingleUseFactory { - ls := cidlink.DefaultLinkSystem() - ls.TrustedStorage = true - ls.StorageReadOpener = blockOpener(ctx, ng) - ls.NodeReifier = unixfsnode.Reify - - pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - - return &nodeGetterFetcherSingleUseFactory{ls, pc} -} - -func (n *nodeGetterFetcherSingleUseFactory) NewSession(ctx context.Context) fetcher.Fetcher { - return n -} - -func (n *nodeGetterFetcherSingleUseFactory) NodeMatching(ctx context.Context, root ipld.Node, selector ipld.Node, cb fetcher.FetchCallback) error { - return n.nodeMatching(ctx, n.blankProgress(ctx), root, selector, cb) -} - -func (n *nodeGetterFetcherSingleUseFactory) BlockOfType(ctx context.Context, link ipld.Link, nodePrototype ipld.NodePrototype) (ipld.Node, error) { - return n.linkSystem.Load(ipld.LinkContext{}, link, nodePrototype) -} - -func (n *nodeGetterFetcherSingleUseFactory) BlockMatchingOfType(ctx context.Context, root ipld.Link, selector ipld.Node, nodePrototype ipld.NodePrototype, cb fetcher.FetchCallback) error { - // retrieve first node - prototype, err := n.PrototypeFromLink(root) - if err != nil { - return err - } - node, err := n.BlockOfType(ctx, root, prototype) - if err != nil { - return err - } - - progress := n.blankProgress(ctx) - progress.LastBlock.Link = root - return n.nodeMatching(ctx, progress, node, selector, cb) -} - -func (n *nodeGetterFetcherSingleUseFactory) PrototypeFromLink(lnk ipld.Link) (ipld.NodePrototype, error) { - return n.protoChooser(lnk, ipld.LinkContext{}) -} - -func (n *nodeGetterFetcherSingleUseFactory) nodeMatching(ctx context.Context, initialProgress traversal.Progress, node ipld.Node, match ipld.Node, cb fetcher.FetchCallback) error { - matchSelector, err := selector.ParseSelector(match) - if err != nil { - return err - } - return initialProgress.WalkMatching(node, matchSelector, func(prog traversal.Progress, n ipld.Node) error { - return cb(fetcher.FetchResult{ - Node: n, - Path: prog.Path, - LastBlockPath: prog.LastBlock.Path, - LastBlockLink: prog.LastBlock.Link, - }) - }) -} - -func (n *nodeGetterFetcherSingleUseFactory) blankProgress(ctx context.Context) traversal.Progress { - return traversal.Progress{ - Cfg: &traversal.Config{ - LinkSystem: n.linkSystem, - LinkTargetNodePrototypeChooser: n.protoChooser, - }, - } -} - -func blockOpener(ctx context.Context, ng format.NodeGetter) ipld.BlockReadOpener { - return func(_ ipld.LinkContext, lnk ipld.Link) (io.Reader, error) { - cidLink, ok := lnk.(cidlink.Link) - if !ok { - return nil, fmt.Errorf("invalid link type for loading: %v", lnk) - } - - blk, err := ng.Get(ctx, cidLink.Cid) - if err != nil { - return nil, err - } - - return bytes.NewReader(blk.RawData()), nil - } -} - -var _ fetcher.Fetcher = (*nodeGetterFetcherSingleUseFactory)(nil) -var _ fetcher.Factory = (*nodeGetterFetcherSingleUseFactory)(nil) - -func rangeStrToByteRange(rangeStr string) (*gateway.ByteRange, error) { - rangeElems := strings.Split(rangeStr, ":") - if len(rangeElems) > 2 { - return nil, fmt.Errorf("invalid range") - } - first, err := strconv.ParseUint(rangeElems[0], 10, 64) - if err != nil { - return nil, err - } - - if rangeElems[1] == "*" { - return &gateway.ByteRange{ - From: first, - To: nil, - }, nil - } - - second, err := strconv.ParseInt(rangeElems[1], 10, 64) - if err != nil { - return nil, err - } - - if second < 0 { - // TODO: fix, might also require a fix in boxo/gateway - return nil, fmt.Errorf("unsupported") - } - - if uint64(second) < first { - return nil, fmt.Errorf("invalid range") - } - - return &gateway.ByteRange{ - From: first, - To: &second, - }, nil -} - -func makeGatewayBlockHandler(bsrv blockservice.BlockService, port int) (*http.Server, error) { - mux := http.NewServeMux() - mux.HandleFunc("/ipfs/", func(w http.ResponseWriter, r *http.Request) { - // the hour is a hard fallback, we don't expect it to happen, but just in case - ctx, cancel := context.WithTimeout(r.Context(), time.Hour) - defer cancel() - r = r.WithContext(ctx) - - defer func() { - if r := recover(); r != nil { - goLog.Error("A panic occurred in the gateway handler!") - goLog.Error(r) - debug.PrintStack() - } - }() - - if r.Method != http.MethodGet { - w.Header().Add("Allow", http.MethodGet) - - errmsg := "Method " + r.Method + " not allowed" - http.Error(w, errmsg, http.StatusMethodNotAllowed) - return - } - - isBlock := false - if formatParam := r.URL.Query().Get("format"); formatParam != "" { - isBlock = formatParam == "raw" - if !isBlock { - http.Error(w, "only raw format supported", http.StatusBadRequest) - return - } - } else { - for _, header := range r.Header.Values("Accept") { - for _, value := range strings.Split(header, ",") { - accept := strings.TrimSpace(value) - if accept == "application/vnd.ipld.raw" { - isBlock = true - break - } - } - } - } - if !isBlock { - http.Error(w, "only raw format supported", http.StatusBadRequest) - return - } - - contentPath := ipath.New(r.URL.Path) - if contentPath.Mutable() { - http.Error(w, "only immutable block requests supported", http.StatusBadRequest) - return - } else if contentPath.Namespace() != "ipfs" { - http.Error(w, "only the ipfs names is supported", http.StatusBadRequest) - return - } - - strComps := strings.Split(strings.TrimRight(contentPath.String(), "/"), "/") - if len(strComps) != 3 { - http.Error(w, "requests must be for single raw blocks", http.StatusBadRequest) - return - } - c, err := cid.Decode(strComps[2]) - if err != nil { - http.Error(w, fmt.Sprintf("not a valid cid %s", strComps[2]), http.StatusBadRequest) - return - } - - blk, err := bsrv.GetBlock(r.Context(), c) - if err != nil { - http.Error(w, fmt.Sprintf("could not get cid %s", c), http.StatusInternalServerError) - return - } - - const immutableCacheControl = "public, max-age=29030400, immutable" - // immutable! CACHE ALL THE THINGS, FOREVER! wolololol - w.Header().Set("Cache-Control", immutableCacheControl) - w.Header().Set("Server", userAgent) - - // Set modtime to 'zero time' to disable Last-Modified header (superseded by Cache-Control) - - http.ServeContent(w, r, c.String()+".bin", noModtime, bytes.NewReader(blk.RawData())) - }) - - // Creates metrics handler for total response size. Matches the same metrics - // from Kubo: - // https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152 - sum := prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Namespace: "ipfs", - Subsystem: "http", - Name: "response_size_bytes", - Help: "The HTTP response sizes in bytes.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, nil) - err := prometheus.Register(sum) - if err != nil { - return nil, err - } - - // Construct the HTTP handler for the gateway. - handler := promhttp.InstrumentHandlerResponseSize(sum, mux) - - // Add logging - handler = withRequestLogger(handler) - - return &http.Server{ - Handler: handler, - Addr: ":" + strconv.Itoa(port), - }, nil -} diff --git a/test-backend/main.go b/test-backend/main.go deleted file mode 100644 index a61b27f..0000000 --- a/test-backend/main.go +++ /dev/null @@ -1,147 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "log" - "net/http" - "os" - "os/signal" - "sync" - - "github.com/ipfs/bifrost-gateway/lib" - "github.com/ipfs/boxo/bitswap/client" - "github.com/ipfs/boxo/bitswap/network" - "github.com/ipfs/boxo/blockservice" - offline "github.com/ipfs/boxo/exchange/offline" - golog "github.com/ipfs/go-log/v2" - carbs "github.com/ipld/go-car/v2/blockstore" - "github.com/libp2p/go-libp2p" - dht "github.com/libp2p/go-libp2p-kad-dht" - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/routing" - "github.com/spf13/cobra" -) - -var goLog = golog.Logger("test-backend") - -func main() { - if err := rootCmd.Execute(); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} - -func init() { - rootCmd.Flags().Int("gateway-port", 8082, "gateway port") - rootCmd.Flags().Int("metrics-port", 8042, "metrics port") - rootCmd.Flags().String("car-blockstore", "", "a CAR file to use for serving data instead of network requests") - golog.SetLogLevel("test-backend", "debug") -} - -var rootCmd = &cobra.Command{ - Use: name, - Version: version, - CompletionOptions: cobra.CompletionOptions{DisableDefaultCmd: true}, - Short: "Test Gateway backend for https://github.com/ipfs/bifrost-gateway (EXPERIMENTAL)", - RunE: func(cmd *cobra.Command, args []string) error { - // Get flags. - gatewayPort, _ := cmd.Flags().GetInt("gateway-port") - metricsPort, _ := cmd.Flags().GetInt("metrics-port") - carbsLocation, _ := cmd.Flags().GetString("car-blockstore") - - var bsrv blockservice.BlockService - if carbsLocation != "" { - bs, err := carbs.OpenReadOnly(carbsLocation) - if err != nil { - return err - } - bsrv = blockservice.New(bs, offline.Exchange(bs)) - } else { - //blockCacheSize, err := getEnvInt(EnvBlockCacheSize, lib.DefaultCacheBlockStoreSize) - //if err != nil { - // return err - //} - blockCacheSize := lib.DefaultCacheBlockStoreSize - - bs, err := lib.NewCacheBlockStore(blockCacheSize) - if err != nil { - return err - } - - var r routing.Routing - h, err := libp2p.New(libp2p.Routing(func(host host.Host) (routing.PeerRouting, error) { - r, err = dht.New(cmd.Context(), host, dht.BootstrapPeersFunc(dht.GetDefaultBootstrapPeerAddrInfos)) - return r, err - })) - if err != nil { - return err - } - n := network.NewFromIpfsHost(h, r) - bsc := client.New(cmd.Context(), n, bs) - n.Start(bsc) - defer n.Stop() - - bsrv = blockservice.New(bs, bsc) - } - - log.Printf("Starting %s %s", name, version) - - var gatewaySrv *http.Server - var err error - - if true { - gatewaySrv, err = makeGatewayCARHandler(bsrv, gatewayPort) - if err != nil { - return err - } - } else { - gatewaySrv, err = makeGatewayBlockHandler(bsrv, gatewayPort) - if err != nil { - return err - } - } - - metricsSrv, err := makeMetricsHandler(metricsPort) - if err != nil { - return err - } - - quit := make(chan os.Signal, 1) - var wg sync.WaitGroup - wg.Add(2) - - go func() { - defer wg.Done() - - //log.Printf("%s: %d", EnvBlockCacheSize, blockCacheSize) - log.Printf("Path gateway listening on http://127.0.0.1:%d", gatewayPort) - log.Printf(" Smoke test (JPG): http://127.0.0.1:%d/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?format=raw", gatewayPort) - log.Printf("Subdomain gateway configured on dweb.link and http://localhost:%d", gatewayPort) - log.Printf(" Smoke test (Subdomain+DNSLink+UnixFS+HAMT): http://localhost:%d/ipns/en.wikipedia-on-ipfs.org/wiki/", gatewayPort) - err := gatewaySrv.ListenAndServe() - if err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Printf("Failed to start gateway: %s", err) - quit <- os.Interrupt - } - }() - - go func() { - defer wg.Done() - log.Printf("Metrics exposed at http://127.0.0.1:%d/debug/metrics/prometheus", metricsPort) - err := metricsSrv.ListenAndServe() - if err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Printf("Failed to start metrics: %s", err) - quit <- os.Interrupt - } - }() - - signal.Notify(quit, os.Interrupt) - <-quit - log.Printf("Closing servers...") - go gatewaySrv.Close() - go metricsSrv.Close() - wg.Wait() - return nil - }, -} diff --git a/test-backend/version.go b/test-backend/version.go deleted file mode 100644 index 318d7f5..0000000 --- a/test-backend/version.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "runtime/debug" - "time" -) - -var name = "test-backend" -var version = buildVersion() -var userAgent = name + "/" + version - -func buildVersion() string { - var revision string - var day string - var dirty bool - - info, ok := debug.ReadBuildInfo() - if !ok { - return "dev-build" - } - for _, kv := range info.Settings { - switch kv.Key { - case "vcs.revision": - revision = kv.Value[:7] - case "vcs.time": - t, _ := time.Parse(time.RFC3339, kv.Value) - day = t.UTC().Format("2006-01-02") - case "vcs.modified": - dirty = kv.Value == "true" - } - } - if dirty { - revision += "-dirty" - } - if revision != "" { - return day + "-" + revision - } - return "dev-build" -} From 7f8eb459e60efc1a40a1bfc9458f1aaefeb01280 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 18 Jul 2023 13:20:39 -0400 Subject: [PATCH 12/46] feat: add some partial traversal tests and fixes --- go.mod | 42 +- go.sum | 221 +----- graph_gateway_test.go | 641 ++++++++++++++++++ lib/files.go | 19 +- ...h-multilayer-hamt-and-multiblock-files.car | Bin 0 -> 6045 bytes lib/gateway_traversal.go | 2 +- lib/graph_gateway.go | 120 ++-- 7 files changed, 732 insertions(+), 313 deletions(-) create mode 100644 graph_gateway_test.go create mode 100644 lib/fixtures/directory-with-multilayer-hamt-and-multiblock-files.car diff --git a/go.mod b/go.mod index 5790895..a2dcc9c 100644 --- a/go.mod +++ b/go.mod @@ -5,19 +5,19 @@ go 1.19 require ( github.com/cskr/pubsub v1.0.2 github.com/filecoin-saturn/caboose v0.0.3 + github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/ipfs/boxo v0.11.1-0.20230803093408-3b83506a2ea5 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-ipld-format v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipfs/go-unixfsnode v1.7.1 + github.com/ipfs/go-unixfsnode v1.7.3-0.20230718163022-62f4a1c8d46e github.com/ipld/go-car v0.6.0 github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 github.com/ipld/go-codec-dagpb v1.6.0 github.com/ipld/go-ipld-prime v0.20.0 github.com/libp2p/go-libp2p v0.26.3 - github.com/libp2p/go-libp2p-kad-dht v0.23.0 github.com/libp2p/go-libp2p-routing-helpers v0.7.0 github.com/mitchellh/go-server-timing v1.0.1 github.com/multiformats/go-multicodec v0.9.0 @@ -39,46 +39,31 @@ require ( require ( github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/asecurityteam/rolling v0.0.0-20230418204413-b4052899307d // indirect - github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/containerd/cgroups v1.0.4 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/elastic/gosigar v0.14.2 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/flynn/noise v1.0.0 // indirect - github.com/francoispqt/gojay v1.2.13 // indirect github.com/gabriel-vasile/mimetype v1.4.1 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect - github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 // indirect - github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect github.com/google/uuid v1.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/huin/goupnp v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-blockservice v0.5.0 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect - github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect @@ -90,46 +75,34 @@ require ( github.com/ipfs/go-merkledag v0.11.0 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-verifcid v0.0.2 // indirect - github.com/jackpal/go-nat-pmp v1.0.2 // indirect - github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/klauspost/compress v1.15.15 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect - github.com/koron/go-ssdp v0.0.3 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-doh-resolver v0.4.0 // indirect - github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.23.0 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect - github.com/libp2p/go-nat v0.1.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect - github.com/libp2p/go-reuseport v0.2.0 // indirect - github.com/libp2p/go-yamux/v4 v4.0.0 // indirect - github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/miekg/dns v1.1.50 // indirect - github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect - github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr v0.8.0 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect - github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/onsi/ginkgo/v2 v2.5.1 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/openzipkin/zipkin-go v0.4.1 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect - github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -137,12 +110,6 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/quic-go/qpack v0.4.0 // indirect - github.com/quic-go/qtls-go1-19 v0.2.1 // indirect - github.com/quic-go/qtls-go1-20 v0.1.1 // indirect - github.com/quic-go/quic-go v0.33.0 // indirect - github.com/quic-go/webtransport-go v0.5.2 // indirect - github.com/raulk/go-watchdog v1.3.0 // indirect github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -166,8 +133,6 @@ require ( go.opentelemetry.io/otel/exporters/zipkin v1.14.0 // indirect go.opentelemetry.io/otel/metric v0.37.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect - go.uber.org/dig v1.15.0 // indirect - go.uber.org/fx v1.18.2 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.6.0 // indirect golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb // indirect @@ -184,5 +149,4 @@ require ( google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect - nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index c839483..29735e9 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= @@ -32,29 +30,19 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= -dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= -dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= -git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/asecurityteam/rolling v0.0.0-20230418204413-b4052899307d h1:OD9AM8JZUHPrkEa9/SgQW2cCX6Om8d6n0akPS5leWJA= github.com/asecurityteam/rolling v0.0.0-20230418204413-b4052899307d/go.mod h1:tWDU1S7csNXWrzNpkbCk/dXpZkVcL4PfKn6Akwrffok= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= -github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -65,7 +53,6 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -74,15 +61,9 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= -github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= -github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= @@ -92,18 +73,13 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= -github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= -github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -116,22 +92,12 @@ github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBd github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/filecoin-saturn/caboose v0.0.3 h1:VcfNc3J6qwCAdxKIVkkCglG0ll1qL043S2qcChmUr6o= github.com/filecoin-saturn/caboose v0.0.3/go.mod h1:CSXZMijzD8z0Q/K5JQPeDZnb1o50MBK9VGZhOleqIDk= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= -github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= -github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= -github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -140,28 +106,9 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= -github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= -github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 h1:yrv1uUvgXH/tEat+wdvJMRJ4g51GlIydtDpU9pFjaaI= @@ -173,7 +120,6 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -182,7 +128,6 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -210,15 +155,10 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -231,22 +171,15 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20221203041831-ce31453925ec h1:fR20TYVVwhK4O7r7y+McjRYyaTH6/vjwJOajE+XhlzM= -github.com/google/pprof v0.0.0-20221203041831-ce31453925ec/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= -github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= @@ -263,10 +196,7 @@ github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+l github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= -github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= -github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -299,7 +229,6 @@ github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IW github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= -github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= @@ -332,8 +261,8 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= -github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s= -github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= +github.com/ipfs/go-unixfsnode v1.7.3-0.20230718163022-62f4a1c8d46e h1:LZkN2wQ49qj4g47ET4OgsI7gZ/8mFMbdXNvAoESC54U= +github.com/ipfs/go-unixfsnode v1.7.3-0.20230718163022-62f4a1c8d46e/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= github.com/ipld/go-car v0.6.0 h1:d5QrGLnHAxiNLHor+DKGrLdqnM0dQJh2whfSXRDq6J0= @@ -346,44 +275,30 @@ github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3u github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= -github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= -github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= -github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= -github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= @@ -391,7 +306,6 @@ github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0 github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+0S7FQqw= github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= -github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= github.com/libp2p/go-libp2p v0.26.3 h1:6g/psubqwdaBqNNoidbRKSTBEYgaOuKBhHl8Q5tO+PM= github.com/libp2p/go-libp2p v0.26.3/go.mod h1:x75BN32YbwuY0Awm2Uix4d4KOz+/4piInkp4Wr3yOo8= github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= @@ -408,36 +322,21 @@ github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUI github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= -github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= -github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= github.com/libp2p/go-reuseport v0.2.0 h1:18PRvIMlpY6ZK85nIAicSBuXXvrYoSw3dsBAR7zc560= -github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= -github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rBfZQ= -github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= -github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= -github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= -github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= -github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= -github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= @@ -445,12 +344,6 @@ github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/go-server-timing v1.0.1 h1:f00/aIe8T3MrnLhQHu3tSWvnwc5GV/p5eutuu3hF/tE= github.com/mitchellh/go-server-timing v1.0.1/go.mod h1:Mo6GKi9FSLwWFAMn3bqVPWe20y5ri5QGQuO9D9MCOxk= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -462,7 +355,6 @@ github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYg github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= @@ -470,7 +362,6 @@ github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUo github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= -github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= @@ -489,23 +380,17 @@ github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= -github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= -github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.5.1 h1:auzK7OI497k6x4OvWq+TKAcpcSAlod0doAH72oIN0Jw= -github.com/onsi/ginkgo/v2 v2.5.1/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= -github.com/onsi/gomega v1.24.0 h1:+0glovB9Jd6z3VR+ScSwQqXVTIfJcGA9UBM8yzQxhqg= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a h1:9iT75RHhYHWwWRlVWU7wnmtFulYcURCglzQOpT+cAF8= github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openzipkin/zipkin-go v0.4.1 h1:kNd/ST2yLLWhaWrkgchya40TJabe8Hioj9udfPcEO5A= github.com/openzipkin/zipkin-go v0.4.1/go.mod h1:qY0VqDSN1pOBN94dBc6w2GJlWLiovAyg7Qt6/I9HecM= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= -github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -516,74 +401,37 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= -github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= -github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/qtls-go1-19 v0.2.1 h1:aJcKNMkH5ASEJB9FXNeZCyTEIHU1J7MmHyz1Q1TSG1A= -github.com/quic-go/qtls-go1-19 v0.2.1/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= github.com/quic-go/qtls-go1-20 v0.1.1 h1:KbChDlg82d3IHqaj2bn6GfKRj84Per2VGf5XV3wSwQk= -github.com/quic-go/qtls-go1-20 v0.1.1/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= github.com/quic-go/quic-go v0.33.0 h1:ItNoTDN/Fm/zBlq769lLJc8ECe9gYaW40veHCCco7y0= -github.com/quic-go/quic-go v0.33.0/go.mod h1:YMuhaAV9/jIu0XclDXwZPAsP/2Kgr5yMYhe9oxhhOFA= github.com/quic-go/webtransport-go v0.5.2 h1:GA6Bl6oZY+g/flt00Pnu0XtivSD8vukOu3lYhJjnGEk= -github.com/quic-go/webtransport-go v0.5.2/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= -github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs= github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQEo87pStk/a99dzIO1mM9KxIyLPGTU= github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc= -github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= -github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= -github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= -github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= -github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= -github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= -github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= -github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= -github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= -github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= -github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= -github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= -github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= -github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= -github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= -github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= -github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= -github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= -github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= -github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= -github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -595,7 +443,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -605,20 +452,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0= github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM= -github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= -github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= @@ -640,7 +479,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -690,9 +528,7 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/dig v1.15.0 h1:vq3YWr8zRj1eFGC7Gvf907hE0eRjPTZ1d3xHadD6liE= -go.uber.org/dig v1.15.0/go.mod h1:pKHs0wMynzL6brANhB2hLMro+zalv1osARTviTcqHLM= go.uber.org/fx v1.18.2 h1:bUNI6oShr+OVFQeU8cDNbnN7VFsu+SsjHzUF51V/GAU= -go.uber.org/fx v1.18.2/go.mod h1:g0V1KMQ66zIRk8bLu3Ea5Jt2w/cHlOIp4wdRsgh0JaY= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -704,19 +540,13 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= -golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= -golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -733,7 +563,6 @@ golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb h1:PaBZQdo+iSDyHT053FjUCgZQ/ golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -757,14 +586,9 @@ golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -788,7 +612,6 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -796,14 +619,11 @@ golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -816,16 +636,10 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -833,13 +647,10 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -850,7 +661,6 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -877,14 +687,10 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -929,7 +735,6 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.5.0 h1:+bSpV5HIeWkuvgaMfI3UmKRThoTA5ODJTUd8T17NO+4= @@ -942,9 +747,6 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3j golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= -google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -962,18 +764,12 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= -google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1006,9 +802,6 @@ google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1047,10 +840,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1059,8 +849,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1071,9 +859,6 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= -nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/graph_gateway_test.go b/graph_gateway_test.go new file mode 100644 index 0000000..d748ec0 --- /dev/null +++ b/graph_gateway_test.go @@ -0,0 +1,641 @@ +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + + _ "embed" + + "github.com/ipfs/bifrost-gateway/lib" + "github.com/ipfs/boxo/blockservice" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/exchange/offline" + "github.com/ipfs/boxo/files" + "github.com/ipfs/boxo/gateway" + "github.com/ipfs/boxo/ipld/merkledag" + unixfile "github.com/ipfs/boxo/ipld/unixfs/file" + "github.com/ipfs/go-cid" + carv2 "github.com/ipld/go-car/v2" + carbs "github.com/ipld/go-car/v2/blockstore" + "github.com/ipld/go-car/v2/storage" +) + +//go:embed lib/fixtures/directory-with-multilayer-hamt-and-multiblock-files.car +var dirWithMultiblockHAMTandFiles []byte + +func TestTar(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + requestNum := 0 + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates in the middle of the HAMT + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeifdv255wmsrh75vcsrtkcwyktvewgihegeeyhhj2ju4lzt4lqfoze", // basicDir + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", // exampleB + "bafkreihgbi345degbcyxaf5b3boiyiaxhnuxdysvqmbdyaop2swmhh3s3m", + "bafkreiaugmh5gal5rgiems6gslcdt2ixfncahintrmcqvrgxqamwtlrmz4", + "bafkreiaxwwb7der2qvmteymgtlj7ww7w5vc44phdxfnexog3vnuwdkxuea", + "bafkreic5zyan5rk4ccfum4d4mu3h5eqsllbudlj4texlzj6xdgxvldzngi", + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamtDir + }); err != nil { + panic(err) + } + case 2: + // Expect a request for the HAMT only and give it + // Note: this is an implementation detail, it could be in the future that we request less or more data + // (e.g. requesting the blocks to fill out the HAMT, or with spec changes asking for HAMT ranges, or asking for the HAMT and its children) + expectedUri := "/ipfs/bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamtDir + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", + "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", + }); err != nil { + panic(err) + } + case 3: + // Starting here expect requests for each file in the directory + expectedUri := "/ipfs/bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", // exampleB + "bafkreihgbi345degbcyxaf5b3boiyiaxhnuxdysvqmbdyaop2swmhh3s3m", + "bafkreiaugmh5gal5rgiems6gslcdt2ixfncahintrmcqvrgxqamwtlrmz4", + "bafkreiaxwwb7der2qvmteymgtlj7ww7w5vc44phdxfnexog3vnuwdkxuea", + "bafkreic5zyan5rk4ccfum4d4mu3h5eqsllbudlj4texlzj6xdgxvldzngi", + }); err != nil { + panic(err) + } + case 4: + // Expect a request for one of the directory items and give it + expectedUri := "/ipfs/bafkreih2grj7p2bo5yk2guqazxfjzapv6hpm3mwrinv6s3cyayd72ke5he" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafkreih2grj7p2bo5yk2guqazxfjzapv6hpm3mwrinv6s3cyayd72ke5he", // exampleD + }); err != nil { + panic(err) + } + case 5: + // Expect a request for one of the directory items and give it + expectedUri := "/ipfs/bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee", // exampleC + }); err != nil { + panic(err) + } + case 6: + // Expect a request for one of the directory items and give part of it + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + }); err != nil { + panic(err) + } + case 7: + // Expect a partial request for one of the directory items and give it + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + }); err != nil { + panic(err) + } + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + exch, err := newExchange(bs) + if err != nil { + t.Fatal(err) + } + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}, exch) + if err != nil { + t.Fatal(err) + } + + p, err := gateway.NewImmutablePath(path.New("/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi")) + if err != nil { + t.Fatal(err) + } + _, nd, err := backend.GetAll(ctx, p) + if err != nil { + t.Fatal(err) + } + + assertNextEntryNameEquals := func(t *testing.T, dirIter files.DirIterator, expectedName string) { + t.Helper() + if !dirIter.Next() { + iterErr := dirIter.Err() + t.Fatalf("expected entry, but errored with %s", iterErr.Error()) + } + if expectedName != dirIter.Name() { + t.Fatalf("expected %s, got %s", expectedName, dirIter.Name()) + } + } + + robs, err := carbs.NewReadOnly(bytes.NewReader(dirWithMultiblockHAMTandFiles), nil) + if err != nil { + t.Fatal(err) + } + + dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) + assertFileEqual := func(t *testing.T, expectedCidString string, receivedFile files.File) { + t.Helper() + + expected := cid.MustParse(expectedCidString) + receivedFileData, err := io.ReadAll(receivedFile) + if err != nil { + t.Fatal(err) + } + nd, err := dsrv.Get(ctx, expected) + if err != nil { + t.Fatal(err) + } + expectedFile, err := unixfile.NewUnixfsFile(ctx, dsrv, nd) + if err != nil { + t.Fatal(err) + } + + expectedFileData, err := io.ReadAll(expectedFile.(files.File)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(expectedFileData, receivedFileData) { + t.Fatalf("expected %s, got %s", string(expectedFileData), string(receivedFileData)) + } + } + + rootDirIter := nd.(files.Directory).Entries() + assertNextEntryNameEquals(t, rootDirIter, "basicDir") + + basicDirIter := rootDirIter.Node().(files.Directory).Entries() + assertNextEntryNameEquals(t, basicDirIter, "exampleA") + assertFileEqual(t, "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", basicDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, basicDirIter, "exampleB") + assertFileEqual(t, "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", basicDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, rootDirIter, "hamtDir") + hamtDirIter := rootDirIter.Node().(files.Directory).Entries() + + assertNextEntryNameEquals(t, hamtDirIter, "exampleB") + assertFileEqual(t, "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", hamtDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, hamtDirIter, "exampleD-hamt-collide-exampleB-seed-364") + assertFileEqual(t, "bafkreih2grj7p2bo5yk2guqazxfjzapv6hpm3mwrinv6s3cyayd72ke5he", hamtDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, hamtDirIter, "exampleC-hamt-collide-exampleA-seed-52") + assertFileEqual(t, "bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee", hamtDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, hamtDirIter, "exampleA") + assertFileEqual(t, "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", hamtDirIter.Node().(files.File)) + + if rootDirIter.Next() || basicDirIter.Next() || hamtDirIter.Next() { + t.Fatal("expected directories to be fully enumerated") + } +} + +func TestTarAtEndOfPath(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + requestNum := 0 + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates in the middle of the path + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + }); err != nil { + panic(err) + } + case 2: + // Expect the full request and give the path and the children from one of the HAMT nodes but not the other + // Note: this is an implementation detail, it could be in the future that we request less or more data + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamtDir + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", + "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", // exampleB + "bafkreihgbi345degbcyxaf5b3boiyiaxhnuxdysvqmbdyaop2swmhh3s3m", + "bafkreiaugmh5gal5rgiems6gslcdt2ixfncahintrmcqvrgxqamwtlrmz4", + "bafkreiaxwwb7der2qvmteymgtlj7ww7w5vc44phdxfnexog3vnuwdkxuea", + "bafkreic5zyan5rk4ccfum4d4mu3h5eqsllbudlj4texlzj6xdgxvldzngi", + "bafkreih2grj7p2bo5yk2guqazxfjzapv6hpm3mwrinv6s3cyayd72ke5he", // exampleD + }); err != nil { + panic(err) + } + case 3: + // Expect a request for the HAMT only and give it + // Note: this is an implementation detail, it could be in the future that we request less or more data + // (e.g. requesting the blocks to fill out the HAMT, or with spec changes asking for HAMT ranges, or asking for the HAMT and its children) + expectedUri := "/ipfs/bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamtDir + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", + "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", + }); err != nil { + panic(err) + } + case 4: + // Expect a request for one of the directory items and give it + expectedUri := "/ipfs/bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee", // exampleC + }); err != nil { + panic(err) + } + case 5: + // Expect a request for the multiblock file in the directory and give some of it + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + }); err != nil { + panic(err) + } + case 6: + // Expect a request for the rest of the multiblock file in the directory and give it + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa?dag-scope=entity&entity-bytes=768:*" + if request.RequestURI != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + }); err != nil { + panic(err) + } + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + exch, err := newExchange(bs) + if err != nil { + t.Fatal(err) + } + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}, exch) + if err != nil { + t.Fatal(err) + } + + p, err := gateway.NewImmutablePath(path.New("/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir")) + if err != nil { + t.Fatal(err) + } + _, nd, err := backend.GetAll(ctx, p) + if err != nil { + t.Fatal(err) + } + + assertNextEntryNameEquals := func(t *testing.T, dirIter files.DirIterator, expectedName string) { + t.Helper() + if !dirIter.Next() { + t.Fatal("expected entry") + } + if expectedName != dirIter.Name() { + t.Fatalf("expected %s, got %s", expectedName, dirIter.Name()) + } + } + + robs, err := carbs.NewReadOnly(bytes.NewReader(dirWithMultiblockHAMTandFiles), nil) + if err != nil { + t.Fatal(err) + } + + dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) + assertFileEqual := func(t *testing.T, expectedCidString string, receivedFile files.File) { + t.Helper() + + expected := cid.MustParse(expectedCidString) + receivedFileData, err := io.ReadAll(receivedFile) + if err != nil { + t.Fatal(err) + } + nd, err := dsrv.Get(ctx, expected) + if err != nil { + t.Fatal(err) + } + expectedFile, err := unixfile.NewUnixfsFile(ctx, dsrv, nd) + if err != nil { + t.Fatal(err) + } + + expectedFileData, err := io.ReadAll(expectedFile.(files.File)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(expectedFileData, receivedFileData) { + t.Fatalf("expected %s, got %s", string(expectedFileData), string(receivedFileData)) + } + } + + hamtDirIter := nd.(files.Directory).Entries() + + assertNextEntryNameEquals(t, hamtDirIter, "exampleB") + assertFileEqual(t, "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", hamtDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, hamtDirIter, "exampleD-hamt-collide-exampleB-seed-364") + assertFileEqual(t, "bafkreih2grj7p2bo5yk2guqazxfjzapv6hpm3mwrinv6s3cyayd72ke5he", hamtDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, hamtDirIter, "exampleC-hamt-collide-exampleA-seed-52") + assertFileEqual(t, "bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee", hamtDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, hamtDirIter, "exampleA") + assertFileEqual(t, "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", hamtDirIter.Node().(files.File)) + + if hamtDirIter.Next() { + t.Fatal("expected directories to be fully enumerated") + } +} + +func sendBlocks(ctx context.Context, carFixture []byte, writer io.Writer, cidStrList []string) error { + rd, err := storage.OpenReadable(bytes.NewReader(carFixture)) + if err != nil { + return err + } + + cw, err := storage.NewWritable(writer, []cid.Cid{cid.MustParse("bafkqaaa")}, carv2.WriteAsCarV1(true)) + if err != nil { + return err + } + + for _, s := range cidStrList { + c := cid.MustParse(s) + blockData, err := rd.Get(ctx, c.KeyString()) + if err != nil { + return err + } + + if err := cw.Put(ctx, c.KeyString(), blockData); err != nil { + return err + } + } + return nil +} + +func TestGetCAR(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + requestNum := 0 + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates in the middle of the path + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + }); err != nil { + panic(err) + } + case 2: + // Expect the full request, but return one that terminates in the middle of the HAMT + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. partial path) + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + }); err != nil { + panic(err) + } + + case 3: + // Expect the full request and return the full HAMT + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. requesting the blocks to fill out the HAMT, or with spec changes asking for HAMT ranges) + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeifdv255wmsrh75vcsrtkcwyktvewgihegeeyhhj2ju4lzt4lqfoze", // basicDir + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", // exampleB + "bafkreihgbi345degbcyxaf5b3boiyiaxhnuxdysvqmbdyaop2swmhh3s3m", + "bafkreiaugmh5gal5rgiems6gslcdt2ixfncahintrmcqvrgxqamwtlrmz4", + "bafkreiaxwwb7der2qvmteymgtlj7ww7w5vc44phdxfnexog3vnuwdkxuea", + "bafkreic5zyan5rk4ccfum4d4mu3h5eqsllbudlj4texlzj6xdgxvldzngi", + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamtDir + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", + "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", // exampleB + "bafkreihgbi345degbcyxaf5b3boiyiaxhnuxdysvqmbdyaop2swmhh3s3m", + "bafkreiaugmh5gal5rgiems6gslcdt2ixfncahintrmcqvrgxqamwtlrmz4", + "bafkreiaxwwb7der2qvmteymgtlj7ww7w5vc44phdxfnexog3vnuwdkxuea", + "bafkreic5zyan5rk4ccfum4d4mu3h5eqsllbudlj4texlzj6xdgxvldzngi", + "bafkreih2grj7p2bo5yk2guqazxfjzapv6hpm3mwrinv6s3cyayd72ke5he", // exampleD + "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", + "bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee", // exampleC + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + }); err != nil { + panic(err) + } + + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + exch, err := newExchange(bs) + if err != nil { + t.Fatal(err) + } + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}, exch) + if err != nil { + t.Fatal(err) + } + + p, err := gateway.NewImmutablePath(path.New("/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi")) + if err != nil { + t.Fatal(err) + } + var carReader io.Reader + _, carReader, err = backend.GetCAR(ctx, p, gateway.CarParams{Scope: gateway.DagScopeAll}) + if err != nil { + t.Fatal(err) + } + + carBytes, err := io.ReadAll(carReader) + if err != nil { + t.Fatal(err) + } + carReader = bytes.NewReader(carBytes) + + blkReader, err := carv2.NewBlockReader(carReader) + if err != nil { + t.Fatal(err) + } + + responseCarBlock := []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeifdv255wmsrh75vcsrtkcwyktvewgihegeeyhhj2ju4lzt4lqfoze", // basicDir + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", // exampleB + "bafkreihgbi345degbcyxaf5b3boiyiaxhnuxdysvqmbdyaop2swmhh3s3m", + "bafkreiaugmh5gal5rgiems6gslcdt2ixfncahintrmcqvrgxqamwtlrmz4", + "bafkreiaxwwb7der2qvmteymgtlj7ww7w5vc44phdxfnexog3vnuwdkxuea", + "bafkreic5zyan5rk4ccfum4d4mu3h5eqsllbudlj4texlzj6xdgxvldzngi", + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamtDir + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", + "bafkreih2grj7p2bo5yk2guqazxfjzapv6hpm3mwrinv6s3cyayd72ke5he", // exampleD + "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", + "bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee", // exampleC + } + + for i := 0; i < len(responseCarBlock); i++ { + expectedCid := cid.MustParse(responseCarBlock[i]) + blk, err := blkReader.Next() + if err != nil { + t.Fatal(err) + } + if !blk.Cid().Equals(expectedCid) { + t.Fatalf("expected cid %s, got %s", expectedCid, blk.Cid()) + } + } + _, err = blkReader.Next() + if !errors.Is(err, io.EOF) { + t.Fatal("expected an EOF") + } +} + +type retryFetcher struct { + inner lib.CarFetcher + allowedRetries int + retriesRemaining int +} + +func (r *retryFetcher) Fetch(ctx context.Context, path string, cb lib.DataCallback) error { + err := r.inner.Fetch(ctx, path, cb) + if err == nil { + return nil + } + + if r.retriesRemaining > 0 { + r.retriesRemaining-- + } else { + return fmt.Errorf("retry fetcher out of retries: %w", err) + } + + switch t := err.(type) { + case *lib.ErrPartialResponse: + if len(t.StillNeed) > 1 { + panic("only a single request at a time supported") + } + + // Mimicking the Caboose logic reset the number of retries for partials + r.retriesRemaining = r.allowedRetries + + return r.Fetch(ctx, t.StillNeed[0], cb) + default: + return r.Fetch(ctx, path, cb) + } +} + +var _ lib.CarFetcher = (*retryFetcher)(nil) diff --git a/lib/files.go b/lib/files.go index ffac463..be17c32 100644 --- a/lib/files.go +++ b/lib/files.go @@ -66,7 +66,7 @@ func (b *backpressuredFile) Read(p []byte) (n int, err error) { err = b.retErr } - from, err := b.f.Seek(io.SeekCurrent, 0) + from, err := b.f.Seek(0, io.SeekCurrent) if err != nil { return 0, err } @@ -85,8 +85,7 @@ func (b *backpressuredFile) Read(p []byte) (n int, err error) { } func (b *backpressuredFile) Seek(offset int64, whence int) (int64, error) { - //TODO implement me - panic("implement me") + return b.f.Seek(offset, whence) } var _ files.File = (*backpressuredFile)(nil) @@ -266,7 +265,7 @@ func (it *backpressuredHAMTDirIter) Next() bool { } _, pbn, fieldData, _, _, ufsBaseErr := loadUnixFSBase(it.ctx, it.dirCid, nil, lsys) - if err != nil { + if ufsBaseErr != nil { err = ufsBaseErr continue } @@ -285,8 +284,7 @@ func (it *backpressuredHAMTDirIter) Next() bool { } } - it.linksItr = nd.MapIterator() - iter = it.linksItr + it.linksItr = iter } var k, v ipld.Node @@ -514,12 +512,17 @@ func tv(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld.LinkSystem, if err != nil { return nil, fmt.Errorf("unable to get UnixFS file size: %w", err) } - _, err = f.Seek(0, io.SeekStart) + + from := int64(0) + if params.Range != nil { + from = params.Range.From + } + _, err = f.Seek(from, io.SeekStart) if err != nil { return nil, fmt.Errorf("unable to get reset UnixFS file reader: %w", err) } - return &backpressuredFile{ctx: ctx, size: fileSize, f: f, getLsys: getLsys, closed: make(chan error)}, nil + return &backpressuredFile{ctx: ctx, fileCid: c, size: fileSize, f: f, getLsys: getLsys, closed: make(chan error)}, nil default: return nil, fmt.Errorf("unknown UnixFS field type") } diff --git a/lib/fixtures/directory-with-multilayer-hamt-and-multiblock-files.car b/lib/fixtures/directory-with-multilayer-hamt-and-multiblock-files.car new file mode 100644 index 0000000000000000000000000000000000000000..cb2a4875dc9169e97cd906a472d069b5bfd0f50f GIT binary patch literal 6045 zcma)A3vg7`8BSJ_?G{F6fo3Vt9*81pvV{Z?gfWm;MXHfU6hWEZ+p zMQuAW6lk@jJTyRSrxg$o1}a4hVh3gjV}(M6Mg~SSGx(q?h}hcyKaab2wKFTjY?8g_ z{?G6KzW?4^*HhPVeDBc{H_g7jq%@l(i$;FBXldPveb06cJ9lw$EcwgD3!8^+pA}y< z_Hx@ccaw_)?kC#$B{T$z4H_1&AExNGCqzT!u}pw`rD-_nf*R5sR0uo=i?CJrZ zfB(zJ=Id1tEl-yGS^MCx?tHOk(hnMp+5dg(FV?&V-dVk2=ZSNN z=H7N}b@OdA`X@_L9kHZ5{6%@!)u%_t7Pod!*=An*#Y@fQw`}>K>D^1u41M>9b$@%i4cteb_KT9KK1!ylvgveY&1*dU#K~k6W|$qeV(Jbm$ z5Z$pos{7OrT%s8)%RHUg8A46wr^#J`trOd^3A3`)H5m1Y4PT|n-LAuIqUmHopjjk< zuUShQ+(cO`wGE<~Y(YT5z&+ISSxaDoW6k!dj3MhXi+Kbrv}%D4*FEC6IwNMF`|vw> z2~6K*I`v4_@dD~nq6aO^WE%W*H9C>Zqy{lrCNMRTP(>s{Ps=dh2m6|<2Vks%h$y<<}yWAfoTluipW0j+?r8_Pf4-0dr7Pw%b4XWi*7rNm&hGs&-CNu+UkO7aa)U~MX2bL;$fxle7 zwz^|kj^RMPSx&Ryb_`ir3Pi}tH!)boW}e3^ehT_;Fz!WEsTjyDGC%Nqa5dE?Z2*8a zOjh$eO0uTrQ|JeS1S}7lFLTUM6H5W!02Ap%LdDq105pMDL8uL}Xg)DDfFAT%)U87H z&xI=)SIaR&!xMOeLT~uJrDbeQU?{u>)k$-lAbw5ih?^19*bkCpz{0L35vfJmZfIGI zOP(azz|1jQb0ywHm5NPwK12@t5YC7pG%e;Q8@SLbgc;-{z~UjWofgMM03*gK!X$H0 zXE{M>G6g3L(y*AkKMa!wq=mHru3+Yb0cdef1E9i0=we)W_f%1NKf%920Ff9^9l61?X;1$4%`0;+zTeO_>Fn+~({`Qw6Vv|q2`M1!2f9kS z58U7HsmAP2=(vYhCg<#*vT4$)^q%LB58BfFi^?iEyfD!Z$okSXd8S?{a8oRWscryd zmxGCb-qLoY9Sddxh6h2c0WQL;l8&;##0K<0(G%kq_)%zhYmi|uKQMww=MC)?Xx7(! zSU!{!p$CHbfy#jTIY(*Sqm5V)5g@T;VFuxD&MW z5n@Bzb(-}lgyg7OMD?Qr&HG5Gs8TV|VwAG)Jm84XhpA8?Zm3N|Aj(+KGzA`2R;(=q z5A}xBfbWs(6$&T|JWwDYT*L$q8R!X+f;7dlBS^&Xg$_|mq zhVvNaR*+3SU$pox!}Xsquq?~eH_^$NujqDODYp39>b-X)-@g2GuOX}K?(@$KukDx0 z%sRDh{HTL}xY9QD_C?9q*xJ;G{gd)|&#R}tmmJzO=#~Z)FvJ zJ~>NlzMqeodFj3M2Lm_EC^>v+&7+@Q=svvdjk*VW%-PrU{?Ev@6OuR9*2>%JEAdgK zQpf%pv{D(X^e8ndtH+H=ElDJXK}M-)CXu8j^Q@jvAS#NvY`WOH72*p=Ot2>47B*c)Q0OgA|xNbHSVd-*L-ouF#nD!ox#)2y^3n5~&}$ zm1U5Whvb7MpeHIMmPO`*%}ws*obS9m=QypcAZVx*YR1t(VFtW9xLy<%A#+ zaYakUS&9dpCQUMRaSHWIM#Q&9uCx-CDm(=piEZd!ST?zA0*mvQ!w{qx3X`hMf|eE7 zL8SX8xFp~%6X^}umtc@J4>1uD*1Wo)+z2lIN}Nh7p@CpMZv=V(52c}j@B%Kr@N_37 zSFkzUh%$*=4N^4x8EyxKjpbm${R>X3rvp9}`~wt9$YMb&r@~V*Y)hyGR0*n-H&sXw zKDCXAlFG_NmIDb>4!baG_#;*InJU&J6 z_}Hq{zTQcxDMerTBpMr~QGCIaxDnW&bd*evkp9yCfpl1nYQOkf>EO@i| zroov2sV*b}i4eZH2)xRB6;9>2=a)xL{Tv?nvV^bwA!;Qm@tus=%=uxVLHt)atZLfw zRzqmP^davf1Mr~u2j?ZxZSfi6onKKm^MS}ui(n8HT$(iF#N=aB&=RyMAjJ>m@rqFZ z@_6W>OP;$}jnia03Ly-xs9oXoaR=~Ng4qHk0fOKCgw6wgSnI7X5346RykrDKOtG2`wmngkRBJgk-?8w5RlvM~_4S zilSmHP^Adf<;fNqRLOzVqbTX(dmea>#fK$)BH@n-;Wy#qqnsOFGL&sjqVNt8W2uu5 QS)$l^wpMjk0ER~W54hCPHUIzs literal 0 HcmV?d00001 diff --git a/lib/gateway_traversal.go b/lib/gateway_traversal.go index 854a3ef..231118e 100644 --- a/lib/gateway_traversal.go +++ b/lib/gateway_traversal.go @@ -118,7 +118,7 @@ func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *Grap } if !ok || blkRead.err != nil { if !ok || errors.Is(blkRead.err, io.EOF) { - return nil, io.EOF + return nil, io.ErrUnexpectedEOF } return nil, blkRead.err } diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index e47fb49..345f84e 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -14,6 +14,8 @@ import ( "sync" "time" + "github.com/hashicorp/go-multierror" + "github.com/ipld/go-ipld-prime/traversal" "github.com/filecoin-saturn/caboose" @@ -58,6 +60,8 @@ type DataCallback = caboose.DataCallback // TODO: Don't use a caboose type type ErrPartialResponse = caboose.ErrPartialResponse +var ErrFetcherUnexpectedEOF = fmt.Errorf("failed to fetch IPLD data") + type CarFetcher interface { Fetch(ctx context.Context, path string, cb DataCallback) error } @@ -801,7 +805,6 @@ var _ io.ReadCloser = (*multiReadCloser)(nil) func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "all", "entityRanges": "0"}).Inc() - p := ipfspath.FromString(path.String()) type terminalPathType struct { resp files.Node @@ -840,59 +843,80 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) } } - err := api.fetchCAR(cctx, path, gateway.CarParams{Scope: gateway.DagScopeAll}, func(resource string, reader io.Reader) error { + p := ipfspath.FromString(path.String()) + params := gateway.CarParams{Scope: gateway.DagScopeAll} + + err := api.fetchCAR(cctx, path, params, func(resource string, reader io.Reader) error { gb, err := carToLinearBlockGetter(cctx, reader, api.metrics) if err != nil { return err } lsys := getLinksystem(gb) - // First resolve the path since we always need to. - pathRootCids, terminalCid, remainder, terminalBlk, err := api.resolvePathWithRootsAndBlock(cctx, p, lsys) - if err != nil { - return err - } - md := contentMetadataFromRootsAndRemainder(p, pathRootCids, terminalCid, remainder) - - if len(remainder) > 0 { - terminalPathElementCh <- terminalPathType{err: errNotUnixFS} - return nil - } if hasSentAsyncData { + _, _, _, _, err = api.resolvePathToLastWithRoots(cctx, p, lsys) + if err != nil { + return err + } + select { case sendResponse <- lsys: - case <-ctx.Done(): - return ctx.Err() + case <-cctx.Done(): + return cctx.Err() } - } + } else { + // First resolve the path since we always need to. + pathRootCids, terminalCid, remainder, terminalBlk, err := api.resolvePathWithRootsAndBlock(cctx, p, lsys) + if err != nil { + return err + } + md := contentMetadataFromRootsAndRemainder(p, pathRootCids, terminalCid, remainder) - nd, err := tv(cctx, terminalCid, terminalBlk, lsys, gateway.CarParams{Scope: gateway.DagScopeAll}, getLsys) - if err != nil { - return err - } + if len(remainder) > 0 { + terminalPathElementCh <- terminalPathType{err: errNotUnixFS} + return nil + } - ndAc, ok := nd.(AwaitCloser) - if !ok { + if hasSentAsyncData { + select { + case sendResponse <- lsys: + case <-ctx.Done(): + return ctx.Err() + } + } + + nd, err := tv(cctx, terminalCid, terminalBlk, lsys, params, getLsys) + if err != nil { + return err + } + + ndAc, ok := nd.(AwaitCloser) + if !ok { + terminalPathElementCh <- terminalPathType{ + resp: nd, + md: md, + } + return nil + } + + hasSentAsyncData = true terminalPathElementCh <- terminalPathType{ resp: nd, md: md, } - return nil - } - hasSentAsyncData = true - terminalPathElementCh <- terminalPathType{ - resp: nd, - md: md, + closeCh = ndAc.AwaitClose() } - closeCh = ndAc.AwaitClose() select { case closeErr := <-closeCh: return closeErr case req := <-sendRequest: requestStr := fmt.Sprintf("/ipfs/%s?%s", req.c.String(), paramsToString(req.params)) + // set path and params for next iteration + p = ipfspath.FromCid(req.c) + params = req.params return &ErrPartialResponse{StillNeed: []string{requestStr}} case <-cctx.Done(): return cctx.Err() @@ -905,19 +929,15 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) } if err != nil { + lsys := getLinksystem(func(ctx context.Context, cid cid.Cid) (blocks.Block, error) { + return nil, multierror.Append(ErrFetcherUnexpectedEOF, format.ErrNotFound{Cid: cid}) + }) for { select { case <-closeCh: return case <-sendRequest: - lsys := getLinksystem(func(ctx context.Context, cid cid.Cid) (blocks.Block, error) { - return nil, fmt.Errorf("done trying to fetch CAR data: %w", format.ErrNotFound{Cid: cid}) - }) - select { - case sendResponse <- lsys: - case <-cctx.Done(): - return - } + case sendResponse <- lsys: case <-cctx.Done(): return } @@ -1205,20 +1225,22 @@ func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath, return nil } - cw, err = storage.NewWritable(w, []cid.Cid{terminalCid}, carv2.WriteAsCarV1(true)) - if err != nil { - // io.PipeWriter.CloseWithError always returns nil. - _ = w.CloseWithError(err) - return nil - } - for _, blk := range blockBuffer { - err = cw.Put(ctx, blk.Cid().KeyString(), blk.RawData()) + if cw == nil { + cw, err = storage.NewWritable(w, []cid.Cid{terminalCid}, carv2.WriteAsCarV1(true)) if err != nil { - _ = w.CloseWithError(fmt.Errorf("error writing car block: %w", err)) + // io.PipeWriter.CloseWithError always returns nil. + _ = w.CloseWithError(err) return nil } + for _, blk := range blockBuffer { + err = cw.Put(ctx, blk.Cid().KeyString(), blk.RawData()) + if err != nil { + _ = w.CloseWithError(fmt.Errorf("error writing car block: %w", err)) + return nil + } + } + blockBuffer = nil } - blockBuffer = nil err = walkGatewaySimpleSelector(ctx, terminalBlk, params, l) if err != nil { @@ -1333,6 +1355,10 @@ func checkRetryableError(e *error, fn func() error) error { } func isRetryableError(err error) (bool, error) { + if errors.Is(err, ErrFetcherUnexpectedEOF) { + return false, err + } + if format.IsNotFound(err) { return true, err } From 60f2d4ab3181c91e8d1c37c324aa8bba25611571 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 21 Jul 2023 18:17:03 -0400 Subject: [PATCH 13/46] fix: HAMT fetching bug for Get requests --- lib/graph_gateway.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 345f84e..a49c108 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -648,7 +648,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by }} dirLinkNum++ - if dirLinkNum-1 <= lastDirLinkNum { + if dirLinkNum-1 < lastDirLinkNum { continue } From 1c28ef456122ed8e64913fc810f0d013d0aa96cb Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 21 Jul 2023 18:17:49 -0400 Subject: [PATCH 14/46] test: add tests for Get on HAMT directories --- graph_gateway_test.go | 106 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/graph_gateway_test.go b/graph_gateway_test.go index d748ec0..b4f99bf 100644 --- a/graph_gateway_test.go +++ b/graph_gateway_test.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/http/httptest" + "strings" "testing" _ "embed" @@ -454,6 +455,111 @@ func sendBlocks(ctx context.Context, carFixture []byte, writer io.Writer, cidStr return nil } +func TestGetHAMTDirectory(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + requestNum := 0 + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates in the middle of the path + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + }); err != nil { + panic(err) + } + case 2: + // Expect the full request, but return one that terminates in the middle of the HAMT + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. partial path) + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + }); err != nil { + panic(err) + } + + case 3: + // Expect the full request and return the full HAMT + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. ask for index.html first, or make a spec change to allow asking for it with a fallback to the directory) + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", // inner hamt nodes start here + "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", + }); err != nil { + panic(err) + } + case 4: + // Expect a request for a non-existent index.html file + // Note: this is an implementation detail related to the directory request above + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir//index.html" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", // inner hamt nodes start here + }); err != nil { + panic(err) + } + + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + exch, err := newExchange(bs) + if err != nil { + t.Fatal(err) + } + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}, exch) + if err != nil { + t.Fatal(err) + } + + trustedGatewayServer := httptest.NewServer(gateway.NewHandler(gateway.Config{DeserializedResponses: true}, backend)) + defer trustedGatewayServer.Close() + + resp, err := http.Get(trustedGatewayServer.URL + "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/") + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + if strings.Count(string(data), ">exampleD-hamt-collide-exampleB-seed-364<") == 1 && + strings.Count(string(data), ">exampleC-hamt-collide-exampleA-seed-52<") == 1 && + strings.Count(string(data), ">exampleA<") == 1 && + strings.Count(string(data), ">exampleB<") == 1 { + return + } + t.Fatal("directory does not contain the expected links") +} + func TestGetCAR(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() From f473fea6b8a222a9fc826944abb511e5f5bf02bc Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 24 Jul 2023 15:55:35 -0400 Subject: [PATCH 15/46] feat: fix and add tests for Get on large files. Consolidate some Get and GetAll code which results in Get gaining partial resumption of terminal entities --- graph_gateway_test.go | 133 +++++++- lib/files.go | 28 +- lib/gateway_traversal.go | 33 +- lib/graph_gateway.go | 687 +++++++++++++++++++-------------------- 4 files changed, 472 insertions(+), 409 deletions(-) diff --git a/graph_gateway_test.go b/graph_gateway_test.go index b4f99bf..6721d3c 100644 --- a/graph_gateway_test.go +++ b/graph_gateway_test.go @@ -455,6 +455,131 @@ func sendBlocks(ctx context.Context, carFixture []byte, writer io.Writer, cidStr return nil } +func TestGetFile(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + requestNum := 0 + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates in the middle of the path + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + }); err != nil { + panic(err) + } + case 2: + // Expect the full request, but return one that terminates in the middle of the file + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. partial path) + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + }); err != nil { + panic(err) + } + + case 3: + // Expect the full request and return the path and most of the file + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. partial path and file range) + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", // inner hamt + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", // file chunks start here + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + }); err != nil { + panic(err) + } + + case 4: + // Expect a request for the remainder of the file + // Note: this is an implementation detail, it could be that the requester really asks for more information + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", // middle of the file starts here + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + }); err != nil { + panic(err) + } + + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + exch, err := newExchange(bs) + if err != nil { + t.Fatal(err) + } + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}, exch) + if err != nil { + t.Fatal(err) + } + + trustedGatewayServer := httptest.NewServer(gateway.NewHandler(gateway.Config{DeserializedResponses: true}, backend)) + defer trustedGatewayServer.Close() + + resp, err := http.Get(trustedGatewayServer.URL + "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA") + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + robs, err := carbs.NewReadOnly(bytes.NewReader(dirWithMultiblockHAMTandFiles), nil) + if err != nil { + t.Fatal(err) + } + + dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) + fileRootNd, err := dsrv.Get(ctx, cid.MustParse("bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa")) + if err != nil { + t.Fatal(err) + } + uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd) + if err != nil { + t.Fatal(err) + } + f := uio.(files.File) + expectedFileData, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data, expectedFileData) { + t.Fatalf("expected %s, got %s", string(expectedFileData), string(data)) + } +} + func TestGetHAMTDirectory(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -491,15 +616,15 @@ func TestGetHAMTDirectory(t *testing.T) { } case 3: - // Expect the full request and return the full HAMT - // Note: this is an implementation detail, it could be in the future that we request less data (e.g. ask for index.html first, or make a spec change to allow asking for it with a fallback to the directory) - expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/" + // Expect a request for the full HAMT and return it + // Note: this is an implementation detail, it could be in the future that we request more or less data + // (e.g. ask for the full path, ask for index.html first, make a spec change to allow asking for index.html with a fallback to the directory, etc.) + expectedUri := "/ipfs/bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm" if request.URL.Path != expectedUri { panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) } if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ - "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", // inner hamt nodes start here "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", diff --git a/lib/files.go b/lib/files.go index be17c32..05e5f2e 100644 --- a/lib/files.go +++ b/lib/files.go @@ -70,7 +70,7 @@ func (b *backpressuredFile) Read(p []byte) (n int, err error) { if err != nil { return 0, err } - nd, err := tv(b.ctx, b.fileCid, nil, nil, gateway.CarParams{Scope: gateway.DagScopeEntity, Range: &gateway.DagByteRange{From: from, To: b.byteRange.To}}, b.getLsys) + nd, err := loadTerminalUnixFSElementWithRecursiveDirectories(b.ctx, b.fileCid, nil, nil, gateway.CarParams{Scope: gateway.DagScopeEntity, Range: &gateway.DagByteRange{From: from, To: b.byteRange.To}}, b.getLsys) if err != nil { return 0, err } @@ -166,7 +166,7 @@ func (it *backpressuredFlatDirIter) Next() bool { it.lsys, err = it.getLsys(it.ctx, c, params) continue } - nd, err = tv(it.ctx, c, nil, it.lsys, params, it.getLsys) + nd, err = loadTerminalUnixFSElementWithRecursiveDirectories(it.ctx, c, nil, it.lsys, params, it.getLsys) if err != nil { if err := it.ctx.Err(); err == nil { retry, processedErr := isRetryableError(err) @@ -184,16 +184,6 @@ func (it *backpressuredFlatDirIter) Next() bool { it.curName = name it.curFile = nd return true - - /* - Load link, figure out what it is: - 1. If it's a symlink just stash it - 2. If it's a raw block stash it? - 3. If it's metadata then skip it - 4. If it's a file return a wrapper around multireadcloser that waits for new lsys' to show up. There's got to be a way to feedback through the top requests for new lsys' along with the request we're waiting on for the retry - 5. If it's a flat directory stash it (will need to carry the feedback layer for its children) - 6. If it's a HAMT return an iterator that for "next" sends a request and waits for an lsys if there's a problem. Create the next object and return it passing through the request + response mechanisms for continuations - */ } func (it *backpressuredFlatDirIter) Err() error { @@ -338,7 +328,7 @@ func (it *backpressuredHAMTDirIter) Next() bool { continue } - childNd, err = tv(it.ctx, c, nil, it.lsys, params, it.getLsys) + childNd, err = loadTerminalUnixFSElementWithRecursiveDirectories(it.ctx, c, nil, it.lsys, params, it.getLsys) if err != nil { continue } @@ -352,16 +342,6 @@ func (it *backpressuredHAMTDirIter) Next() bool { } return true - - /* - Load link, figure out what it is: - 1. If it's a symlink just stash it - 2. If it's a raw block stash it? - 3. If it's metadata then skip it - 4. If it's a file return a wrapper around multireadcloser that waits for new lsys' to show up. There's got to be a way to feedback through the top requests for new lsys' along with the request we're waiting on for the retry - 5. If it's a flat directory stash it (will need to carry the feedback layer for its children) - 6. If it's a HAMT return an iterator that for "next" sends a request and waits for an lsys if there's a problem. Create the next object and return it passing through the request + response mechanisms for continuations - */ } func (it *backpressuredHAMTDirIter) Err() error { @@ -438,7 +418,7 @@ func loadUnixFSBase(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld } } -func tv(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld.LinkSystem, params gateway.CarParams, getLsys lsysGetter) (files.Node, error) { +func loadTerminalUnixFSElementWithRecursiveDirectories(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld.LinkSystem, params gateway.CarParams, getLsys lsysGetter) (files.Node, error) { var err error if lsys == nil { lsys, err = getLsys(ctx, c, params) diff --git a/lib/gateway_traversal.go b/lib/gateway_traversal.go index 231118e..8c71cfc 100644 --- a/lib/gateway_traversal.go +++ b/lib/gateway_traversal.go @@ -9,9 +9,6 @@ import ( "sync" "time" - "github.com/ipld/go-ipld-prime/linking" - - "github.com/ipfs/boxo/exchange" bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/gateway" blocks "github.com/ipfs/go-block-format" @@ -22,6 +19,7 @@ import ( dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/schema" @@ -32,35 +30,6 @@ import ( type getBlock func(ctx context.Context, cid cid.Cid) (blocks.Block, error) -type gbf struct { - fn getBlock -} - -func (g *gbf) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - return g.fn(ctx, c) -} - -func (g *gbf) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { - ch := make(chan blocks.Block, len(cids)) - go func() { - defer close(ch) - for _, c := range cids { - blk, err := g.fn(ctx, c) - if err != nil { - return - } - select { - case ch <- blk: - case <-ctx.Done(): - return - } - } - }() - return ch, nil -} - -var _ exchange.Fetcher = (*gbf)(nil) - func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *GraphGatewayMetrics) (getBlock, error) { cr, err := car.NewCarReader(reader) if err != nil { diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index a49c108..8a723e9 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -14,11 +14,8 @@ import ( "sync" "time" - "github.com/hashicorp/go-multierror" - - "github.com/ipld/go-ipld-prime/traversal" - "github.com/filecoin-saturn/caboose" + "github.com/hashicorp/go-multierror" blockstore "github.com/ipfs/boxo/blockstore" nsopts "github.com/ipfs/boxo/coreiface/options/namesys" ifacepath "github.com/ipfs/boxo/coreiface/path" @@ -36,6 +33,7 @@ import ( format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-unixfsnode" ufsData "github.com/ipfs/go-unixfsnode/data" + "github.com/ipfs/go-unixfsnode/hamt" carv2 "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/storage" dagpb "github.com/ipld/go-codec-dagpb" @@ -44,6 +42,7 @@ import ( cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/schema" + "github.com/ipld/go-ipld-prime/traversal" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" @@ -302,8 +301,8 @@ func (api *GraphGateway) fetchCAR(ctx context.Context, path gateway.ImmutablePat } // resolvePathWithRootsAndBlock takes a path and linksystem and returns the set of non-terminal cids, the terminal cid, the remainder, and the block corresponding to the terminal cid -func (api *GraphGateway) resolvePathWithRootsAndBlock(ctx context.Context, fpath ipfspath.Path, unixFSLsys *ipld.LinkSystem) ([]cid.Cid, cid.Cid, []string, blocks.Block, error) { - pathRootCids, terminalCid, remainder, terminalBlk, err := api.resolvePathToLastWithRoots(ctx, fpath, unixFSLsys) +func resolvePathWithRootsAndBlock(ctx context.Context, fpath ipfspath.Path, unixFSLsys *ipld.LinkSystem) ([]cid.Cid, cid.Cid, []string, blocks.Block, error) { + pathRootCids, terminalCid, remainder, terminalBlk, err := resolvePathToLastWithRoots(ctx, fpath, unixFSLsys) if err != nil { return nil, cid.Undef, nil, nil, err } @@ -328,7 +327,7 @@ func (api *GraphGateway) resolvePathWithRootsAndBlock(ctx context.Context, fpath // the remainder pathing, the last block loaded, and the last node loaded. // // Note: the block returned will be nil if the terminal element is a link or the path is just a CID -func (api *GraphGateway) resolvePathToLastWithRoots(ctx context.Context, fpath ipfspath.Path, unixFSLsys *ipld.LinkSystem) ([]cid.Cid, cid.Cid, []string, blocks.Block, error) { +func resolvePathToLastWithRoots(ctx context.Context, fpath ipfspath.Path, unixFSLsys *ipld.LinkSystem) ([]cid.Cid, cid.Cid, []string, blocks.Block, error) { c, p, err := ipfspath.SplitAbsPath(fpath) if err != nil { return nil, cid.Undef, nil, nil, err @@ -344,10 +343,17 @@ func (api *GraphGateway) resolvePathToLastWithRoots(ctx context.Context, fpath i var cids []cid.Cid cids = append(cids, c) + pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + loadNode := func(ctx context.Context, c cid.Cid) (blocks.Block, ipld.Node, error) { lctx := ipld.LinkContext{Ctx: ctx} rootLnk := cidlink.Link{Cid: c} - np, err := api.pc(rootLnk, lctx) + np, err := pc(rootLnk, lctx) if err != nil { return nil, nil, err } @@ -447,372 +453,360 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by } } - p := ipfspath.FromString(path.String()) - - type terminalPathType struct { - resp *gateway.GetResponse - err error - md gateway.ContentPathMetadata + md, terminalElem, err := fetchWithPartialRetries(ctx, path, carParams, loadTerminalEntity, api.metrics, api.fetchCAR) + if err != nil { + return gateway.ContentPathMetadata{}, nil, err } - terminalPathElementCh := make(chan terminalPathType, 1) + var resp *gateway.GetResponse - var terminalFile *multiReadCloser - var terminalDir chan unixfs.LinkResult - lastDirLinkNum := 0 - hasSentAsyncData := false - go func() { - cctx, cancel := context.WithCancel(ctx) - defer cancel() - err := api.fetchCAR(cctx, path, carParams, func(resource string, reader io.Reader) error { - gb, err := carToLinearBlockGetter(cctx, reader, api.metrics) - if err != nil { - return err - } + switch typedTerminalElem := terminalElem.(type) { + case *gateway.GetResponse: + resp = typedTerminalElem + case *backpressuredFile: + resp = gateway.NewGetResponseFromReader(typedTerminalElem, typedTerminalElem.size) + case *backpressuredHAMTDirIterNoRecursion: + iter, ok := terminalElem.(*backpressuredHAMTDirIterNoRecursion) + if !ok { + } - lsys := getLinksystem(gb) - // First resolve the path since we always need to. - pathRootCids, terminalCid, remainder, terminalBlk, err := api.resolvePathWithRootsAndBlock(cctx, p, lsys) - if err != nil { - return err + ch := make(chan unixfs.LinkResult) + go func() { + defer close(ch) + for iter.Next() { + l := iter.Link() + select { + case ch <- l: + case <-ctx.Done(): + return + } } - md := contentMetadataFromRootsAndRemainder(p, pathRootCids, terminalCid, remainder) + if err := iter.Err(); err != nil { + select { + case ch <- unixfs.LinkResult{Err: err}: + case <-ctx.Done(): + return + } + } + }() + resp = gateway.NewGetResponseFromDirectoryListing(iter.dagSize, ch, nil) + default: + return gateway.ContentPathMetadata{}, nil, fmt.Errorf("invalid data type") + } - lctx := ipld.LinkContext{Ctx: cctx} - pathTerminalCidLink := cidlink.Link{Cid: terminalCid} + return md, resp, nil - if len(remainder) > 0 { - terminalPathElementCh <- terminalPathType{err: errNotUnixFS} - return nil - } +} - // From now on, dag-scope=entity! - // Since we need more of the graph load it to figure out what we have - // This includes determining if the terminal node is UnixFS or not - np, err := api.pc(pathTerminalCidLink, lctx) +// loadTerminalEntity returns either a [*gateway.GetResponse], [*backpressuredFile], or [*backpressuredHAMTDirIterNoRecursion] +func loadTerminalEntity(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld.LinkSystem, params gateway.CarParams, getLsys lsysGetter) (interface{}, error) { + var err error + if lsys == nil { + lsys, err = getLsys(ctx, c, params) + if err != nil { + return nil, err + } + } + + lctx := ipld.LinkContext{Ctx: ctx} + + if c.Type() != uint64(multicodec.DagPb) { + var blockData []byte + + if blk != nil { + blockData = blk.RawData() + } else { + blockData, err = lsys.LoadRaw(lctx, cidlink.Link{Cid: c}) if err != nil { - return err + return nil, err } + } + + return gateway.NewGetResponseFromReader(files.NewBytesFile(blockData), int64(len(blockData))), nil + } + + blockData, pbn, _, fieldNum, fieldDataBytes, err := loadUnixFSBase(ctx, c, blk, lsys) + if err != nil { + return nil, err + } + + switch fieldNum { + case ufsData.Data_Symlink: + lnkTarget := string(fieldDataBytes) + f := gateway.NewGetResponseFromSymlink(files.NewLinkFile(lnkTarget, nil).(*files.Symlink), int64(len(fieldDataBytes))) + return f, nil + case ufsData.Data_Metadata: + return nil, fmt.Errorf("UnixFS Metadata unsupported") + case ufsData.Data_HAMTShard, ufsData.Data_Directory: + blk, err := blocks.NewBlockWithCid(blockData, c) + if err != nil { + return nil, fmt.Errorf("could not create block: %w", err) + } + dirRootNd, err := merkledag.ProtoNodeConverter(blk, pbn) + if err != nil { + return nil, fmt.Errorf("could not create dag-pb universal block from UnixFS directory root: %w", err) + } + pn, ok := dirRootNd.(*merkledag.ProtoNode) + if !ok { + return nil, fmt.Errorf("could not create dag-pb node from UnixFS directory root: %w", err) + } + + dirDagSize, err := pn.Size() + if err != nil { + return nil, fmt.Errorf("could not get cumulative size from dag-pb node: %w", err) + } - decoder, err := lsys.DecoderChooser(pathTerminalCidLink) + switch fieldNum { + case ufsData.Data_Directory: + ch := make(chan unixfs.LinkResult, pbn.Links.Length()) + defer close(ch) + iter := pbn.Links.Iterator() + for !iter.Done() { + _, v := iter.Next() + c := v.Hash.Link().(cidlink.Link).Cid + var name string + var size int64 + if v.Name.Exists() { + name = v.Name.Must().String() + } + if v.Tsize.Exists() { + size = v.Tsize.Must().Int() + } + lnk := unixfs.LinkResult{Link: &format.Link{ + Name: name, + Size: uint64(size), + Cid: c, + }} + ch <- lnk + } + return gateway.NewGetResponseFromDirectoryListing(dirDagSize, ch, nil), nil + case ufsData.Data_HAMTShard: + dirNd, err := unixfsnode.Reify(lctx, pbn, lsys) if err != nil { - return err + return nil, fmt.Errorf("could not reify sharded directory: %w", err) } - nb := np.NewBuilder() - blockData := terminalBlk.RawData() - if err := decoder(nb, bytes.NewReader(blockData)); err != nil { - return err + + d := &backpressuredHAMTDirIterNoRecursion{ + dagSize: dirDagSize, + linksItr: dirNd.MapIterator(), + dirCid: c, + lsys: lsys, + getLsys: getLsys, + ctx: ctx, + closed: make(chan error), + hasClosed: false, } - lastCidNode := nb.Build() + return d, nil + default: + return nil, fmt.Errorf("not a basic or HAMT directory: should be unreachable") + } + case ufsData.Data_Raw, ufsData.Data_File: + nd, err := unixfsnode.Reify(lctx, pbn, lsys) + if err != nil { + return nil, err + } - if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { - // If it's not valid dag-pb then we're done - terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromReader(files.NewBytesFile(blockData), int64(len(blockData))), md: md} - return nil - } else if len(remainder) > 0 { - // If we're trying to path into dag-pb node that's invalid and we're done - terminalPathElementCh <- terminalPathType{err: fmt.Errorf("cannot path into non-UnixFS dagpb")} - return nil - } else if !pbn.FieldData().Exists() { - // If it's not valid UnixFS then we're done - terminalPathElementCh <- terminalPathType{err: fmt.Errorf("not valid UnixFS")} - return nil - } else if unixfsFieldData, decodeErr := ufsData.DecodeUnixFSData(pbn.Data.Must().Bytes()); decodeErr != nil { - // If it's not valid dag-pb and UnixFS then we're done - terminalPathElementCh <- terminalPathType{err: fmt.Errorf("not valid UnixFS")} - return nil - } else { - switch fieldNum := unixfsFieldData.FieldDataType().Int(); fieldNum { - case ufsData.Data_Symlink: - fd := unixfsFieldData.FieldData() - if fd.Exists() { - lnkTarget := string(fd.Must().Bytes()) - f := files.NewLinkFile(lnkTarget, nil) - s, ok := f.(*files.Symlink) - if !ok { - terminalPathElementCh <- terminalPathType{err: fmt.Errorf("should be unreachable: symlink does not have a symlink type")} - return nil - } - terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromSymlink(s, int64(len(lnkTarget))), md: md} - return nil - } - terminalPathElementCh <- terminalPathType{err: fmt.Errorf("UnixFS Symlink does not contain target")} - return nil - case ufsData.Data_Metadata: - terminalPathElementCh <- terminalPathType{err: fmt.Errorf("UnixFS Metadata unsupported")} - return nil - case ufsData.Data_HAMTShard, ufsData.Data_Directory: - blk, err := blocks.NewBlockWithCid(blockData, pathTerminalCidLink.Cid) - if err != nil { - terminalPathElementCh <- terminalPathType{err: fmt.Errorf("could not create block: %w", err)} - return nil - } - dirRootNd, err := merkledag.ProtoNodeConverter(blk, lastCidNode) - if err != nil { - terminalPathElementCh <- terminalPathType{err: fmt.Errorf("could not create dag-pb universal block from UnixFS directory root: %w", err)} - return nil - } - pn, ok := dirRootNd.(*merkledag.ProtoNode) - if !ok { - terminalPathElementCh <- terminalPathType{err: fmt.Errorf("could not create dag-pb node from UnixFS directory root: %w", err)} - return nil - } + fnd, ok := nd.(datamodel.LargeBytesNode) + if !ok { + return nil, fmt.Errorf("could not process file since it did not present as large bytes") + } + f, err := fnd.AsLargeBytes() + if err != nil { + return nil, err + } - sz, err := pn.Size() - if err != nil { - terminalPathElementCh <- terminalPathType{err: fmt.Errorf("could not get cumulative size from dag-pb node: %w", err)} - return nil - } + fileSize, err := f.Seek(0, io.SeekEnd) + if err != nil { + return nil, fmt.Errorf("unable to get UnixFS file size: %w", err) + } - if terminalDir == nil { - terminalDir = make(chan unixfs.LinkResult) - terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromDirectoryListing(sz, terminalDir, nil), md: md} - } + from := int64(0) + if params.Range != nil { + from = params.Range.From + } + _, err = f.Seek(from, io.SeekStart) + if err != nil { + return nil, fmt.Errorf("unable to get reset UnixFS file reader: %w", err) + } - dirLinkNum := 0 - switch fieldNum { - case ufsData.Data_Directory: - iter := pbn.Links.Iterator() - for !iter.Done() { - _, v := iter.Next() - c := v.Hash.Link().(cidlink.Link).Cid - var name string - var size int64 - if v.Name.Exists() { - name = v.Name.Must().String() - } - if v.Tsize.Exists() { - size = v.Tsize.Must().Int() - } - lnk := unixfs.LinkResult{Link: &format.Link{ - Name: name, - Size: uint64(size), - Cid: c, - }} - - dirLinkNum++ - if dirLinkNum-1 < lastDirLinkNum { - continue - } - - select { - case terminalDir <- lnk: - lastDirLinkNum++ - case <-cctx.Done(): - // TODO: what here, send an error with another select? - return nil - } - } - case ufsData.Data_HAMTShard: - hasSentAsyncData = true - // Note: we are making up the entries - dirNd, err := unixfsnode.Reify(lctx, pbn, lsys) - if err != nil { - select { - case terminalDir <- unixfs.LinkResult{Err: fmt.Errorf("could not reify sharded directory: %w", err)}: - case <-cctx.Done(): - // TODO: what here? - } - return nil - } + return &backpressuredFile{ctx: ctx, fileCid: c, size: fileSize, f: f, getLsys: getLsys, closed: make(chan error)}, nil + default: + return nil, fmt.Errorf("unknown UnixFS field type") + } +} - mi := dirNd.MapIterator() - for !mi.Done() { - k, v, err := mi.Next() - if err != nil { - return err - } - keyStr, err := k.AsString() - if err != nil { - select { - case terminalDir <- unixfs.LinkResult{Err: fmt.Errorf("could not interpret directory key as string: %w", err)}: - case <-cctx.Done(): - // TODO: what here? - } - return nil - } - valLink, err := v.AsLink() - if err != nil { - select { - case terminalDir <- unixfs.LinkResult{Err: fmt.Errorf("could not interpret directory value as link: %w", err)}: - case <-cctx.Done(): - // TODO: what here? - } - return nil - } - valCid := valLink.(cidlink.Link).Cid - lnk := unixfs.LinkResult{Link: &format.Link{ - Name: keyStr, - Size: uint64(0), - Cid: valCid, - }} - - dirLinkNum++ - if dirLinkNum-1 < lastDirLinkNum { - continue - } - - select { - case terminalDir <- lnk: - lastDirLinkNum++ - case <-cctx.Done(): - // TODO: what here? - } - } - } - return nil - case ufsData.Data_Raw, ufsData.Data_File: - nd, err := unixfsnode.Reify(lctx, lastCidNode, lsys) - if err != nil { - return err - } +type backpressuredHAMTDirIterNoRecursion struct { + dagSize uint64 + linksItr ipld.MapIterator + dirCid cid.Cid - fnd, ok := nd.(datamodel.LargeBytesNode) - if !ok { - return fmt.Errorf("could not process file since it did not present as large bytes") - } - f, err := fnd.AsLargeBytes() - if err != nil { - return err - } + lsys *ipld.LinkSystem + getLsys lsysGetter + ctx context.Context - fileSize, err := f.Seek(0, io.SeekEnd) - if err != nil { - terminalPathElementCh <- terminalPathType{err: fmt.Errorf("unable to get UnixFS file size: %w", err)} - return nil - } - _, err = f.Seek(0, io.SeekStart) - if err != nil { - terminalPathElementCh <- terminalPathType{err: fmt.Errorf("unable to get reset UnixFS file reader: %w", err)} - return nil - } + curLnk unixfs.LinkResult + curProcessed int - if terminalFile == nil { - mrc := &multiReadCloser{ - closeFn: nil, - mr: f, - closed: make(chan struct{}), - newReader: make(chan io.Reader), - retErr: nil, - isClosed: false, - } - terminalFile = mrc - terminalPathElementCh <- terminalPathType{resp: gateway.NewGetResponseFromReader(files.NewReaderFile(mrc), fileSize), md: md} - hasSentAsyncData = true - } else { - select { - case terminalFile.newReader <- f: - case <-cctx.Done(): - // TODO: what here? - return nil - } - } + closed chan error + hasClosed bool + err error +} - select { - case <-terminalFile.closed: - return nil - case <-cctx.Done(): - // TODO: what here? - } - return nil - default: - terminalPathElementCh <- terminalPathType{err: fmt.Errorf("unknown UnixFS field type")} - return nil - } - } - }) - if err != nil { - if !hasSentAsyncData { - terminalPathElementCh <- terminalPathType{err: fmt.Errorf("failed fetch: %w", err)} - } else if terminalFile != nil { - // TODO: this error should surface through the io.Reader - } else if terminalDir != nil { - select { - case terminalDir <- unixfs.LinkResult{Err: err}: - case <-cctx.Done(): - // TODO: what here? - } +func (it *backpressuredHAMTDirIterNoRecursion) AwaitClose() <-chan error { + return it.closed +} + +func (it *backpressuredHAMTDirIterNoRecursion) Link() unixfs.LinkResult { + return it.curLnk +} + +func (it *backpressuredHAMTDirIterNoRecursion) Next() bool { + defer func() { + if it.linksItr.Done() || it.err != nil { + if !it.hasClosed { + it.hasClosed = true + close(it.closed) } } - if terminalDir != nil { - close(terminalDir) - } }() - select { - case t := <-terminalPathElementCh: - if t.err != nil { - return gateway.ContentPathMetadata{}, nil, t.err - } + if it.err != nil { + return false + } - return t.md, t.resp, nil - case <-ctx.Done(): - return gateway.ContentPathMetadata{}, nil, ctx.Err() + iter := it.linksItr + if iter.Done() { + return false } -} -type multiReadCloser struct { - closeFn func() error - mr io.Reader - closed chan struct{} - newReader chan io.Reader - retErr error - isClosed bool -} + /* + Since there is no way to make a graph request for part of a HAMT during errors we can either fill in the HAMT with + block requests, or we can re-request the HAMT and skip over the parts we already have. + + Here we choose the latter, however in the event of a re-request we request the entity rather than the entire DAG as + a compromise between more requests and over-fetching data. + */ -func (r *multiReadCloser) Read(p []byte) (n int, err error) { - if r.retErr == nil { - n, err = r.mr.Read(p) - if err == nil || err == io.EOF { - return n, err + var err error + for { + if it.ctx.Err() != nil { + it.err = it.ctx.Err() + return false } - if n > 0 { - r.retErr = err - return n, nil + retry, processedErr := isRetryableError(err) + if !retry { + it.err = processedErr + return false } - } else { - err = r.retErr - } - select { - case <-r.closed: - return n, err - case newReader, ok := <-r.newReader: - if ok { - r.mr = io.MultiReader(r.mr, newReader) - return r.Read(p) + var nd ipld.Node + if err != nil { + var lsys *ipld.LinkSystem + lsys, err = it.getLsys(it.ctx, it.dirCid, gateway.CarParams{Scope: gateway.DagScopeEntity}) + if err != nil { + continue + } + + _, pbn, fieldData, _, _, ufsBaseErr := loadUnixFSBase(it.ctx, it.dirCid, nil, lsys) + if ufsBaseErr != nil { + err = ufsBaseErr + continue + } + + nd, err = hamt.NewUnixFSHAMTShard(it.ctx, pbn, fieldData, lsys) + if err != nil { + err = fmt.Errorf("could not reify sharded directory: %w", err) + continue + } + + iter = nd.MapIterator() + for i := 0; i < it.curProcessed; i++ { + _, _, err = iter.Next() + if err != nil { + continue + } + } + + it.linksItr = iter } - return n, err - } -} -func (r *multiReadCloser) Close() error { - if r.isClosed { - return nil - } + var k, v ipld.Node + k, v, err = iter.Next() + if err != nil { + retry, processedErr = isRetryableError(err) + if retry { + err = processedErr + continue + } + it.err = processedErr + return false + } - close(r.newReader) - close(r.closed) + var name string + name, err = k.AsString() + if err != nil { + it.err = err + return false + } + var lnk ipld.Link + lnk, err = v.AsLink() + if err != nil { + it.err = err + return false + } - if r.closeFn == nil { - return nil + cl, ok := lnk.(cidlink.Link) + if !ok { + it.err = fmt.Errorf("link not a cidlink") + return false + } + + c := cl.Cid + + it.curLnk = unixfs.LinkResult{ + Link: &format.Link{ + Name: name, + Size: 0, + Cid: c, + }, + } + it.curProcessed++ + break } - return r.closeFn() + + return true +} + +func (it *backpressuredHAMTDirIterNoRecursion) Err() error { + return it.err } -var _ io.ReadCloser = (*multiReadCloser)(nil) +var _ AwaitCloser = (*backpressuredHAMTDirIterNoRecursion)(nil) func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "all", "entityRanges": "0"}).Inc() + return fetchWithPartialRetries(ctx, path, gateway.CarParams{Scope: gateway.DagScopeAll}, loadTerminalUnixFSElementWithRecursiveDirectories, api.metrics, api.fetchCAR) +} - type terminalPathType struct { - resp files.Node - err error - md gateway.ContentPathMetadata - } +type loadTerminalElement[T any] func(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld.LinkSystem, params gateway.CarParams, getLsys lsysGetter) (T, error) +type fetchCarFn = func(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams, cb DataCallback) error + +type terminalPathType[T any] struct { + resp T + err error + md gateway.ContentPathMetadata +} + +type nextReq struct { + c cid.Cid + params gateway.CarParams +} - terminalPathElementCh := make(chan terminalPathType, 1) +func fetchWithPartialRetries[T any](ctx context.Context, path gateway.ImmutablePath, initialParams gateway.CarParams, resolveTerminalElementFn loadTerminalElement[T], metrics *GraphGatewayMetrics, fetchCAR fetchCarFn) (gateway.ContentPathMetadata, T, error) { + var zeroReturnType T + + terminalPathElementCh := make(chan terminalPathType[T], 1) go func() { cctx, cancel := context.WithCancel(ctx) @@ -821,11 +815,6 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) hasSentAsyncData := false var closeCh <-chan error - type nextReq struct { - c cid.Cid - params gateway.CarParams - } - sendRequest := make(chan nextReq, 1) sendResponse := make(chan *ipld.LinkSystem, 1) getLsys := func(ctx context.Context, c cid.Cid, params gateway.CarParams) (*ipld.LinkSystem, error) { @@ -844,10 +833,10 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) } p := ipfspath.FromString(path.String()) - params := gateway.CarParams{Scope: gateway.DagScopeAll} + params := initialParams - err := api.fetchCAR(cctx, path, params, func(resource string, reader io.Reader) error { - gb, err := carToLinearBlockGetter(cctx, reader, api.metrics) + err := fetchCAR(cctx, path, params, func(resource string, reader io.Reader) error { + gb, err := carToLinearBlockGetter(cctx, reader, metrics) if err != nil { return err } @@ -855,7 +844,7 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) lsys := getLinksystem(gb) if hasSentAsyncData { - _, _, _, _, err = api.resolvePathToLastWithRoots(cctx, p, lsys) + _, _, _, _, err = resolvePathToLastWithRoots(cctx, p, lsys) if err != nil { return err } @@ -867,14 +856,14 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) } } else { // First resolve the path since we always need to. - pathRootCids, terminalCid, remainder, terminalBlk, err := api.resolvePathWithRootsAndBlock(cctx, p, lsys) + pathRootCids, terminalCid, remainder, terminalBlk, err := resolvePathWithRootsAndBlock(cctx, p, lsys) if err != nil { return err } md := contentMetadataFromRootsAndRemainder(p, pathRootCids, terminalCid, remainder) if len(remainder) > 0 { - terminalPathElementCh <- terminalPathType{err: errNotUnixFS} + terminalPathElementCh <- terminalPathType[T]{err: errNotUnixFS} return nil } @@ -886,14 +875,14 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) } } - nd, err := tv(cctx, terminalCid, terminalBlk, lsys, params, getLsys) + nd, err := resolveTerminalElementFn(cctx, terminalCid, terminalBlk, lsys, params, getLsys) if err != nil { return err } - ndAc, ok := nd.(AwaitCloser) + ndAc, ok := any(nd).(AwaitCloser) if !ok { - terminalPathElementCh <- terminalPathType{ + terminalPathElementCh <- terminalPathType[T]{ resp: nd, md: md, } @@ -901,7 +890,7 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) } hasSentAsyncData = true - terminalPathElementCh <- terminalPathType{ + terminalPathElementCh <- terminalPathType[T]{ resp: nd, md: md, } @@ -924,7 +913,7 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) }) if !hasSentAsyncData && err != nil { - terminalPathElementCh <- terminalPathType{err: err} + terminalPathElementCh <- terminalPathType[T]{err: err} return } @@ -948,11 +937,11 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) select { case t := <-terminalPathElementCh: if t.err != nil { - return gateway.ContentPathMetadata{}, nil, t.err + return gateway.ContentPathMetadata{}, zeroReturnType, t.err } return t.md, t.resp, nil case <-ctx.Done(): - return gateway.ContentPathMetadata{}, nil, ctx.Err() + return gateway.ContentPathMetadata{}, zeroReturnType, ctx.Err() } } @@ -971,7 +960,7 @@ func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePat lsys := getLinksystem(gb) // First resolve the path since we always need to. - pathRoots, terminalCid, remainder, terminalBlk, err := api.resolvePathToLastWithRoots(ctx, p, lsys) + pathRoots, terminalCid, remainder, terminalBlk, err := resolvePathToLastWithRoots(ctx, p, lsys) if err != nil { return err } @@ -1022,7 +1011,7 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) ( lsys := getLinksystem(gb) // First resolve the path since we always need to. - pathRoots, terminalCid, remainder, terminalBlk, err := api.resolvePathWithRootsAndBlock(ctx, p, lsys) + pathRoots, terminalCid, remainder, terminalBlk, err := resolvePathWithRootsAndBlock(ctx, p, lsys) if err != nil { return err } @@ -1155,7 +1144,7 @@ func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.Immutable // First resolve the path since we always need to. p := ipfspath.FromString(path.String()) - pathRoots, terminalCid, remainder, _, err := api.resolvePathToLastWithRoots(ctx, p, lsys) + pathRoots, terminalCid, remainder, _, err := resolvePathToLastWithRoots(ctx, p, lsys) if err != nil { return err } @@ -1217,7 +1206,7 @@ func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath, l := getLinksystem(teeBlock) // First resolve the path since we always need to. - _, terminalCid, remainder, terminalBlk, err := api.resolvePathWithRootsAndBlock(ctx, p, l) + _, terminalCid, remainder, terminalBlk, err := resolvePathWithRootsAndBlock(ctx, p, l) if err != nil { return err } From 5a99a19add85d7b307ef0dbb65ec573d519bef1c Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 24 Jul 2023 16:14:21 -0400 Subject: [PATCH 16/46] feat: allow passing a custom prometheus registry to the graph gateway backend, and at the library level stop using the global default registry by default --- handlers.go | 2 +- lib/graph_gateway.go | 39 ++++++++++++++++++++++++++------------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/handlers.go b/handlers.go index 0397a39..49e8b68 100644 --- a/handlers.go +++ b/handlers.go @@ -104,7 +104,7 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC return nil, err } - gwAPI, err = lib.NewGraphGatewayBackend(bs.(lib.CarFetcher), exch, lib.WithValueStore(routing), lib.WithBlockstore(cacheBlockStore)) + gwAPI, err = lib.NewGraphGatewayBackend(bs.(lib.CarFetcher), exch, lib.WithValueStore(routing), lib.WithBlockstore(cacheBlockStore), lib.WithPrometheusRegistry(prometheus.DefaultRegisterer)) if err != nil { return nil, err } diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 8a723e9..fa78f3d 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -66,9 +66,10 @@ type CarFetcher interface { } type gwOptions struct { - ns namesys.NameSystem - vs routing.ValueStore - bs blockstore.Blockstore + ns namesys.NameSystem + vs routing.ValueStore + bs blockstore.Blockstore + promRegistry prometheus.Registerer } // WithNameSystem sets the name system to use for the gateway. If not set it will use a default DNSLink resolver @@ -96,6 +97,14 @@ func WithBlockstore(bs blockstore.Blockstore) GraphGatewayOption { } } +// WithPrometheusRegistry sets the registry to use for metrics collection +func WithPrometheusRegistry(reg prometheus.Registerer) GraphGatewayOption { + return func(opts *gwOptions) error { + opts.promRegistry = reg + return nil + } +} + type GraphGatewayOption func(gwOptions *gwOptions) error type GraphGateway struct { @@ -162,6 +171,11 @@ func NewGraphGatewayBackend(f CarFetcher, blockFetcher exchange.Fetcher, opts .. bs = cbs } + var promReg prometheus.Registerer = prometheus.NewRegistry() + if compiledOptions.promRegistry != nil { + promReg = compiledOptions.promRegistry + } + return &GraphGateway{ fetcher: f, blockFetcher: blockFetcher, @@ -169,7 +183,7 @@ func NewGraphGatewayBackend(f CarFetcher, blockFetcher exchange.Fetcher, opts .. namesys: ns, bstore: bs, notifiers: sync.Map{}, - metrics: registerGraphGatewayMetrics(), + metrics: registerGraphGatewayMetrics(promReg), pc: dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { return tlnkNd.LinkTargetNodePrototype(), nil @@ -179,8 +193,7 @@ func NewGraphGatewayBackend(f CarFetcher, blockFetcher exchange.Fetcher, opts .. }, nil } -func registerGraphGatewayMetrics() *GraphGatewayMetrics { - +func registerGraphGatewayMetrics(registerer prometheus.Registerer) *GraphGatewayMetrics { // How many CAR Fetch attempts we had? Need this to calculate % of various graph request types. // We only count attempts here, because success/failure with/without retries are provided by caboose: // - ipfs_caboose_fetch_duration_car_success_count @@ -193,7 +206,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { Name: "car_fetch_attempts", Help: "The number of times a CAR fetch was attempted by IPFSBackend.", }) - prometheus.MustRegister(carFetchAttemptMetric) + registerer.MustRegister(carFetchAttemptMetric) contextAlreadyCancelledMetric := prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "ipfs", @@ -201,7 +214,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { Name: "car_fetch_context_already_cancelled", Help: "The number of times context is already cancelled when a CAR fetch was attempted by IPFSBackend.", }) - prometheus.MustRegister(contextAlreadyCancelledMetric) + registerer.MustRegister(contextAlreadyCancelledMetric) // How many blocks were read via CARs? // Need this as a baseline to reason about error ratio vs raw_block_recovery_attempts. @@ -211,7 +224,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { Name: "car_blocks_fetched", Help: "The number of blocks successfully read via CAR fetch.", }) - prometheus.MustRegister(carBlocksFetchedMetric) + registerer.MustRegister(carBlocksFetchedMetric) // How many times CAR response was not enough or just failed, and we had to read a block via ?format=raw // We only count attempts here, because success/failure with/without retries are provided by caboose: @@ -225,7 +238,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { Name: "raw_block_recovery_attempts", Help: "The number of ?format=raw block fetch attempts due to GraphGateway failure (CAR fetch error, missing block in CAR response, or a block evicted from cache too soon).", }) - prometheus.MustRegister(blockRecoveryAttemptMetric) + registerer.MustRegister(blockRecoveryAttemptMetric) carParamsMetric := prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "ipfs", @@ -233,7 +246,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { Name: "car_fetch_params", Help: "How many times specific CAR parameter was used during CAR data fetch.", }, []string{"dagScope", "entityRanges"}) // we use 'ranges' instead of 'bytes' here because we only count the number of ranges present - prometheus.MustRegister(carParamsMetric) + registerer.MustRegister(carParamsMetric) bytesRangeStartMetric := prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "ipfs", @@ -242,7 +255,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { Help: "Tracks where did the range request start.", Buckets: prometheus.ExponentialBuckets(1024, 2, 24), // 1024 bytes to 8 GiB }) - prometheus.MustRegister(bytesRangeStartMetric) + registerer.MustRegister(bytesRangeStartMetric) bytesRangeSizeMetric := prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "ipfs", @@ -251,7 +264,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { Help: "Tracks the size of range requests.", Buckets: prometheus.ExponentialBuckets(256*1024, 2, 10), // From 256KiB to 100MiB }) - prometheus.MustRegister(bytesRangeSizeMetric) + registerer.MustRegister(bytesRangeSizeMetric) return &GraphGatewayMetrics{ contextAlreadyCancelledMetric, From d7383de017d3e2ab93254a97c5c3b3756a496af9 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 24 Jul 2023 17:19:39 -0400 Subject: [PATCH 17/46] fix: remove unneeded block fetching remnants from graph backend. also some linter cleanup --- graph_gateway_test.go | 30 +++-------------- handlers.go | 8 +---- lib/graph_gateway.go | 78 ++++++++----------------------------------- 3 files changed, 20 insertions(+), 96 deletions(-) diff --git a/graph_gateway_test.go b/graph_gateway_test.go index 6721d3c..adcefa6 100644 --- a/graph_gateway_test.go +++ b/graph_gateway_test.go @@ -155,11 +155,7 @@ func TestTar(t *testing.T) { defer s.Close() bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) - exch, err := newExchange(bs) - if err != nil { - t.Fatal(err) - } - backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}, exch) + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}) if err != nil { t.Fatal(err) } @@ -351,11 +347,7 @@ func TestTarAtEndOfPath(t *testing.T) { defer s.Close() bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) - exch, err := newExchange(bs) - if err != nil { - t.Fatal(err) - } - backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}, exch) + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}) if err != nil { t.Fatal(err) } @@ -533,11 +525,7 @@ func TestGetFile(t *testing.T) { defer s.Close() bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) - exch, err := newExchange(bs) - if err != nil { - t.Fatal(err) - } - backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}, exch) + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}) if err != nil { t.Fatal(err) } @@ -654,11 +642,7 @@ func TestGetHAMTDirectory(t *testing.T) { defer s.Close() bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) - exch, err := newExchange(bs) - if err != nil { - t.Fatal(err) - } - backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}, exch) + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}) if err != nil { t.Fatal(err) } @@ -769,11 +753,7 @@ func TestGetCAR(t *testing.T) { defer s.Close() bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) - exch, err := newExchange(bs) - if err != nil { - t.Fatal(err) - } - backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}, exch) + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}) if err != nil { t.Fatal(err) } diff --git a/handlers.go b/handlers.go index 49e8b68..fb844dd 100644 --- a/handlers.go +++ b/handlers.go @@ -98,13 +98,7 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC return nil, err } } else { - // Sets up an exchange based on the given Block Store - exch, err := newExchange(bs) - if err != nil { - return nil, err - } - - gwAPI, err = lib.NewGraphGatewayBackend(bs.(lib.CarFetcher), exch, lib.WithValueStore(routing), lib.WithBlockstore(cacheBlockStore), lib.WithPrometheusRegistry(prometheus.DefaultRegisterer)) + gwAPI, err = lib.NewGraphGatewayBackend(bs.(lib.CarFetcher), lib.WithValueStore(routing), lib.WithPrometheusRegistry(prometheus.DefaultRegisterer)) if err != nil { return nil, err } diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index fa78f3d..d797a81 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -11,15 +11,12 @@ import ( gopath "path" "strconv" "strings" - "sync" "time" "github.com/filecoin-saturn/caboose" "github.com/hashicorp/go-multierror" - blockstore "github.com/ipfs/boxo/blockstore" nsopts "github.com/ipfs/boxo/coreiface/options/namesys" ifacepath "github.com/ipfs/boxo/coreiface/path" - exchange "github.com/ipfs/boxo/exchange" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/gateway" "github.com/ipfs/boxo/ipld/merkledag" @@ -68,7 +65,6 @@ type CarFetcher interface { type gwOptions struct { ns namesys.NameSystem vs routing.ValueStore - bs blockstore.Blockstore promRegistry prometheus.Registerer } @@ -89,14 +85,6 @@ func WithValueStore(vs routing.ValueStore) GraphGatewayOption { } } -// WithBlockstore sets the Blockstore to use for the gateway -func WithBlockstore(bs blockstore.Blockstore) GraphGatewayOption { - return func(opts *gwOptions) error { - opts.bs = bs - return nil - } -} - // WithPrometheusRegistry sets the registry to use for metrics collection func WithPrometheusRegistry(reg prometheus.Registerer) GraphGatewayOption { return func(opts *gwOptions) error { @@ -108,30 +96,26 @@ func WithPrometheusRegistry(reg prometheus.Registerer) GraphGatewayOption { type GraphGatewayOption func(gwOptions *gwOptions) error type GraphGateway struct { - fetcher CarFetcher - blockFetcher exchange.Fetcher - routing routing.ValueStore - namesys namesys.NameSystem - bstore blockstore.Blockstore + fetcher CarFetcher + routing routing.ValueStore + namesys namesys.NameSystem pc traversal.LinkTargetNodePrototypeChooser - notifiers sync.Map // cid -> notifiersForRootCid - metrics *GraphGatewayMetrics + metrics *GraphGatewayMetrics } type GraphGatewayMetrics struct { contextAlreadyCancelledMetric prometheus.Counter carFetchAttemptMetric prometheus.Counter carBlocksFetchedMetric prometheus.Counter - blockRecoveryAttemptMetric prometheus.Counter carParamsMetric *prometheus.CounterVec bytesRangeStartMetric prometheus.Histogram bytesRangeSizeMetric prometheus.Histogram } -func NewGraphGatewayBackend(f CarFetcher, blockFetcher exchange.Fetcher, opts ...GraphGatewayOption) (*GraphGateway, error) { +func NewGraphGatewayBackend(f CarFetcher, opts ...GraphGatewayOption) (*GraphGateway, error) { var compiledOptions gwOptions for _, o := range opts { if err := o(&compiledOptions); err != nil { @@ -158,32 +142,16 @@ func NewGraphGatewayBackend(f CarFetcher, blockFetcher exchange.Fetcher, opts .. } } - bs := compiledOptions.bs - if compiledOptions.bs == nil { - // Sets up a cache to store blocks in - cbs, err := NewCacheBlockStore(DefaultCacheBlockStoreSize) - if err != nil { - return nil, err - } - - // Set up support for identity hashes (https://github.com/ipfs/bifrost-gateway/issues/38) - cbs = blockstore.NewIdStore(cbs) - bs = cbs - } - var promReg prometheus.Registerer = prometheus.NewRegistry() if compiledOptions.promRegistry != nil { promReg = compiledOptions.promRegistry } return &GraphGateway{ - fetcher: f, - blockFetcher: blockFetcher, - routing: vs, - namesys: ns, - bstore: bs, - notifiers: sync.Map{}, - metrics: registerGraphGatewayMetrics(promReg), + fetcher: f, + routing: vs, + namesys: ns, + metrics: registerGraphGatewayMetrics(promReg), pc: dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { return tlnkNd.LinkTargetNodePrototype(), nil @@ -226,20 +194,6 @@ func registerGraphGatewayMetrics(registerer prometheus.Registerer) *GraphGateway }) registerer.MustRegister(carBlocksFetchedMetric) - // How many times CAR response was not enough or just failed, and we had to read a block via ?format=raw - // We only count attempts here, because success/failure with/without retries are provided by caboose: - // - ipfs_caboose_fetch_duration_block_success_count - // - ipfs_caboose_fetch_duration_block_failure_count - // - ipfs_caboose_fetch_duration_block_peer_success_count - // - ipfs_caboose_fetch_duration_block_peer_failure_count - blockRecoveryAttemptMetric := prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "ipfs", - Subsystem: "gw_graph_backend", - Name: "raw_block_recovery_attempts", - Help: "The number of ?format=raw block fetch attempts due to GraphGateway failure (CAR fetch error, missing block in CAR response, or a block evicted from cache too soon).", - }) - registerer.MustRegister(blockRecoveryAttemptMetric) - carParamsMetric := prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "ipfs", Subsystem: "gw_graph_backend", @@ -270,7 +224,6 @@ func registerGraphGatewayMetrics(registerer prometheus.Registerer) *GraphGateway contextAlreadyCancelledMetric, carFetchAttemptMetric, carBlocksFetchedMetric, - blockRecoveryAttemptMetric, carParamsMetric, bytesRangeStartMetric, bytesRangeSizeMetric, @@ -299,6 +252,7 @@ func (api *GraphGateway) fetchCAR(ctx context.Context, path gateway.ImmutablePat paramsStr := paramsToString(params) urlWithoutHost := fmt.Sprintf("/%s?%s", escapedPath, paramsStr) + api.metrics.carFetchAttemptMetric.Inc() var ipldError error fetchErr := api.fetcher.Fetch(ctx, urlWithoutHost, func(resource string, reader io.Reader) error { return checkRetryableError(&ipldError, func() error { @@ -479,22 +433,18 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by case *backpressuredFile: resp = gateway.NewGetResponseFromReader(typedTerminalElem, typedTerminalElem.size) case *backpressuredHAMTDirIterNoRecursion: - iter, ok := terminalElem.(*backpressuredHAMTDirIterNoRecursion) - if !ok { - } - ch := make(chan unixfs.LinkResult) go func() { defer close(ch) - for iter.Next() { - l := iter.Link() + for typedTerminalElem.Next() { + l := typedTerminalElem.Link() select { case ch <- l: case <-ctx.Done(): return } } - if err := iter.Err(); err != nil { + if err := typedTerminalElem.Err(); err != nil { select { case ch <- unixfs.LinkResult{Err: err}: case <-ctx.Done(): @@ -502,7 +452,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by } } }() - resp = gateway.NewGetResponseFromDirectoryListing(iter.dagSize, ch, nil) + resp = gateway.NewGetResponseFromDirectoryListing(typedTerminalElem.dagSize, ch, nil) default: return gateway.ContentPathMetadata{}, nil, fmt.Errorf("invalid data type") } From 79523e729af4ff0fb42c499fcc362b7e5a3e14d3 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 24 Jul 2023 17:20:22 -0400 Subject: [PATCH 18/46] fix: resolve issue with file resumption on error and add test --- graph_gateway_test.go | 115 ++++++++++++++++++++++++++++++++++++++++++ lib/files.go | 57 ++++++++++++++------- 2 files changed, 153 insertions(+), 19 deletions(-) diff --git a/graph_gateway_test.go b/graph_gateway_test.go index adcefa6..d70ef96 100644 --- a/graph_gateway_test.go +++ b/graph_gateway_test.go @@ -568,6 +568,121 @@ func TestGetFile(t *testing.T) { } } +func TestGetFileWithBadBlockReturned(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + requestNum := 0 + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates at the root block + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + }); err != nil { + panic(err) + } + case 2: + // Expect the full request, but return a totally unrelated block + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // file root + }); err != nil { + panic(err) + } + case 3: + // Expect the full request and return most of the file + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. partial path and file range) + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", // file chunks start here + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + }); err != nil { + panic(err) + } + + case 4: + // Expect a request for the remainder of the file + // Note: this is an implementation detail, it could be that the requester really asks for more information + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", // middle of the file starts here + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + }); err != nil { + panic(err) + } + + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}) + if err != nil { + t.Fatal(err) + } + + trustedGatewayServer := httptest.NewServer(gateway.NewHandler(gateway.Config{DeserializedResponses: true}, backend)) + defer trustedGatewayServer.Close() + + resp, err := http.Get(trustedGatewayServer.URL + "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa") + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + robs, err := carbs.NewReadOnly(bytes.NewReader(dirWithMultiblockHAMTandFiles), nil) + if err != nil { + t.Fatal(err) + } + + dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) + fileRootNd, err := dsrv.Get(ctx, cid.MustParse("bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa")) + if err != nil { + t.Fatal(err) + } + uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd) + if err != nil { + t.Fatal(err) + } + f := uio.(files.File) + expectedFileData, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data, expectedFileData) { + t.Fatalf("expected %s, got %s", string(expectedFileData), string(data)) + } +} + func TestGetHAMTDirectory(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/lib/files.go b/lib/files.go index 05e5f2e..fdbffe3 100644 --- a/lib/files.go +++ b/lib/files.go @@ -66,21 +66,39 @@ func (b *backpressuredFile) Read(p []byte) (n int, err error) { err = b.retErr } - from, err := b.f.Seek(0, io.SeekCurrent) - if err != nil { - return 0, err - } - nd, err := loadTerminalUnixFSElementWithRecursiveDirectories(b.ctx, b.fileCid, nil, nil, gateway.CarParams{Scope: gateway.DagScopeEntity, Range: &gateway.DagByteRange{From: from, To: b.byteRange.To}}, b.getLsys) - if err != nil { - return 0, err + from, seekErr := b.f.Seek(0, io.SeekCurrent) + if seekErr != nil { + // Return the seek error since by this point seeking failures like this should be impossible + return 0, seekErr } - f, ok := nd.(files.File) - if !ok { - return 0, fmt.Errorf("not a file, should be unreachable") + // we had an error while reading so attempt to reset the underlying reader + for { + if b.ctx.Err() != nil { + return 0, b.ctx.Err() + } + + retry, processedErr := isRetryableError(err) + if !retry { + return 0, processedErr + } + + var nd files.Node + nd, err = loadTerminalUnixFSElementWithRecursiveDirectories(b.ctx, b.fileCid, nil, nil, gateway.CarParams{Scope: gateway.DagScopeEntity, Range: &gateway.DagByteRange{From: from, To: b.byteRange.To}}, b.getLsys) + if err != nil { + continue + } + + f, ok := nd.(files.File) + if !ok { + return 0, fmt.Errorf("not a file, should be unreachable") + } + + b.f = f + break } - b.f = f + // now that we've reset the reader try reading again return b.Read(p) } @@ -168,15 +186,16 @@ func (it *backpressuredFlatDirIter) Next() bool { } nd, err = loadTerminalUnixFSElementWithRecursiveDirectories(it.ctx, c, nil, it.lsys, params, it.getLsys) if err != nil { - if err := it.ctx.Err(); err == nil { - retry, processedErr := isRetryableError(err) - if retry { - err = processedErr - continue - } - it.err = processedErr - return false + if ctxErr := it.ctx.Err(); ctxErr != nil { + continue } + retry, processedErr := isRetryableError(err) + if retry { + err = processedErr + continue + } + it.err = processedErr + return false } break } From 5e44660c844bb9c64a85bfbb083b4a533571b83e Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 24 Jul 2023 17:22:03 -0400 Subject: [PATCH 19/46] chore: remove unneeded blocks pubsub implementation --- go.mod | 3 +- go.sum | 1 - lib/pubsub.go | 146 -------------------------------------------------- 3 files changed, 1 insertion(+), 149 deletions(-) delete mode 100644 lib/pubsub.go diff --git a/go.mod b/go.mod index a2dcc9c..6c5ef9d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/ipfs/bifrost-gateway go 1.19 require ( - github.com/cskr/pubsub v1.0.2 github.com/filecoin-saturn/caboose v0.0.3 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru/v2 v2.0.1 @@ -21,7 +20,6 @@ require ( github.com/libp2p/go-libp2p-routing-helpers v0.7.0 github.com/mitchellh/go-server-timing v1.0.1 github.com/multiformats/go-multicodec v0.9.0 - github.com/multiformats/go-multihash v0.2.3 github.com/prometheus/client_golang v1.15.1 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 github.com/spf13/cobra v1.6.1 @@ -97,6 +95,7 @@ require ( github.com/multiformats/go-multiaddr v0.8.0 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a // indirect diff --git a/go.sum b/go.sum index 29735e9..5657851 100644 --- a/go.sum +++ b/go.sum @@ -68,7 +68,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= -github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/lib/pubsub.go b/lib/pubsub.go deleted file mode 100644 index 0d1d553..0000000 --- a/lib/pubsub.go +++ /dev/null @@ -1,146 +0,0 @@ -package lib - -import ( - "context" - "sync" - - "github.com/multiformats/go-multihash" - - pubsub "github.com/cskr/pubsub" - blocks "github.com/ipfs/go-block-format" -) - -// Note: blantantly copied from boxo/bitswap internals https://github.com/ipfs/boxo/blob/664b3e5ee4a997c67909da9b017a28efa40cc8ae/bitswap/client/internal/notifications/notifications.go -// some minor changes were made - -const bufferSize = 16 - -// BlockPubSub is a simple interface for publishing blocks and being able to subscribe -// for multihashes. It's used internally by an exchange to decouple receiving blocks -// and actually providing them back to the GetBlocks caller. -// Note: because multihashes are being requested and blocks returned the codecs could be anything -type BlockPubSub interface { - Publish(blocks ...blocks.Block) - Subscribe(ctx context.Context, keys ...multihash.Multihash) <-chan blocks.Block - Shutdown() -} - -// NewBlockPubSub generates a new BlockPubSub interface. -func NewBlockPubSub() BlockPubSub { - return &impl{ - wrapped: *pubsub.New(bufferSize), - closed: make(chan struct{}), - } -} - -type impl struct { - lk sync.RWMutex - wrapped pubsub.PubSub - - closed chan struct{} -} - -func (ps *impl) Publish(blocks ...blocks.Block) { - ps.lk.RLock() - defer ps.lk.RUnlock() - select { - case <-ps.closed: - return - default: - } - - for _, block := range blocks { - ps.wrapped.Pub(block, string(block.Cid().Hash())) - } -} - -func (ps *impl) Shutdown() { - ps.lk.Lock() - defer ps.lk.Unlock() - select { - case <-ps.closed: - return - default: - } - close(ps.closed) - ps.wrapped.Shutdown() -} - -// Subscribe returns a channel of blocks for the given |keys|. |blockChannel| -// is closed if the |ctx| times out or is cancelled, or after receiving the blocks -// corresponding to |keys|. -func (ps *impl) Subscribe(ctx context.Context, keys ...multihash.Multihash) <-chan blocks.Block { - - blocksCh := make(chan blocks.Block, len(keys)) - valuesCh := make(chan interface{}, len(keys)) // provide our own channel to control buffer, prevent blocking - if len(keys) == 0 { - close(blocksCh) - return blocksCh - } - - // prevent shutdown - ps.lk.RLock() - defer ps.lk.RUnlock() - - select { - case <-ps.closed: - close(blocksCh) - return blocksCh - default: - } - - // AddSubOnceEach listens for each key in the list, and closes the channel - // once all keys have been received - ps.wrapped.AddSubOnceEach(valuesCh, toStrings(keys)...) - go func() { - defer func() { - close(blocksCh) - - ps.lk.RLock() - defer ps.lk.RUnlock() - // Don't touch the pubsub instance if we're - // already closed. - select { - case <-ps.closed: - return - default: - } - - ps.wrapped.Unsub(valuesCh) - }() - - for { - select { - case <-ctx.Done(): - return - case <-ps.closed: - return - case val, ok := <-valuesCh: - if !ok { - return - } - block, ok := val.(blocks.Block) - if !ok { - return - } - select { - case <-ctx.Done(): - return - case blocksCh <- block: // continue - case <-ps.closed: - return - } - } - } - }() - - return blocksCh -} - -func toStrings(keys []multihash.Multihash) []string { - strs := make([]string, 0, len(keys)) - for _, key := range keys { - strs = append(strs, string(key)) - } - return strs -} From c0f250ee95f9dfe32733292c48280be0f7a5f98e Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 24 Jul 2023 20:13:20 -0400 Subject: [PATCH 20/46] fix: TestGetHAMTDirectory is no longer flaky, although it is more implementation specific --- graph_gateway_test.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/graph_gateway_test.go b/graph_gateway_test.go index d70ef96..85c8d43 100644 --- a/graph_gateway_test.go +++ b/graph_gateway_test.go @@ -714,38 +714,39 @@ func TestGetHAMTDirectory(t *testing.T) { if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", // inner hamt nodes start here }); err != nil { panic(err) } - case 3: - // Expect a request for the full HAMT and return it - // Note: this is an implementation detail, it could be in the future that we request more or less data - // (e.g. ask for the full path, ask for index.html first, make a spec change to allow asking for index.html with a fallback to the directory, etc.) - expectedUri := "/ipfs/bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm" + // Expect a request for a non-existent index.html file + // Note: this is an implementation detail related to the directory request above + // Note: the order of cases 3 and 4 here are implementation specific as well + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir//index.html" if request.URL.Path != expectedUri { panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) } if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", // inner hamt nodes start here - "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", }); err != nil { panic(err) } case 4: - // Expect a request for a non-existent index.html file - // Note: this is an implementation detail related to the directory request above - expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir//index.html" + // Expect a request for the full HAMT and return it + // Note: this is an implementation detail, it could be in the future that we request more or less data + // (e.g. ask for the full path, ask for index.html first, make a spec change to allow asking for index.html with a fallback to the directory, etc.) + expectedUri := "/ipfs/bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm" if request.URL.Path != expectedUri { panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) } if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ - "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", // inner hamt nodes start here + "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", }); err != nil { panic(err) } From be81dfc3cdc9c3c5ab46d7172e036df7228159c7 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 25 Jul 2023 10:11:32 -0400 Subject: [PATCH 21/46] fix: preserve '/' characters even after escaping to help URL parsers that don't autodecode the URL --- lib/graph_gateway.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index d797a81..bf0922f 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -248,9 +248,10 @@ func paramsToString(params gateway.CarParams) string { } func (api *GraphGateway) fetchCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams, cb DataCallback) error { - escapedPath := url.PathEscape(path.String()[1:]) + escapedPath := url.PathEscape(path.String()) + escapedPath = strings.ReplaceAll(escapedPath, "%2F", "/") paramsStr := paramsToString(params) - urlWithoutHost := fmt.Sprintf("/%s?%s", escapedPath, paramsStr) + urlWithoutHost := fmt.Sprintf("%s?%s", escapedPath, paramsStr) api.metrics.carFetchAttemptMetric.Inc() var ipldError error From c3fb8d86cd8ac1cc6f5c463daf30d9ad64372493 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 26 Jul 2023 19:01:41 +0200 Subject: [PATCH 22/46] refactor: add contentPathToCarUrl and tests this implements part of https://github.com/ipfs/bifrost-gateway/pull/160#discussion_r1273818495 that fixes the way we encode content paths, without changing the interface of fetcher in caboose. --- graph_gateway_test.go | 2 +- lib/gateway_utils.go | 39 +++++++++++++++++++++++++++ lib/gateway_utils_test.go | 57 +++++++++++++++++++++++++++++++++++++++ lib/graph_gateway.go | 35 ++++++++---------------- 4 files changed, 108 insertions(+), 25 deletions(-) create mode 100644 lib/gateway_utils.go create mode 100644 lib/gateway_utils_test.go diff --git a/graph_gateway_test.go b/graph_gateway_test.go index 85c8d43..69910b0 100644 --- a/graph_gateway_test.go +++ b/graph_gateway_test.go @@ -328,7 +328,7 @@ func TestTarAtEndOfPath(t *testing.T) { } case 6: // Expect a request for the rest of the multiblock file in the directory and give it - expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa?dag-scope=entity&entity-bytes=768:*" + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa?format=car&dag-scope=entity&entity-bytes=768:*" if request.RequestURI != expectedUri { panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) } diff --git a/lib/gateway_utils.go b/lib/gateway_utils.go new file mode 100644 index 0000000..bee79a5 --- /dev/null +++ b/lib/gateway_utils.go @@ -0,0 +1,39 @@ +package lib + +import ( + "net/url" + "strconv" + "strings" + + "github.com/ipfs/boxo/gateway" +) + +// contentPathToCarUrl returns an URL that allows retrieval of specified resource +// from a trustless gateway that implements IPIP-402 +func contentPathToCarUrl(path gateway.ImmutablePath, params gateway.CarParams) *url.URL { + return &url.URL{ + Path: path.String(), + RawQuery: carParamsToString(params), + } +} + +// carParamsToString converts CarParams to URL parameters compatible with IPIP-402 +func carParamsToString(params gateway.CarParams) string { + paramsBuilder := strings.Builder{} + paramsBuilder.WriteString("format=car") // always send explicit format in URL, this makes debugging easier, even when Accept header was set + if params.Scope != "" { + paramsBuilder.WriteString("&dag-scope=") + paramsBuilder.WriteString(string(params.Scope)) + } + if params.Range != nil { + paramsBuilder.WriteString("&entity-bytes=") + paramsBuilder.WriteString(strconv.FormatInt(params.Range.From, 10)) + paramsBuilder.WriteString(":") + if params.Range.To != nil { + paramsBuilder.WriteString(strconv.FormatInt(*params.Range.To, 10)) + } else { + paramsBuilder.WriteString("*") + } + } + return paramsBuilder.String() +} diff --git a/lib/gateway_utils_test.go b/lib/gateway_utils_test.go new file mode 100644 index 0000000..dc7db0b --- /dev/null +++ b/lib/gateway_utils_test.go @@ -0,0 +1,57 @@ +package lib + +import ( + "testing" + + ifacepath "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/gateway" +) + +func TestContentPathToCarUrl(t *testing.T) { + negativeOffset := int64(-42) + testCases := []struct { + contentPath string // to be turned into gateway.ImmutablePath + carParams gateway.CarParams + expectedUrl string // url.URL.String() + }{ + { + contentPath: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", + carParams: gateway.CarParams{}, + expectedUrl: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?format=car", + }, + { + contentPath: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", + carParams: gateway.CarParams{Scope: "entity", Range: &gateway.DagByteRange{From: 0, To: nil}}, + expectedUrl: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?format=car&dag-scope=entity&entity-bytes=0:*", + }, + { + contentPath: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", + carParams: gateway.CarParams{Scope: "block"}, + expectedUrl: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?format=car&dag-scope=block", + }, + { + contentPath: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", + carParams: gateway.CarParams{Scope: "entity", Range: &gateway.DagByteRange{From: 4, To: &negativeOffset}}, + expectedUrl: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?format=car&dag-scope=entity&entity-bytes=4:-42", + }, + { + // a regression test for case described in https://github.com/ipfs/gateway-conformance/issues/115 + contentPath: "/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze/I/Auditorio_de_Tenerife%2C_Santa_Cruz_de_Tenerife%2C_España%2C_2012-12-15%2C_DD_02.jpg.webp", + carParams: gateway.CarParams{Scope: "entity", Range: &gateway.DagByteRange{From: 0, To: nil}}, + expectedUrl: "/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze/I/Auditorio_de_Tenerife%252C_Santa_Cruz_de_Tenerife%252C_Espa%C3%B1a%252C_2012-12-15%252C_DD_02.jpg.webp?format=car&dag-scope=entity&entity-bytes=0:*", + }, + } + + for _, tc := range testCases { + t.Run("TestContentPathToCarUrl", func(t *testing.T) { + contentPath, err := gateway.NewImmutablePath(ifacepath.New(tc.contentPath)) + if err != nil { + t.Fatal(err) + } + result := contentPathToCarUrl(contentPath, tc.carParams).String() + if result != tc.expectedUrl { + t.Errorf("Expected %q, but got %q", tc.expectedUrl, result) + } + }) + } +} diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index bf0922f..66a38e7 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -7,12 +7,15 @@ import ( "fmt" "io" "net/http" - "net/url" gopath "path" "strconv" "strings" "time" + "github.com/hashicorp/go-multierror" + "github.com/ipfs/go-unixfsnode/hamt" + "github.com/ipld/go-ipld-prime/traversal" + "github.com/filecoin-saturn/caboose" "github.com/hashicorp/go-multierror" nsopts "github.com/ipfs/boxo/coreiface/options/namesys" @@ -230,28 +233,8 @@ func registerGraphGatewayMetrics(registerer prometheus.Registerer) *GraphGateway } } -func paramsToString(params gateway.CarParams) string { - paramsBuilder := strings.Builder{} - paramsBuilder.WriteString("dag-scope=") - paramsBuilder.WriteString(string(params.Scope)) - if params.Range != nil { - paramsBuilder.WriteString("&entity-bytes=") - paramsBuilder.WriteString(strconv.FormatInt(params.Range.From, 10)) - paramsBuilder.WriteString(":") - if params.Range.To != nil { - paramsBuilder.WriteString(strconv.FormatInt(*params.Range.To, 10)) - } else { - paramsBuilder.WriteString("*") - } - } - return paramsBuilder.String() -} - func (api *GraphGateway) fetchCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams, cb DataCallback) error { - escapedPath := url.PathEscape(path.String()) - escapedPath = strings.ReplaceAll(escapedPath, "%2F", "/") - paramsStr := paramsToString(params) - urlWithoutHost := fmt.Sprintf("%s?%s", escapedPath, paramsStr) + urlWithoutHost := contentPathToCarUrl(path, params).String() api.metrics.carFetchAttemptMetric.Inc() var ipldError error @@ -866,11 +849,15 @@ func fetchWithPartialRetries[T any](ctx context.Context, path gateway.ImmutableP case closeErr := <-closeCh: return closeErr case req := <-sendRequest: - requestStr := fmt.Sprintf("/ipfs/%s?%s", req.c.String(), paramsToString(req.params)) // set path and params for next iteration p = ipfspath.FromCid(req.c) + imPath, err := gateway.NewImmutablePath(ifacepath.New(p.String())) + if err != nil { + return err + } params = req.params - return &ErrPartialResponse{StillNeed: []string{requestStr}} + remainderUrl := contentPathToCarUrl(imPath, params).String() + return &ErrPartialResponse{StillNeed: []string{remainderUrl}} case <-cctx.Done(): return cctx.Err() } From 7691b62a5dfbed1cac03f2f15609e6dd463be98b Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 27 Jul 2023 01:55:54 -0400 Subject: [PATCH 23/46] bump boxo dependency to handle _redirects better and switch from boxo/ipld/car to ipld/go-car --- lib/graph_gateway.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 66a38e7..b0b0619 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -12,10 +12,6 @@ import ( "strings" "time" - "github.com/hashicorp/go-multierror" - "github.com/ipfs/go-unixfsnode/hamt" - "github.com/ipld/go-ipld-prime/traversal" - "github.com/filecoin-saturn/caboose" "github.com/hashicorp/go-multierror" nsopts "github.com/ipfs/boxo/coreiface/options/namesys" From 236009c64cc30a12ed5eb440511f7df694ffe2cc Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 27 Jul 2023 03:31:03 -0400 Subject: [PATCH 24/46] update caboose to fix issues draining responses --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6c5ef9d..8ba8e9a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ipfs/bifrost-gateway go 1.19 require ( - github.com/filecoin-saturn/caboose v0.0.3 + github.com/filecoin-saturn/caboose v0.0.3-no-response-drain-2 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/ipfs/boxo v0.11.1-0.20230803093408-3b83506a2ea5 diff --git a/go.sum b/go.sum index 5657851..5456a7f 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/felixge/httpsnoop v1.0.0/go.mod h1:3+D9sFq0ahK/JeJPhCBUV1xlf4/eIYrUQaxulT0VzX8= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/filecoin-saturn/caboose v0.0.3 h1:VcfNc3J6qwCAdxKIVkkCglG0ll1qL043S2qcChmUr6o= -github.com/filecoin-saturn/caboose v0.0.3/go.mod h1:CSXZMijzD8z0Q/K5JQPeDZnb1o50MBK9VGZhOleqIDk= +github.com/filecoin-saturn/caboose v0.0.3-no-response-drain-2 h1:RhP0ncoFd6MpAeXa1ybireidGDiC6DZFmBR+hoiiGw8= +github.com/filecoin-saturn/caboose v0.0.3-no-response-drain-2/go.mod h1:CSXZMijzD8z0Q/K5JQPeDZnb1o50MBK9VGZhOleqIDk= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= From 53101eb2052dcca4ccd9ef736e4fd4ddb88fa264 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 27 Jul 2023 18:37:08 -0400 Subject: [PATCH 25/46] add more information when erroring on receiving an unexpected cid --- lib/gateway_traversal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gateway_traversal.go b/lib/gateway_traversal.go index 8c71cfc..0e491d5 100644 --- a/lib/gateway_traversal.go +++ b/lib/gateway_traversal.go @@ -94,7 +94,7 @@ func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *Grap if blkRead.block != nil { metrics.carBlocksFetchedMetric.Inc() if !blkRead.block.Cid().Equals(c) { - return nil, fmt.Errorf("unexpected block received") + return nil, fmt.Errorf("unexpected block received: expected %s, got %s", c, blkRead.block.Cid()) } return blkRead.block, nil } From 7df26b059cc4cdb978ec80b62732683fc09d8b2b Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 27 Jul 2023 18:40:07 -0400 Subject: [PATCH 26/46] fix: switch use of ErrPartialResponse to not use pointers so they work with the caboose checks --- graph_gateway_test.go | 2 +- lib/graph_gateway.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/graph_gateway_test.go b/graph_gateway_test.go index 69910b0..6902491 100644 --- a/graph_gateway_test.go +++ b/graph_gateway_test.go @@ -951,7 +951,7 @@ func (r *retryFetcher) Fetch(ctx context.Context, path string, cb lib.DataCallba } switch t := err.(type) { - case *lib.ErrPartialResponse: + case lib.ErrPartialResponse: if len(t.StillNeed) > 1 { panic("only a single request at a time supported") } diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index b0b0619..11e1fb0 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -853,7 +853,7 @@ func fetchWithPartialRetries[T any](ctx context.Context, path gateway.ImmutableP } params = req.params remainderUrl := contentPathToCarUrl(imPath, params).String() - return &ErrPartialResponse{StillNeed: []string{remainderUrl}} + return ErrPartialResponse{StillNeed: []string{remainderUrl}} case <-cctx.Done(): return cctx.Err() } From b1e8074d2c571956198e5e3c86152ed5c0df2dbf Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 27 Jul 2023 19:09:11 -0400 Subject: [PATCH 27/46] test: add test for file range requests with errors during download --- graph_gateway_test.go | 131 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/graph_gateway_test.go b/graph_gateway_test.go index 6902491..276079c 100644 --- a/graph_gateway_test.go +++ b/graph_gateway_test.go @@ -568,6 +568,137 @@ func TestGetFile(t *testing.T) { } } +func TestGetFileRangeRequest(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + requestNum := 0 + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates at the root block + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + }); err != nil { + panic(err) + } + case 2: + // Expect the full request, and return the whole file which should be invalid + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", // file chunks start here + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + }); err != nil { + panic(err) + } + case 3: + // Expect the full request and return the first block + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + }); err != nil { + panic(err) + } + + case 4: + // Expect a request for the remainder of the file + // Note: this is an implementation detail, it could be that the requester really asks for more information + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + }); err != nil { + panic(err) + } + + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}) + if err != nil { + t.Fatal(err) + } + + trustedGatewayServer := httptest.NewServer(gateway.NewHandler(gateway.Config{DeserializedResponses: true}, backend)) + defer trustedGatewayServer.Close() + + req, err := http.NewRequestWithContext(ctx, "GET", trustedGatewayServer.URL+"/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", nil) + if err != nil { + t.Fatal(err) + } + startIndex := 256 + endIndex := 750 + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", startIndex, endIndex)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + robs, err := carbs.NewReadOnly(bytes.NewReader(dirWithMultiblockHAMTandFiles), nil) + if err != nil { + t.Fatal(err) + } + + dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) + fileRootNd, err := dsrv.Get(ctx, cid.MustParse("bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa")) + if err != nil { + t.Fatal(err) + } + uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd) + if err != nil { + t.Fatal(err) + } + f := uio.(files.File) + if _, err := f.Seek(int64(startIndex), io.SeekStart); err != nil { + t.Fatal(err) + } + expectedFileData, err := io.ReadAll(io.LimitReader(f, int64(endIndex)-int64(startIndex)+1)) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data, expectedFileData) { + t.Fatalf("expected %s, got %s", string(expectedFileData), string(data)) + } + + if requestNum != 4 { + t.Fatalf("expected exactly 4 requests, got %d", requestNum) + } +} + func TestGetFileWithBadBlockReturned(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() From b3d290961eb03036cf5753a0642c4c05c65427da Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 27 Jul 2023 18:39:09 -0400 Subject: [PATCH 28/46] fix: pass through range parameters for partial file load resumption --- lib/files.go | 4 +++- lib/graph_gateway.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/files.go b/lib/files.go index fdbffe3..180e92d 100644 --- a/lib/files.go +++ b/lib/files.go @@ -513,7 +513,9 @@ func loadTerminalUnixFSElementWithRecursiveDirectories(ctx context.Context, c ci } from := int64(0) + var byteRange gateway.DagByteRange if params.Range != nil { + byteRange = *params.Range from = params.Range.From } _, err = f.Seek(from, io.SeekStart) @@ -521,7 +523,7 @@ func loadTerminalUnixFSElementWithRecursiveDirectories(ctx context.Context, c ci return nil, fmt.Errorf("unable to get reset UnixFS file reader: %w", err) } - return &backpressuredFile{ctx: ctx, fileCid: c, size: fileSize, f: f, getLsys: getLsys, closed: make(chan error)}, nil + return &backpressuredFile{ctx: ctx, fileCid: c, byteRange: byteRange, size: fileSize, f: f, getLsys: getLsys, closed: make(chan error)}, nil default: return nil, fmt.Errorf("unknown UnixFS field type") } diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 11e1fb0..69857ce 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -564,15 +564,17 @@ func loadTerminalEntity(ctx context.Context, c cid.Cid, blk blocks.Block, lsys * } from := int64(0) + var byteRange gateway.DagByteRange if params.Range != nil { from = params.Range.From + byteRange = *params.Range } _, err = f.Seek(from, io.SeekStart) if err != nil { return nil, fmt.Errorf("unable to get reset UnixFS file reader: %w", err) } - return &backpressuredFile{ctx: ctx, fileCid: c, size: fileSize, f: f, getLsys: getLsys, closed: make(chan error)}, nil + return &backpressuredFile{ctx: ctx, fileCid: c, byteRange: byteRange, size: fileSize, f: f, getLsys: getLsys, closed: make(chan error)}, nil default: return nil, fmt.Errorf("unknown UnixFS field type") } From 03cba57ae6668e534b02f6dad295f7c1c7865326 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 31 Jul 2023 15:00:54 -0400 Subject: [PATCH 29/46] fix: perform gateway error conversion for graph backend --- blockstore.go | 36 +-------------- blockstore_test.go | 44 ------------------ graph_gateway_test.go | 96 +++++++++++++++++++++++++++++++++++++++ lib/gateway_traversal.go | 2 +- lib/gateway_utils.go | 34 ++++++++++++++ lib/gateway_utils_test.go | 37 +++++++++++++++ 6 files changed, 170 insertions(+), 79 deletions(-) delete mode 100644 blockstore_test.go diff --git a/blockstore.go b/blockstore.go index 571b436..825cb08 100644 --- a/blockstore.go +++ b/blockstore.go @@ -3,14 +3,11 @@ package main import ( "context" "errors" - "fmt" - "strings" "time" - "github.com/filecoin-saturn/caboose" + "github.com/ipfs/bifrost-gateway/lib" blockstore "github.com/ipfs/boxo/blockstore" exchange "github.com/ipfs/boxo/exchange" - "github.com/ipfs/boxo/gateway" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "go.uber.org/zap/zapcore" @@ -38,7 +35,7 @@ func (e *exchangeBsWrapper) GetBlock(ctx context.Context, c cid.Cid) (blocks.Blo blk, err := e.bstore.Get(ctx, c) if err != nil { - return nil, gatewayError(err) + return nil, lib.GatewayError(err) } return blk, nil } @@ -67,33 +64,4 @@ func (e *exchangeBsWrapper) Close() error { return nil } -// gatewayError translates underlying blockstore error into one that gateway code will return as HTTP 502 or 504 -// it also makes sure Retry-After hint from remote blockstore will be passed to HTTP client, if present. -func gatewayError(err error) error { - if errors.Is(err, &gateway.ErrorStatusCode{}) || - errors.Is(err, &gateway.ErrorRetryAfter{}) { - // already correct error - return err - } - - // All timeouts should produce 504 Gateway Timeout - if errors.Is(err, context.DeadlineExceeded) || - errors.Is(err, caboose.ErrSaturnTimeout) || - // Unfortunately this is not an exported type so we have to check for the content. - strings.Contains(err.Error(), "Client.Timeout exceeded") { - return fmt.Errorf("%w: %s", gateway.ErrGatewayTimeout, err.Error()) - } - - // (Saturn) errors that support the RetryAfter interface need to be converted - // to the correct gateway error, such that the HTTP header is set. - for v := err; v != nil; v = errors.Unwrap(v) { - if r, ok := v.(interface{ RetryAfter() time.Duration }); ok { - return gateway.NewErrorRetryAfter(err, r.RetryAfter()) - } - } - - // everything else returns 502 Bad Gateway - return fmt.Errorf("%w: %s", gateway.ErrBadGateway, err.Error()) -} - var _ exchange.Interface = (*exchangeBsWrapper)(nil) diff --git a/blockstore_test.go b/blockstore_test.go deleted file mode 100644 index c47827a..0000000 --- a/blockstore_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "testing" - "time" - - "github.com/ipfs/boxo/gateway" - "github.com/stretchr/testify/assert" -) - -type testErr struct { - message string - retryAfter time.Duration -} - -func (e *testErr) Error() string { - return e.message -} - -func (e *testErr) RetryAfter() time.Duration { - return e.retryAfter -} - -func TestGatewayErrorRetryAfter(t *testing.T) { - originalErr := &testErr{message: "test", retryAfter: time.Minute} - var ( - convertedErr error - gatewayErr *gateway.ErrorRetryAfter - ) - - // Test unwrapped - convertedErr = gatewayError(originalErr) - ok := errors.As(convertedErr, &gatewayErr) - assert.True(t, ok) - assert.EqualValues(t, originalErr.retryAfter, gatewayErr.RetryAfter) - - // Test wrapped. - convertedErr = gatewayError(fmt.Errorf("wrapped error: %w", originalErr)) - ok = errors.As(convertedErr, &gatewayErr) - assert.True(t, ok) - assert.EqualValues(t, originalErr.retryAfter, gatewayErr.RetryAfter) -} diff --git a/graph_gateway_test.go b/graph_gateway_test.go index 276079c..7b979b9 100644 --- a/graph_gateway_test.go +++ b/graph_gateway_test.go @@ -1063,6 +1063,102 @@ func TestGetCAR(t *testing.T) { } } +func TestPassthroughErrors(t *testing.T) { + t.Run("PathTraversalError", func(t *testing.T) { + pathTraversalTest := func(t *testing.T, traversal func(ctx context.Context, p gateway.ImmutablePath, backend *lib.GraphGateway) error) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var requestNum int + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates in the middle of the path + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + }); err != nil { + panic(err) + } + case 2: + // Expect the full request, but return one that terminates in the middle of the file + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. partial path) + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + }); err != nil { + panic(err) + } + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + p, err := gateway.NewImmutablePath(path.New("/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA")) + if err != nil { + t.Fatal(err) + } + + bogusErr := gateway.NewErrorStatusCode(fmt.Errorf("this is a test error"), 418) + + clientRequestNum := 0 + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{ + inner: &fetcherWrapper{fn: func(ctx context.Context, path string, cb lib.DataCallback) error { + clientRequestNum++ + if clientRequestNum > 2 { + return bogusErr + } + return bs.(lib.CarFetcher).Fetch(ctx, path, cb) + }}, + allowedRetries: 3, retriesRemaining: 3}) + if err != nil { + t.Fatal(err) + } + + err = traversal(ctx, p, backend) + parsedErr := &gateway.ErrorStatusCode{} + if errors.As(err, &parsedErr) { + if parsedErr.StatusCode == bogusErr.StatusCode { + return + } + } + t.Fatal("error did not pass through") + } + t.Run("Block", func(t *testing.T) { + pathTraversalTest(t, func(ctx context.Context, p gateway.ImmutablePath, backend *lib.GraphGateway) error { + _, _, err := backend.GetBlock(ctx, p) + return err + }) + }) + t.Run("File", func(t *testing.T) { + pathTraversalTest(t, func(ctx context.Context, p gateway.ImmutablePath, backend *lib.GraphGateway) error { + _, _, err := backend.Get(ctx, p) + return err + }) + }) + }) +} + +type fetcherWrapper struct { + fn func(ctx context.Context, path string, cb lib.DataCallback) error +} + +func (w *fetcherWrapper) Fetch(ctx context.Context, path string, cb lib.DataCallback) error { + return w.fn(ctx, path, cb) +} + type retryFetcher struct { inner lib.CarFetcher allowedRetries int diff --git a/lib/gateway_traversal.go b/lib/gateway_traversal.go index 0e491d5..91d1d99 100644 --- a/lib/gateway_traversal.go +++ b/lib/gateway_traversal.go @@ -89,7 +89,7 @@ func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *Grap if !ok || errors.Is(blkRead.err, io.EOF) { return nil, io.ErrUnexpectedEOF } - return nil, blkRead.err + return nil, GatewayError(blkRead.err) } if blkRead.block != nil { metrics.carBlocksFetchedMetric.Inc() diff --git a/lib/gateway_utils.go b/lib/gateway_utils.go index bee79a5..e7093fb 100644 --- a/lib/gateway_utils.go +++ b/lib/gateway_utils.go @@ -1,10 +1,15 @@ package lib import ( + "context" + "errors" + "fmt" "net/url" "strconv" "strings" + "time" + "github.com/filecoin-saturn/caboose" "github.com/ipfs/boxo/gateway" ) @@ -37,3 +42,32 @@ func carParamsToString(params gateway.CarParams) string { } return paramsBuilder.String() } + +// GatewayError translates underlying blockstore error into one that gateway code will return as HTTP 502 or 504 +// it also makes sure Retry-After hint from remote blockstore will be passed to HTTP client, if present. +func GatewayError(err error) error { + if errors.Is(err, &gateway.ErrorStatusCode{}) || + errors.Is(err, &gateway.ErrorRetryAfter{}) { + // already correct error + return err + } + + // All timeouts should produce 504 Gateway Timeout + if errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, caboose.ErrSaturnTimeout) || + // Unfortunately this is not an exported type so we have to check for the content. + strings.Contains(err.Error(), "Client.Timeout exceeded") { + return fmt.Errorf("%w: %s", gateway.ErrGatewayTimeout, err.Error()) + } + + // (Saturn) errors that support the RetryAfter interface need to be converted + // to the correct gateway error, such that the HTTP header is set. + for v := err; v != nil; v = errors.Unwrap(v) { + if r, ok := v.(interface{ RetryAfter() time.Duration }); ok { + return gateway.NewErrorRetryAfter(err, r.RetryAfter()) + } + } + + // everything else returns 502 Bad Gateway + return fmt.Errorf("%w: %s", gateway.ErrBadGateway, err.Error()) +} diff --git a/lib/gateway_utils_test.go b/lib/gateway_utils_test.go index dc7db0b..f27d96e 100644 --- a/lib/gateway_utils_test.go +++ b/lib/gateway_utils_test.go @@ -1,7 +1,11 @@ package lib import ( + "errors" + "fmt" + "github.com/stretchr/testify/assert" "testing" + "time" ifacepath "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/gateway" @@ -55,3 +59,36 @@ func TestContentPathToCarUrl(t *testing.T) { }) } } + +type testErr struct { + message string + retryAfter time.Duration +} + +func (e *testErr) Error() string { + return e.message +} + +func (e *testErr) RetryAfter() time.Duration { + return e.retryAfter +} + +func TestGatewayErrorRetryAfter(t *testing.T) { + originalErr := &testErr{message: "test", retryAfter: time.Minute} + var ( + convertedErr error + gatewayErr *gateway.ErrorRetryAfter + ) + + // Test unwrapped + convertedErr = GatewayError(originalErr) + ok := errors.As(convertedErr, &gatewayErr) + assert.True(t, ok) + assert.EqualValues(t, originalErr.retryAfter, gatewayErr.RetryAfter) + + // Test wrapped. + convertedErr = GatewayError(fmt.Errorf("wrapped error: %w", originalErr)) + ok = errors.As(convertedErr, &gatewayErr) + assert.True(t, ok) + assert.EqualValues(t, originalErr.retryAfter, gatewayErr.RetryAfter) +} From 28801f0ee1c8dec26958515a22c15d23f470a0cc Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 31 Jul 2023 15:22:14 -0400 Subject: [PATCH 30/46] feat: add ipip-412 parameters for order and duplicates to non-saturn fetcher --- blockstore_proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blockstore_proxy.go b/blockstore_proxy.go index 4f30499..deab4fb 100644 --- a/blockstore_proxy.go +++ b/blockstore_proxy.go @@ -43,7 +43,7 @@ func (ps *proxyBlockStore) Fetch(ctx context.Context, path string, cb lib.DataCa return err } goLog.Debugw("car fetch", "url", req.URL) - req.Header.Set("Accept", "application/vnd.ipld.car") + req.Header.Set("Accept", "application/vnd.ipld.car;order=dfs;dups=y") resp, err := ps.httpClient.Do(req) if err != nil { return err From db66484511f2754e7c8817f1244b9140c42c5716 Mon Sep 17 00:00:00 2001 From: Will Date: Thu, 3 Aug 2023 21:54:41 +0200 Subject: [PATCH 31/46] fix: allow reads with empty car roots (#182) --- go.mod | 3 +-- go.sum | 6 ++---- lib/gateway_traversal.go | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 8ba8e9a..89bada6 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/ipfs/go-ipld-format v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-unixfsnode v1.7.3-0.20230718163022-62f4a1c8d46e - github.com/ipld/go-car v0.6.0 + github.com/ipld/go-car v0.6.2 github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 github.com/ipld/go-codec-dagpb v1.6.0 github.com/ipld/go-ipld-prime v0.20.0 @@ -68,7 +68,6 @@ require ( github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 // indirect github.com/ipfs/go-ipld-legacy v0.2.1 // indirect - github.com/ipfs/go-libipfs v0.6.0 // indirect github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-merkledag v0.11.0 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect diff --git a/go.sum b/go.sum index 5456a7f..adcba0f 100644 --- a/go.sum +++ b/go.sum @@ -247,8 +247,6 @@ github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAF github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M= github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk= github.com/ipfs/go-ipld-legacy v0.2.1/go.mod h1:782MOUghNzMO2DER0FlBR94mllfdCJCkTtDtPM51otM= -github.com/ipfs/go-libipfs v0.6.0 h1:3FuckAJEm+zdHbHbf6lAyk0QUzc45LsFcGw102oBCZM= -github.com/ipfs/go-libipfs v0.6.0/go.mod h1:UjjDIuehp2GzlNP0HEr5I9GfFT7zWgst+YfpUEIThtw= github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= @@ -264,8 +262,8 @@ github.com/ipfs/go-unixfsnode v1.7.3-0.20230718163022-62f4a1c8d46e h1:LZkN2wQ49q github.com/ipfs/go-unixfsnode v1.7.3-0.20230718163022-62f4a1c8d46e/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= -github.com/ipld/go-car v0.6.0 h1:d5QrGLnHAxiNLHor+DKGrLdqnM0dQJh2whfSXRDq6J0= -github.com/ipld/go-car v0.6.0/go.mod h1:tBrW1XZ3L2XipLxA69RnTVGW3rve6VX4TbaTYkq8aEA= +github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= +github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 h1:0OZwzSYWIuiKEOXd/2vm5cMcEmmGLFn+1h6lHELCm3s= github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= diff --git a/lib/gateway_traversal.go b/lib/gateway_traversal.go index 91d1d99..a9b4b92 100644 --- a/lib/gateway_traversal.go +++ b/lib/gateway_traversal.go @@ -31,7 +31,7 @@ import ( type getBlock func(ctx context.Context, cid cid.Cid) (blocks.Block, error) func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *GraphGatewayMetrics) (getBlock, error) { - cr, err := car.NewCarReader(reader) + cr, err := car.NewCarReaderWithOptions(reader, car.WithErrorOnEmptyRoots(false)) if err != nil { return nil, err } From 8c217f92eb652cf75a0cfd50f58cb7893681c6ec Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Mon, 7 Aug 2023 16:37:12 +0100 Subject: [PATCH 32/46] fix memory leak --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 89bada6..c6c903d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ipfs/bifrost-gateway go 1.19 require ( - github.com/filecoin-saturn/caboose v0.0.3-no-response-drain-2 + github.com/filecoin-saturn/caboose v0.0.5-0.20230807153022-70557b9e3d01 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/ipfs/boxo v0.11.1-0.20230803093408-3b83506a2ea5 diff --git a/go.sum b/go.sum index adcba0f..5c0697e 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/felixge/httpsnoop v1.0.0/go.mod h1:3+D9sFq0ahK/JeJPhCBUV1xlf4/eIYrUQaxulT0VzX8= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/filecoin-saturn/caboose v0.0.3-no-response-drain-2 h1:RhP0ncoFd6MpAeXa1ybireidGDiC6DZFmBR+hoiiGw8= -github.com/filecoin-saturn/caboose v0.0.3-no-response-drain-2/go.mod h1:CSXZMijzD8z0Q/K5JQPeDZnb1o50MBK9VGZhOleqIDk= +github.com/filecoin-saturn/caboose v0.0.5-0.20230807153022-70557b9e3d01 h1:r8wHWd2gN5N3aOyE8j2k+4YlyfIthr18j0ZReC2HaYI= +github.com/filecoin-saturn/caboose v0.0.5-0.20230807153022-70557b9e3d01/go.mod h1:DfukoSiI7YZoWhNyYa5Vi2caHCQbFgmvKMcMAbENcas= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= From 06077bc95feeb38f5eb26e9bac6ca896cc5959f6 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Tue, 8 Aug 2023 09:04:23 +0100 Subject: [PATCH 33/46] update to latest caboose --- go.mod | 39 ++++++++++++++-------------- go.sum | 80 ++++++++++++++++++++++++++++++---------------------------- 2 files changed, 61 insertions(+), 58 deletions(-) diff --git a/go.mod b/go.mod index c6c903d..2ceb1a9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ipfs/bifrost-gateway go 1.19 require ( - github.com/filecoin-saturn/caboose v0.0.5-0.20230807153022-70557b9e3d01 + github.com/filecoin-saturn/caboose v0.0.5-0.20230807155604-954108a2dd1d github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/ipfs/boxo v0.11.1-0.20230803093408-3b83506a2ea5 @@ -25,11 +25,11 @@ require ( github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.4 github.com/willscott/go-requestcontext v0.0.1 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 go.opentelemetry.io/contrib/propagators/autoprop v0.40.0 - go.opentelemetry.io/otel v1.14.0 - go.opentelemetry.io/otel/sdk v1.14.0 - go.opentelemetry.io/otel/trace v1.14.0 + go.opentelemetry.io/otel v1.16.0 + go.opentelemetry.io/otel/sdk v1.16.0 + go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 ) @@ -38,7 +38,7 @@ require ( github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/asecurityteam/rolling v0.0.0-20230418204413-b4052899307d // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -46,7 +46,7 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/gabriel-vasile/mimetype v1.4.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 // indirect @@ -118,32 +118,33 @@ require ( github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.42.0 // indirect go.opentelemetry.io/contrib/propagators/aws v1.15.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.15.0 // indirect go.opentelemetry.io/contrib/propagators/jaeger v1.15.0 // indirect go.opentelemetry.io/contrib/propagators/ot v1.15.0 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect go.opentelemetry.io/otel/exporters/zipkin v1.14.0 // indirect - go.opentelemetry.io/otel/metric v0.37.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.6.0 // indirect golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb // indirect - golang.org/x/mod v0.7.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/net v0.8.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.7.0 // indirect - golang.org/x/tools v0.5.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.8.0 // indirect + golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.11.0 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.53.0 // indirect + google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect + google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect diff --git a/go.sum b/go.sum index 5c0697e..54a5d6a 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -89,8 +89,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/felixge/httpsnoop v1.0.0/go.mod h1:3+D9sFq0ahK/JeJPhCBUV1xlf4/eIYrUQaxulT0VzX8= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/filecoin-saturn/caboose v0.0.5-0.20230807153022-70557b9e3d01 h1:r8wHWd2gN5N3aOyE8j2k+4YlyfIthr18j0ZReC2HaYI= -github.com/filecoin-saturn/caboose v0.0.5-0.20230807153022-70557b9e3d01/go.mod h1:DfukoSiI7YZoWhNyYa5Vi2caHCQbFgmvKMcMAbENcas= +github.com/filecoin-saturn/caboose v0.0.5-0.20230807155604-954108a2dd1d h1:xLtdKE29tKgMF96S5CtMaV2PKzCUQ1zJC9J7cKJ022k= +github.com/filecoin-saturn/caboose v0.0.5-0.20230807155604-954108a2dd1d/go.mod h1:nmDICK/NKu6r3c6V6W/h6vi7Ep2b+jWbRbEg51ltkHs= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= @@ -101,8 +101,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= @@ -113,8 +113,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 h1:yrv1uUvgXH/tEat+wdvJMRJ4g51GlIydtDpU9pFjaaI= github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -483,8 +483,10 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0 h1:lE9EJyw3/JhrjWH/hEy9FptnalDQgj7vpbgC2KCCCxE= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0/go.mod h1:pcQ3MM3SWvrA71U4GDqv9UFDJ3HQsW7y5ZO3tDTlUdI= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.42.0 h1:0vzgiFDsCh/jxRCR1xcRrtMoeCu2itXz/PsXst5P8rI= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.42.0/go.mod h1:y0vOY2OKFMOTvwxKfurStPayUUKGHlNeVqNneHmFXr0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 h1:pginetY7+onl4qN1vl0xW/V/v6OBZ0vVdH+esuJgvmM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0/go.mod h1:XiYsayHc36K3EByOO6nbAXnAWbrUxdjUROCEeeROOH8= go.opentelemetry.io/contrib/propagators/autoprop v0.40.0 h1:Lj33jj7eIrBfIShiK8NU91u2BglKnUS1UUxVemuQJtw= go.opentelemetry.io/contrib/propagators/autoprop v0.40.0/go.mod h1:6QO816FeZ+6zahs6hYqbUCCsnNBm7o+t4iwVySpzcdI= go.opentelemetry.io/contrib/propagators/aws v1.15.0 h1:FLe+bRTMAhEALItDQt1U2S/rdq8/rGGJTJpOpCDvMu0= @@ -495,28 +497,28 @@ go.opentelemetry.io/contrib/propagators/jaeger v1.15.0 h1:xdJjwy5t/8I+TZehMMQ+r2 go.opentelemetry.io/contrib/propagators/jaeger v1.15.0/go.mod h1:tU0nwW4QTvKceNUP60/PQm0FI8zDSwey7gIFt3RR/yw= go.opentelemetry.io/contrib/propagators/ot v1.15.0 h1:iBNejawWy7wWZ5msuZDNcMjBy14Wc0v3gCAXukGHN/Q= go.opentelemetry.io/contrib/propagators/ot v1.15.0/go.mod h1:0P7QQ+MHt6SXR1ATaMpewSiWlp8NbKErNLKcaU4EEKI= -go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/exporters/jaeger v1.14.0 h1:CjbUNd4iN2hHmWekmOqZ+zSCU+dzZppG8XsV+A3oc8Q= go.opentelemetry.io/otel/exporters/jaeger v1.14.0/go.mod h1:4Ay9kk5vELRrbg5z4cpP9EtmQRFap2Wb0woPG4lujZA= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 h1:/fXHZHGvro6MVqV34fJzDhi7sHGpX3Ej/Qjmfn003ho= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0/go.mod h1:UFG7EBMRdXyFstOwH028U0sVf+AvukSGhF0g8+dmNG8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 h1:TKf2uAs2ueguzLaxOCBXNpHxfO/aC7PAdDsSH0IbeRQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0/go.mod h1:HrbCVv40OOLTABmOn1ZWty6CHXkU8DK/Urc43tHug70= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 h1:ap+y8RXX3Mu9apKVtOkM6WSFESLM8K3wNQyOU8sWHcc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0/go.mod h1:5w41DY6S9gZrbjuq6Y+753e96WfPha5IcsOSZTtullM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0 h1:3jAYbRHQAqzLjd9I4tzxwJ8Pk/N6AqBcF6m1ZHrxG94= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0/go.mod h1:+N7zNjIJv4K+DeX67XXET0P+eIciESgaFDBqh+ZJFS4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 h1:iqjq9LAB8aK++sKVcELezzn655JnBNdsDhghU4G/So8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0/go.mod h1:hGXzO5bhhSHZnKvrDaXB82Y9DRFour0Nz/KrBh7reWw= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 h1:sEL90JjOO/4yhquXl5zTAkLLsZ5+MycAgX99SDsxGc8= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0/go.mod h1:oCslUcizYdpKYyS9e8srZEqM6BB8fq41VJBjLAE6z1w= go.opentelemetry.io/otel/exporters/zipkin v1.14.0 h1:reEVE1upBF9tcujgvSqLJS0SrI7JQPaTKP4s4rymnSs= go.opentelemetry.io/otel/exporters/zipkin v1.14.0/go.mod h1:RcjvOAcvhzcufQP8aHmzRw1gE9g/VEZufDdo2w+s4sk= -go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs= -go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s= -go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= -go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= -go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -579,8 +581,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -613,8 +615,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -670,8 +672,8 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -682,8 +684,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -734,8 +736,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.5.0 h1:+bSpV5HIeWkuvgaMfI3UmKRThoTA5ODJTUd8T17NO+4= -golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -797,8 +799,8 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -816,8 +818,8 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 0bbc0c45f255b3b62720f7f528732149a388cec0 Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Tue, 8 Aug 2023 17:17:47 +0400 Subject: [PATCH 34/46] Send verification error to Caboose (#184) Co-authored-by: Will Scott --- go.mod | 39 ++++++++++---------- go.sum | 80 ++++++++++++++++++++-------------------- lib/gateway_traversal.go | 7 +++- 3 files changed, 66 insertions(+), 60 deletions(-) diff --git a/go.mod b/go.mod index c6c903d..b8ff04a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ipfs/bifrost-gateway go 1.19 require ( - github.com/filecoin-saturn/caboose v0.0.5-0.20230807153022-70557b9e3d01 + github.com/filecoin-saturn/caboose v0.0.5-0.20230808125624-a986b542aa10 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/ipfs/boxo v0.11.1-0.20230803093408-3b83506a2ea5 @@ -25,11 +25,11 @@ require ( github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.4 github.com/willscott/go-requestcontext v0.0.1 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 go.opentelemetry.io/contrib/propagators/autoprop v0.40.0 - go.opentelemetry.io/otel v1.14.0 - go.opentelemetry.io/otel/sdk v1.14.0 - go.opentelemetry.io/otel/trace v1.14.0 + go.opentelemetry.io/otel v1.16.0 + go.opentelemetry.io/otel/sdk v1.16.0 + go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 ) @@ -38,7 +38,7 @@ require ( github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/asecurityteam/rolling v0.0.0-20230418204413-b4052899307d // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -46,7 +46,7 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/gabriel-vasile/mimetype v1.4.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 // indirect @@ -118,32 +118,33 @@ require ( github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.42.0 // indirect go.opentelemetry.io/contrib/propagators/aws v1.15.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.15.0 // indirect go.opentelemetry.io/contrib/propagators/jaeger v1.15.0 // indirect go.opentelemetry.io/contrib/propagators/ot v1.15.0 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect go.opentelemetry.io/otel/exporters/zipkin v1.14.0 // indirect - go.opentelemetry.io/otel/metric v0.37.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.6.0 // indirect golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb // indirect - golang.org/x/mod v0.7.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/net v0.8.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.7.0 // indirect - golang.org/x/tools v0.5.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.8.0 // indirect + golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.11.0 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.53.0 // indirect + google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect + google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect diff --git a/go.sum b/go.sum index 5c0697e..67210f3 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -89,8 +89,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/felixge/httpsnoop v1.0.0/go.mod h1:3+D9sFq0ahK/JeJPhCBUV1xlf4/eIYrUQaxulT0VzX8= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/filecoin-saturn/caboose v0.0.5-0.20230807153022-70557b9e3d01 h1:r8wHWd2gN5N3aOyE8j2k+4YlyfIthr18j0ZReC2HaYI= -github.com/filecoin-saturn/caboose v0.0.5-0.20230807153022-70557b9e3d01/go.mod h1:DfukoSiI7YZoWhNyYa5Vi2caHCQbFgmvKMcMAbENcas= +github.com/filecoin-saturn/caboose v0.0.5-0.20230808125624-a986b542aa10 h1:VZhzrXd6IuL52U18KtTQ/q4v48J90a1aDr/maLF3H9Y= +github.com/filecoin-saturn/caboose v0.0.5-0.20230808125624-a986b542aa10/go.mod h1:nmDICK/NKu6r3c6V6W/h6vi7Ep2b+jWbRbEg51ltkHs= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= @@ -101,8 +101,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= @@ -113,8 +113,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 h1:yrv1uUvgXH/tEat+wdvJMRJ4g51GlIydtDpU9pFjaaI= github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -483,8 +483,10 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0 h1:lE9EJyw3/JhrjWH/hEy9FptnalDQgj7vpbgC2KCCCxE= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0/go.mod h1:pcQ3MM3SWvrA71U4GDqv9UFDJ3HQsW7y5ZO3tDTlUdI= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.42.0 h1:0vzgiFDsCh/jxRCR1xcRrtMoeCu2itXz/PsXst5P8rI= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.42.0/go.mod h1:y0vOY2OKFMOTvwxKfurStPayUUKGHlNeVqNneHmFXr0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 h1:pginetY7+onl4qN1vl0xW/V/v6OBZ0vVdH+esuJgvmM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0/go.mod h1:XiYsayHc36K3EByOO6nbAXnAWbrUxdjUROCEeeROOH8= go.opentelemetry.io/contrib/propagators/autoprop v0.40.0 h1:Lj33jj7eIrBfIShiK8NU91u2BglKnUS1UUxVemuQJtw= go.opentelemetry.io/contrib/propagators/autoprop v0.40.0/go.mod h1:6QO816FeZ+6zahs6hYqbUCCsnNBm7o+t4iwVySpzcdI= go.opentelemetry.io/contrib/propagators/aws v1.15.0 h1:FLe+bRTMAhEALItDQt1U2S/rdq8/rGGJTJpOpCDvMu0= @@ -495,28 +497,28 @@ go.opentelemetry.io/contrib/propagators/jaeger v1.15.0 h1:xdJjwy5t/8I+TZehMMQ+r2 go.opentelemetry.io/contrib/propagators/jaeger v1.15.0/go.mod h1:tU0nwW4QTvKceNUP60/PQm0FI8zDSwey7gIFt3RR/yw= go.opentelemetry.io/contrib/propagators/ot v1.15.0 h1:iBNejawWy7wWZ5msuZDNcMjBy14Wc0v3gCAXukGHN/Q= go.opentelemetry.io/contrib/propagators/ot v1.15.0/go.mod h1:0P7QQ+MHt6SXR1ATaMpewSiWlp8NbKErNLKcaU4EEKI= -go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/exporters/jaeger v1.14.0 h1:CjbUNd4iN2hHmWekmOqZ+zSCU+dzZppG8XsV+A3oc8Q= go.opentelemetry.io/otel/exporters/jaeger v1.14.0/go.mod h1:4Ay9kk5vELRrbg5z4cpP9EtmQRFap2Wb0woPG4lujZA= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 h1:/fXHZHGvro6MVqV34fJzDhi7sHGpX3Ej/Qjmfn003ho= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0/go.mod h1:UFG7EBMRdXyFstOwH028U0sVf+AvukSGhF0g8+dmNG8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 h1:TKf2uAs2ueguzLaxOCBXNpHxfO/aC7PAdDsSH0IbeRQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0/go.mod h1:HrbCVv40OOLTABmOn1ZWty6CHXkU8DK/Urc43tHug70= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 h1:ap+y8RXX3Mu9apKVtOkM6WSFESLM8K3wNQyOU8sWHcc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0/go.mod h1:5w41DY6S9gZrbjuq6Y+753e96WfPha5IcsOSZTtullM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0 h1:3jAYbRHQAqzLjd9I4tzxwJ8Pk/N6AqBcF6m1ZHrxG94= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0/go.mod h1:+N7zNjIJv4K+DeX67XXET0P+eIciESgaFDBqh+ZJFS4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 h1:iqjq9LAB8aK++sKVcELezzn655JnBNdsDhghU4G/So8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0/go.mod h1:hGXzO5bhhSHZnKvrDaXB82Y9DRFour0Nz/KrBh7reWw= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 h1:sEL90JjOO/4yhquXl5zTAkLLsZ5+MycAgX99SDsxGc8= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0/go.mod h1:oCslUcizYdpKYyS9e8srZEqM6BB8fq41VJBjLAE6z1w= go.opentelemetry.io/otel/exporters/zipkin v1.14.0 h1:reEVE1upBF9tcujgvSqLJS0SrI7JQPaTKP4s4rymnSs= go.opentelemetry.io/otel/exporters/zipkin v1.14.0/go.mod h1:RcjvOAcvhzcufQP8aHmzRw1gE9g/VEZufDdo2w+s4sk= -go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs= -go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s= -go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= -go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= -go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -579,8 +581,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -613,8 +615,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -670,8 +672,8 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -682,8 +684,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -734,8 +736,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.5.0 h1:+bSpV5HIeWkuvgaMfI3UmKRThoTA5ODJTUd8T17NO+4= -golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -797,8 +799,8 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -816,8 +818,8 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/lib/gateway_traversal.go b/lib/gateway_traversal.go index a9b4b92..8cc320b 100644 --- a/lib/gateway_traversal.go +++ b/lib/gateway_traversal.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "github.com/filecoin-saturn/caboose" "io" "sync" "time" @@ -30,6 +31,8 @@ import ( type getBlock func(ctx context.Context, cid cid.Cid) (blocks.Block, error) +var ErrNilBlock = caboose.ErrInvalidResponse{Message: "received a nil block with no error"} + func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *GraphGatewayMetrics) (getBlock, error) { cr, err := car.NewCarReaderWithOptions(reader, car.WithErrorOnEmptyRoots(false)) if err != nil { @@ -94,11 +97,11 @@ func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *Grap if blkRead.block != nil { metrics.carBlocksFetchedMetric.Inc() if !blkRead.block.Cid().Equals(c) { - return nil, fmt.Errorf("unexpected block received: expected %s, got %s", c, blkRead.block.Cid()) + return nil, caboose.ErrInvalidResponse{Message: fmt.Sprintf("received block with cid %s, expected %s", blkRead.block.Cid(), c)} } return blkRead.block, nil } - return nil, fmt.Errorf("received a nil block with no error") + return nil, ErrNilBlock }, nil } From c9fd369fb164c1ad4d62c10730d212f7dc0aa2cf Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 10 Aug 2023 06:49:13 -0400 Subject: [PATCH 35/46] bump go-unixfsnode and enable returning the sizes of items in HAMT directories for dir_index.html rendering --- go.mod | 2 +- go.sum | 4 ++-- lib/graph_gateway.go | 15 ++++++++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index b8ff04a..76b5176 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-ipld-format v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipfs/go-unixfsnode v1.7.3-0.20230718163022-62f4a1c8d46e + github.com/ipfs/go-unixfsnode v1.7.4 github.com/ipld/go-car v0.6.2 github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 github.com/ipld/go-codec-dagpb v1.6.0 diff --git a/go.sum b/go.sum index 67210f3..d536300 100644 --- a/go.sum +++ b/go.sum @@ -258,8 +258,8 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= -github.com/ipfs/go-unixfsnode v1.7.3-0.20230718163022-62f4a1c8d46e h1:LZkN2wQ49qj4g47ET4OgsI7gZ/8mFMbdXNvAoESC54U= -github.com/ipfs/go-unixfsnode v1.7.3-0.20230718163022-62f4a1c8d46e/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= +github.com/ipfs/go-unixfsnode v1.7.4 h1:iLvKyAVKUYOIAW2t4kDYqsT7VLGj31eXJE2aeqGfbwA= +github.com/ipfs/go-unixfsnode v1.7.4/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 69857ce..b42de0f 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -30,6 +30,7 @@ import ( "github.com/ipfs/go-unixfsnode" ufsData "github.com/ipfs/go-unixfsnode/data" "github.com/ipfs/go-unixfsnode/hamt" + ufsiter "github.com/ipfs/go-unixfsnode/iter" carv2 "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/storage" dagpb "github.com/ipld/go-codec-dagpb" @@ -694,6 +695,7 @@ func (it *backpressuredHAMTDirIterNoRecursion) Next() bool { it.err = err return false } + var lnk ipld.Link lnk, err = v.AsLink() if err != nil { @@ -709,10 +711,21 @@ func (it *backpressuredHAMTDirIterNoRecursion) Next() bool { c := cl.Cid + pbLnk, ok := v.(*ufsiter.IterLink) + if !ok { + it.err = fmt.Errorf("HAMT value is not a dag-pb link") + return false + } + + cumulativeDagSize := uint64(0) + if pbLnk.Substrate.Tsize.Exists() { + cumulativeDagSize = uint64(pbLnk.Substrate.Tsize.Must().Int()) + } + it.curLnk = unixfs.LinkResult{ Link: &format.Link{ Name: name, - Size: 0, + Size: cumulativeDagSize, Cid: c, }, } From e2be930e39955d8f9d9adfff403902afc4cd0a90 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 10 Aug 2023 07:24:40 -0400 Subject: [PATCH 36/46] return gateway errors on CAR fetching failure --- lib/graph_gateway.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index b42de0f..91ebd99 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -243,6 +243,8 @@ func (api *GraphGateway) fetchCAR(ctx context.Context, path gateway.ImmutablePat if ipldError != nil { fetchErr = ipldError + } else if fetchErr != nil { + fetchErr = GatewayError(fetchErr) } return fetchErr From d8485e2f7436357cfd1e458ece6ab2b0973c0e87 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 10 Aug 2023 09:06:21 -0400 Subject: [PATCH 37/46] add verifcid check --- lib/gateway_traversal.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/gateway_traversal.go b/lib/gateway_traversal.go index 8cc320b..2dabc9b 100644 --- a/lib/gateway_traversal.go +++ b/lib/gateway_traversal.go @@ -5,13 +5,14 @@ import ( "context" "errors" "fmt" - "github.com/filecoin-saturn/caboose" "io" "sync" "time" + "github.com/filecoin-saturn/caboose" bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/gateway" + "github.com/ipfs/boxo/verifcid" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-unixfsnode" @@ -69,6 +70,9 @@ func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *Grap return func(ctx context.Context, c cid.Cid) (blocks.Block, error) { mx.Lock() defer mx.Unlock() + if err := verifcid.ValidateCid(c); err != nil { + return nil, err + } // initially set a higher timeout here so that if there's an initial timeout error we get it from the car reader. var t *time.Timer From 1c54c91519a0e2c3fb16f43195cced66e0dda7ad Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 14 Aug 2023 21:17:52 +0200 Subject: [PATCH 38/46] test: run conformance with GRAPH_BACKEND=true --- .github/workflows/gateway-conformance.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 586f545..4d42df6 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -9,6 +9,9 @@ on: jobs: gateway-conformance: runs-on: ubuntu-latest + strategy: + matrix: + car-env: ["GRAPH_BACKEND=false", "GRAPH_BACKEND=true"] steps: # 1. Start the Kubo gateway - name: Download Kubo gateway @@ -59,8 +62,15 @@ jobs: - name: Start bifrost-gateway env: PROXY_GATEWAY_URL: http://127.0.0.1:8080 + KUBO_RPC_URL: http://127.0.0.1:5001 GATEWAY_CONFORMANCE_TEST: true - run: ./bifrost-gateway & + run: | + # NOTE: we export car-env this way ON PURPOSE, to have full key=value + # present on PR status instead of just true/false without context + export ${{ matrix.car-env }} + + # run gw + ./bifrost-gateway & working-directory: bifrost-gateway # 6. Run the gateway-conformance tests @@ -75,6 +85,10 @@ jobs: specs: +trustless-gateway,+path-gateway,+subdomain-gateway,+dnslink-gateway,+redirects-file # use below to skip specific test if needed # args: -skip 'TestFooBr/GET_response_for_something' + # + # only-if-cached: bifrost-gateway does not guarantee local cache, we will adjust upstream test (which was Kubo-specific) + # for now disabling these test cases + args: -skip 'TestGatewayCache/.*_for_/ipfs/_with_only-if-cached_succeeds_when_in_local_datastore' # 7. Upload the results - name: Upload MD summary From a978e34bd2eb423649cbcf59b73ba5bcc4176f3c Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 14 Aug 2023 20:26:27 -0400 Subject: [PATCH 39/46] fix: correctly decode UnixFS symlinks --- go.mod | 2 +- go.sum | 4 ++-- lib/files.go | 33 ++++++++++++++++++--------------- lib/graph_gateway.go | 13 ++++++++----- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 76b5176..9720a9a 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/filecoin-saturn/caboose v0.0.5-0.20230808125624-a986b542aa10 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru/v2 v2.0.1 - github.com/ipfs/boxo v0.11.1-0.20230803093408-3b83506a2ea5 + github.com/ipfs/boxo v0.11.1-0.20230811122208-5de8f5f5ca60 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-ipld-format v0.5.0 diff --git a/go.sum b/go.sum index d536300..d5bc1cc 100644 --- a/go.sum +++ b/go.sum @@ -201,8 +201,8 @@ github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7P github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.11.1-0.20230803093408-3b83506a2ea5 h1:noxhUh8BG2Yxsq2+2+bIFtZeeUTLEKwD5yCKc0Fg+WI= -github.com/ipfs/boxo v0.11.1-0.20230803093408-3b83506a2ea5/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= +github.com/ipfs/boxo v0.11.1-0.20230811122208-5de8f5f5ca60 h1:R8EYqB71skk4IxUlxVifUegUVEFZK4j+7nTdZF37Nus= +github.com/ipfs/boxo v0.11.1-0.20230811122208-5de8f5f5ca60/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/lib/files.go b/lib/files.go index 180e92d..602b818 100644 --- a/lib/files.go +++ b/lib/files.go @@ -273,13 +273,13 @@ func (it *backpressuredHAMTDirIter) Next() bool { continue } - _, pbn, fieldData, _, _, ufsBaseErr := loadUnixFSBase(it.ctx, it.dirCid, nil, lsys) + _, pbn, ufsFieldData, _, ufsBaseErr := loadUnixFSBase(it.ctx, it.dirCid, nil, lsys) if ufsBaseErr != nil { err = ufsBaseErr continue } - nd, err = hamt.NewUnixFSHAMTShard(it.ctx, pbn, fieldData, lsys) + nd, err = hamt.NewUnixFSHAMTShard(it.ctx, pbn, ufsFieldData, lsys) if err != nil { err = fmt.Errorf("could not reify sharded directory: %w", err) continue @@ -376,7 +376,7 @@ var _ files.DirIterator = (*backpressuredHAMTDirIter)(nil) type lsysGetter = func(ctx context.Context, c cid.Cid, params gateway.CarParams) (*ipld.LinkSystem, error) -func loadUnixFSBase(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld.LinkSystem) ([]byte, dagpb.PBNode, ufsData.UnixFSData, int64, []byte, error) { +func loadUnixFSBase(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld.LinkSystem) ([]byte, dagpb.PBNode, ufsData.UnixFSData, int64, error) { lctx := ipld.LinkContext{Ctx: ctx} pathTerminalCidLink := cidlink.Link{Cid: c} @@ -388,12 +388,12 @@ func loadUnixFSBase(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld } else { blockData, err = lsys.LoadRaw(lctx, pathTerminalCidLink) if err != nil { - return nil, nil, nil, 0, nil, err + return nil, nil, nil, 0, err } } if c.Type() == uint64(multicodec.Raw) { - return blockData, nil, nil, 0, nil, nil + return blockData, nil, nil, 0, nil } // decode the terminal block into a node @@ -406,33 +406,33 @@ func loadUnixFSBase(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld np, err := pc(pathTerminalCidLink, lctx) if err != nil { - return nil, nil, nil, 0, nil, err + return nil, nil, nil, 0, err } decoder, err := lsys.DecoderChooser(pathTerminalCidLink) if err != nil { - return nil, nil, nil, 0, nil, err + return nil, nil, nil, 0, err } nb := np.NewBuilder() if err := decoder(nb, bytes.NewReader(blockData)); err != nil { - return nil, nil, nil, 0, nil, err + return nil, nil, nil, 0, err } lastCidNode := nb.Build() if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { // If it's not valid dag-pb then we're done - return nil, nil, nil, 0, nil, errNotUnixFS + return nil, nil, nil, 0, errNotUnixFS } else if !pbn.FieldData().Exists() { // If it's not valid UnixFS then we're done - return nil, nil, nil, 0, nil, errNotUnixFS + return nil, nil, nil, 0, errNotUnixFS } else if unixfsFieldData, decodeErr := ufsData.DecodeUnixFSData(pbn.Data.Must().Bytes()); decodeErr != nil { - return nil, nil, nil, 0, nil, errNotUnixFS + return nil, nil, nil, 0, errNotUnixFS } else { switch fieldNum := unixfsFieldData.FieldDataType().Int(); fieldNum { case ufsData.Data_Symlink, ufsData.Data_Metadata, ufsData.Data_Raw, ufsData.Data_File, ufsData.Data_Directory, ufsData.Data_HAMTShard: - return nil, pbn, unixfsFieldData, fieldNum, pbn.FieldData().Must().Bytes(), nil + return nil, pbn, unixfsFieldData, fieldNum, nil default: - return nil, nil, nil, 0, nil, errNotUnixFS + return nil, nil, nil, 0, errNotUnixFS } } } @@ -447,7 +447,7 @@ func loadTerminalUnixFSElementWithRecursiveDirectories(ctx context.Context, c ci } lctx := ipld.LinkContext{Ctx: ctx} - blockData, pbn, _, fieldNum, fieldDataBytes, err := loadUnixFSBase(ctx, c, blk, lsys) + blockData, pbn, ufsFieldData, fieldNum, err := loadUnixFSBase(ctx, c, blk, lsys) if err != nil { return nil, err } @@ -458,7 +458,10 @@ func loadTerminalUnixFSElementWithRecursiveDirectories(ctx context.Context, c ci switch fieldNum { case ufsData.Data_Symlink: - lnkTarget := string(fieldDataBytes) + if !ufsFieldData.FieldData().Exists() { + return nil, fmt.Errorf("invalid UnixFS symlink object") + } + lnkTarget := string(ufsFieldData.FieldData().Must().Bytes()) f := files.NewLinkFile(lnkTarget, nil) return f, nil case ufsData.Data_Metadata: diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 91ebd99..6d82168 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -471,15 +471,18 @@ func loadTerminalEntity(ctx context.Context, c cid.Cid, blk blocks.Block, lsys * return gateway.NewGetResponseFromReader(files.NewBytesFile(blockData), int64(len(blockData))), nil } - blockData, pbn, _, fieldNum, fieldDataBytes, err := loadUnixFSBase(ctx, c, blk, lsys) + blockData, pbn, ufsFieldData, fieldNum, err := loadUnixFSBase(ctx, c, blk, lsys) if err != nil { return nil, err } switch fieldNum { case ufsData.Data_Symlink: - lnkTarget := string(fieldDataBytes) - f := gateway.NewGetResponseFromSymlink(files.NewLinkFile(lnkTarget, nil).(*files.Symlink), int64(len(fieldDataBytes))) + if !ufsFieldData.FieldData().Exists() { + return nil, fmt.Errorf("invalid UnixFS symlink object") + } + lnkTarget := string(ufsFieldData.FieldData().Must().Bytes()) + f := gateway.NewGetResponseFromSymlink(files.NewLinkFile(lnkTarget, nil).(*files.Symlink), int64(len(lnkTarget))) return f, nil case ufsData.Data_Metadata: return nil, fmt.Errorf("UnixFS Metadata unsupported") @@ -656,13 +659,13 @@ func (it *backpressuredHAMTDirIterNoRecursion) Next() bool { continue } - _, pbn, fieldData, _, _, ufsBaseErr := loadUnixFSBase(it.ctx, it.dirCid, nil, lsys) + _, pbn, ufsFieldData, _, ufsBaseErr := loadUnixFSBase(it.ctx, it.dirCid, nil, lsys) if ufsBaseErr != nil { err = ufsBaseErr continue } - nd, err = hamt.NewUnixFSHAMTShard(it.ctx, pbn, fieldData, lsys) + nd, err = hamt.NewUnixFSHAMTShard(it.ctx, pbn, ufsFieldData, lsys) if err != nil { err = fmt.Errorf("could not reify sharded directory: %w", err) continue From 4c5e7f959575364992fdfdf94f9f7a9cf7ea0c33 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 15 Aug 2023 01:11:44 -0400 Subject: [PATCH 40/46] fix: return starting at the correct offset when doing a Get request that happens to be for a raw block --- lib/graph_gateway.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 6d82168..7d26d42 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -468,7 +468,14 @@ func loadTerminalEntity(ctx context.Context, c cid.Cid, blk blocks.Block, lsys * } } - return gateway.NewGetResponseFromReader(files.NewBytesFile(blockData), int64(len(blockData))), nil + f := files.NewBytesFile(blockData) + if params.Range != nil && params.Range.From != 0 { + if _, err := f.Seek(params.Range.From, io.SeekStart); err != nil { + return nil, err + } + } + + return gateway.NewGetResponseFromReader(f, int64(len(blockData))), nil } blockData, pbn, ufsFieldData, fieldNum, err := loadUnixFSBase(ctx, c, blk, lsys) From ce2ef8948ae8db7918a4a1580f6725e04b88cb0d Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 15 Aug 2023 02:20:42 -0400 Subject: [PATCH 41/46] fix: add identity-blockstore style behavior to carToLinearBlockGetter --- go.mod | 2 +- lib/gateway_traversal.go | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 9720a9a..3de628e 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/libp2p/go-libp2p-routing-helpers v0.7.0 github.com/mitchellh/go-server-timing v1.0.1 github.com/multiformats/go-multicodec v0.9.0 + github.com/multiformats/go-multihash v0.2.3 github.com/prometheus/client_golang v1.15.1 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 github.com/spf13/cobra v1.6.1 @@ -94,7 +95,6 @@ require ( github.com/multiformats/go-multiaddr v0.8.0 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect - github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a // indirect diff --git a/lib/gateway_traversal.go b/lib/gateway_traversal.go index 2dabc9b..7e06177 100644 --- a/lib/gateway_traversal.go +++ b/lib/gateway_traversal.go @@ -28,6 +28,7 @@ import ( "github.com/ipld/go-ipld-prime/traversal" "github.com/ipld/go-ipld-prime/traversal/selector" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" + "github.com/multiformats/go-multihash" ) type getBlock func(ctx context.Context, cid cid.Cid) (blocks.Block, error) @@ -74,6 +75,11 @@ func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *Grap return nil, err } + isId, bdata := extractIdentityMultihashCIDContents(c) + if isId { + return blocks.NewBlockWithCid(bdata, c) + } + // initially set a higher timeout here so that if there's an initial timeout error we get it from the car reader. var t *time.Timer if isFirstBlock { @@ -109,6 +115,22 @@ func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *Grap }, nil } +// extractIdentityMultihashCIDContents will check if a given CID has an identity multihash and if so return true and +// the bytes encoded in the digest, otherwise will return false. +// Taken from https://github.com/ipfs/boxo/blob/b96767cc0971ca279feb36e7844e527a774309ab/blockstore/idstore.go#L30 +func extractIdentityMultihashCIDContents(k cid.Cid) (bool, []byte) { + // Pre-check by calling Prefix(), this much faster than extracting the hash. + if k.Prefix().MhType != multihash.IDENTITY { + return false, nil + } + + dmh, err := multihash.Decode(k.Hash()) + if err != nil || dmh.Code != multihash.IDENTITY { + return false, nil + } + return true, dmh.Digest +} + func getLinksystem(fn getBlock) *ipld.LinkSystem { lsys := cidlink.DefaultLinkSystem() lsys.StorageReadOpener = func(linkContext linking.LinkContext, link datamodel.Link) (io.Reader, error) { From 1275db54d051230c0aef1031a9c9a065e65b9dda Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 15 Aug 2023 01:20:15 -0400 Subject: [PATCH 42/46] chore: add guard to GetCar so we don't accidentally forget to handle a new dag-order --- lib/graph_gateway.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 7d26d42..84e5453 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -1147,6 +1147,12 @@ func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath, } p := ipfspath.FromString(path.String()) + switch params.Order { + case gateway.DagOrderUnspecified, gateway.DagOrderUnknown, gateway.DagOrderDFS: + default: + return gateway.ContentPathMetadata{}, nil, fmt.Errorf("unsupported dag order %q", params.Order) + } + r, w := io.Pipe() go func() { numBlocksSent := 0 From e60557645b4c4052ba370dbc437b6ac222b1d471 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 15 Aug 2023 02:14:55 -0400 Subject: [PATCH 43/46] feat: handle IPIP-412 requests with and without duplicates, backed by CAR fetches for duplicates --- lib/gateway_traversal.go | 28 ++++++++++++++-------------- lib/graph_gateway.go | 4 ++-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/gateway_traversal.go b/lib/gateway_traversal.go index 7e06177..c2a9a1a 100644 --- a/lib/gateway_traversal.go +++ b/lib/gateway_traversal.go @@ -147,12 +147,12 @@ func getLinksystem(fn getBlock) *ipld.LinkSystem { } // walkGatewaySimpleSelector walks the subgraph described by the path and terminal element parameters -func walkGatewaySimpleSelector(ctx context.Context, terminalBlk blocks.Block, params gateway.CarParams, lsys *ipld.LinkSystem) error { +func walkGatewaySimpleSelector(ctx context.Context, terminalBlk blocks.Block, dagScope gateway.DagScope, entityRange *gateway.DagByteRange, lsys *ipld.LinkSystem) error { lctx := ipld.LinkContext{Ctx: ctx} var err error // If the scope is the block, we only need the root block of the last element of the path, which we have. - if params.Scope == gateway.DagScopeBlock { + if dagScope == gateway.DagScopeBlock { return nil } @@ -186,7 +186,7 @@ func walkGatewaySimpleSelector(ctx context.Context, terminalBlk blocks.Block, pa // Does it matter that we're using a linksystem with the UnixFS reifier for dagscope=all? // If we're asking for everything then give it - if params.Scope == gateway.DagScopeAll { + if dagScope == gateway.DagScopeAll { sel, err := selector.ParseSelector(selectorparse.CommonSelector_ExploreAllRecursively) if err != nil { return err @@ -197,7 +197,7 @@ func walkGatewaySimpleSelector(ctx context.Context, terminalBlk blocks.Block, pa Ctx: ctx, LinkSystem: *lsys, LinkTargetNodePrototypeChooser: bsfetcher.DefaultPrototypeChooser, - LinkVisitOnlyOnce: true, // This is safe for the "all" selector + LinkVisitOnlyOnce: false, // Despite being safe for the "all" selector we do this walk anyway since this is how we will be receiving the blocks }, } @@ -255,29 +255,29 @@ func walkGatewaySimpleSelector(ctx context.Context, terminalBlk blocks.Block, pa } // Get the entity range. If it's empty, assume the defaults (whole file). - entityRange := params.Range - if entityRange == nil { - entityRange = &gateway.DagByteRange{ + effectiveRange := entityRange + if effectiveRange == nil { + effectiveRange = &gateway.DagByteRange{ From: 0, } } - from := entityRange.From + from := effectiveRange.From // If we're starting to read based on the end of the file, find out where that is. var fileLength int64 foundFileLength := false - if entityRange.From < 0 { + if effectiveRange.From < 0 { fileLength, err = f.Seek(0, io.SeekEnd) if err != nil { return err } - from = fileLength + entityRange.From + from = fileLength + effectiveRange.From foundFileLength = true } // If we're reading until the end of the file then do it - if entityRange.To == nil { + if effectiveRange.To == nil { if _, err := f.Seek(from, io.SeekStart); err != nil { return err } @@ -285,13 +285,13 @@ func walkGatewaySimpleSelector(ctx context.Context, terminalBlk blocks.Block, pa return err } - to := *entityRange.To - if (*entityRange.To) < 0 && !foundFileLength { + to := *effectiveRange.To + if (*effectiveRange.To) < 0 && !foundFileLength { fileLength, err = f.Seek(0, io.SeekEnd) if err != nil { return err } - to = fileLength + *entityRange.To + to = fileLength + *effectiveRange.To foundFileLength = true } diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 84e5453..2d47eae 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -1195,7 +1195,7 @@ func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath, } if cw == nil { - cw, err = storage.NewWritable(w, []cid.Cid{terminalCid}, carv2.WriteAsCarV1(true)) + cw, err = storage.NewWritable(w, []cid.Cid{terminalCid}, carv2.WriteAsCarV1(true), carv2.AllowDuplicatePuts(params.Duplicates.Bool())) if err != nil { // io.PipeWriter.CloseWithError always returns nil. _ = w.CloseWithError(err) @@ -1211,7 +1211,7 @@ func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath, blockBuffer = nil } - err = walkGatewaySimpleSelector(ctx, terminalBlk, params, l) + err = walkGatewaySimpleSelector(ctx, terminalBlk, params.Scope, params.Range, l) if err != nil { return err } From c23e66f4aac21156721a88fd1e7325b06097e56c Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 15 Aug 2023 02:41:22 -0400 Subject: [PATCH 44/46] ci: use a kubo development commit for conformance tests This is to handle a bug where if there is a CAR request for a non-existent path an error is returned rather than a CAR that proves that the requested path cannot exist. --- .github/workflows/gateway-conformance.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 4d42df6..162ba98 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -14,8 +14,16 @@ jobs: car-env: ["GRAPH_BACKEND=false", "GRAPH_BACKEND=true"] steps: # 1. Start the Kubo gateway - - name: Download Kubo gateway - uses: ipfs/download-ipfs-distribution-action@v1 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.x + - uses: protocol/cache-go-action@v1 + + - name: Install Kubo gateway from source + #uses: ipfs/download-ipfs-distribution-action@v1 + run: | + go install github.com/ipfs/kubo/cmd/ipfs@73552dfd9c6f73fdab887c4d2e99cf7e848a3c23 - name: Start Kubo gateway uses: ipfs/start-ipfs-daemon-action@v1 @@ -46,10 +54,6 @@ jobs: echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV # 4. Build the bifrost-gateway - - name: Setup Go - uses: actions/setup-go@v3 - with: - go-version: 1.19.x - name: Checkout bifrost-gateway uses: actions/checkout@v3 with: From 983ad966d002e8baf4675c72b82f84cdfd3e3ff6 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 15 Aug 2023 03:04:26 -0400 Subject: [PATCH 45/46] test: fix TestGetHAMTDirectory to not look for //index.html --- graph_gateway_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graph_gateway_test.go b/graph_gateway_test.go index 7b979b9..8d70906 100644 --- a/graph_gateway_test.go +++ b/graph_gateway_test.go @@ -853,7 +853,7 @@ func TestGetHAMTDirectory(t *testing.T) { // Expect a request for a non-existent index.html file // Note: this is an implementation detail related to the directory request above // Note: the order of cases 3 and 4 here are implementation specific as well - expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir//index.html" + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/index.html" if request.URL.Path != expectedUri { panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) } From 004ea10d4159498719b166f966daaa524d7f1250 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 15 Aug 2023 03:35:31 -0400 Subject: [PATCH 46/46] test: fix CAR retry tests to use CarWriter that allows duplicate blocks --- graph_gateway_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graph_gateway_test.go b/graph_gateway_test.go index 8d70906..88db33a 100644 --- a/graph_gateway_test.go +++ b/graph_gateway_test.go @@ -428,7 +428,7 @@ func sendBlocks(ctx context.Context, carFixture []byte, writer io.Writer, cidStr return err } - cw, err := storage.NewWritable(writer, []cid.Cid{cid.MustParse("bafkqaaa")}, carv2.WriteAsCarV1(true)) + cw, err := storage.NewWritable(writer, []cid.Cid{cid.MustParse("bafkqaaa")}, carv2.WriteAsCarV1(true), carv2.AllowDuplicatePuts(true)) if err != nil { return err }