From 1b9459318c485c6bc0add1d2cff63b74f96b04b0 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 28 Oct 2023 01:31:44 +0200 Subject: [PATCH] fix: use offline path resolvers where appropriate this fixes the problem described in https://github.com/ipfs/kubo/pull/10161#issuecomment-1782175955 by adding explicit offline path resolvers that are backed by offline exchange, and using them in NoFetch gateways instead of the default online ones --- core/core.go | 58 ++++++++++++++------------ core/corehttp/gateway.go | 16 ++++--- core/node/core.go | 41 +++++++++++++----- docs/examples/kubo-as-a-library/go.mod | 4 +- docs/examples/kubo-as-a-library/go.sum | 8 ++-- go.mod | 4 +- go.sum | 8 ++-- plugin/plugins/nopfs/nopfs.go | 6 ++- 8 files changed, 88 insertions(+), 57 deletions(-) diff --git a/core/core.go b/core/core.go index c35d3e4457fe..c9bec6b8b00c 100644 --- a/core/core.go +++ b/core/core.go @@ -76,35 +76,39 @@ type IpfsNode struct { PNetFingerprint libp2p.PNetFingerprint `optional:"true"` // fingerprint of private network // Services - Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances - Blockstore bstore.GCBlockstore // the block store (lower level) - Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore - BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping - GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc - Blocks bserv.BlockService // the block service, get/add blocks. - DAG ipld.DAGService // the merkle dag service, get/add objects. - IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model - UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data - Reporter *metrics.BandwidthCounter `optional:"true"` - Discovery mdns.Service `optional:"true"` - FilesRoot *mfs.Root - RecordValidator record.Validator + Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances + Blockstore bstore.GCBlockstore // the block store (lower level) + Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore + BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping + GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc + Blocks bserv.BlockService // the block service, get/add blocks. + DAG ipld.DAGService // the merkle dag service, get/add objects. + IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model + UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data + OfflineIPLDFetcherFactory fetcher.Factory `name:"offlineIpldFetcher"` // fetcher that paths over the IPLD data model without fetching new blocks + OfflineUnixFSFetcherFactory fetcher.Factory `name:"offlineUnixfsFetcher"` // fetcher that interprets UnixFS data without fetching new blocks + Reporter *metrics.BandwidthCounter `optional:"true"` + Discovery mdns.Service `optional:"true"` + FilesRoot *mfs.Root + RecordValidator record.Validator // Online - PeerHost p2phost.Host `optional:"true"` // the network host (server+client) - Peering *peering.PeeringService `optional:"true"` - Filters *ma.Filters `optional:"true"` - Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper - Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht - DNSResolver *madns.Resolver // the DNS resolver - IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver - UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver - Exchange exchange.Interface // the block exchange + strategy (bitswap) - Namesys namesys.NameSystem // the name system, resolves paths to hashes - Provider provider.System // the value provider system - IpnsRepub *ipnsrp.Republisher `optional:"true"` - GraphExchange graphsync.GraphExchange `optional:"true"` - ResourceManager network.ResourceManager `optional:"true"` + PeerHost p2phost.Host `optional:"true"` // the network host (server+client) + Peering *peering.PeeringService `optional:"true"` + Filters *ma.Filters `optional:"true"` + Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper + Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht + DNSResolver *madns.Resolver // the DNS resolver + IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver + UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver + OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` // The IPLD path resolver that uses only locally available blocks + OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks + Exchange exchange.Interface // the block exchange + strategy (bitswap) + Namesys namesys.NameSystem // the name system, resolves paths to hashes + Provider provider.System // the value provider system + IpnsRepub *ipnsrp.Republisher `optional:"true"` + GraphExchange graphsync.GraphExchange `optional:"true"` + ResourceManager network.ResourceManager `optional:"true"` PubSub *pubsub.PubSub `optional:"true"` PSRouter *psrouter.PubsubValueStore `optional:"true"` diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index e21df146c8d6..3e0380d5a3c0 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -81,7 +81,11 @@ func Libp2pGatewayOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { bserv := blockservice.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore())) - backend, err := gateway.NewBlocksBackend(bserv) + backend, err := gateway.NewBlocksBackend(bserv, + // GatewayOverLibp2p only returns things that are in local blockstore + // (same as Gateway.NoFetch=true), we have to pass offline path resolver + gateway.WithResolver(n.OfflineUnixFSPathResolver), + ) if err != nil { return nil, err } @@ -111,7 +115,7 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) { bserv := n.Blocks var vsRouting routing.ValueStore = n.Routing nsys := n.Namesys - resolver := n.UnixFSPathResolver + pathResolver := n.UnixFSPathResolver if cfg.Gateway.NoFetch { bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore())) @@ -133,15 +137,15 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) { return nil, fmt.Errorf("error constructing namesys: %w", err) } - // Let NewBlocksBackend setup the default resolver using the - // offline backend. - resolver = nil + // Gateway.NoFetch=true requires offline path resolver + // to avoid fetching missing blocks during path traversal + pathResolver = n.OfflineUnixFSPathResolver } backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys), - gateway.WithResolver(resolver), + gateway.WithResolver(pathResolver), ) if err != nil { return nil, err diff --git a/core/node/core.go b/core/node/core.go index d2d2c63d749d..227e08a9a2ae 100644 --- a/core/node/core.go +++ b/core/node/core.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/boxo/blockservice" blockstore "github.com/ipfs/boxo/blockstore" exchange "github.com/ipfs/boxo/exchange" + offline "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/fetcher" bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/filestore" @@ -87,15 +88,19 @@ func (s *syncDagService) Session(ctx context.Context) format.NodeGetter { // FetchersOut allows injection of fetchers. type FetchersOut struct { fx.Out - IPLDFetcher fetcher.Factory `name:"ipldFetcher"` - UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` + IPLDFetcher fetcher.Factory `name:"ipldFetcher"` + UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` + OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"` + OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"` } // FetchersIn allows using fetchers for other dependencies. type FetchersIn struct { fx.In - IPLDFetcher fetcher.Factory `name:"ipldFetcher"` - UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` + IPLDFetcher fetcher.Factory `name:"ipldFetcher"` + UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` + OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"` + OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"` } // FetcherConfig returns a fetcher config that can build new fetcher instances @@ -107,23 +112,39 @@ func FetcherConfig(bs blockservice.BlockService) FetchersOut { } return basicnode.Prototype.Any, nil }) - unixFSFetcher := ipldFetcher.WithReifier(unixfsnode.Reify) - return FetchersOut{IPLDFetcher: ipldFetcher, UnixfsFetcher: unixFSFetcher} + + // Construct offline versions which we can safely use in contexts where + // path resolution should not fetch new blocks via exchange. + offlineBs := blockservice.New(bs.Blockstore(), offline.Exchange(bs.Blockstore())) + offlineIpldFetcher := bsfetcher.NewFetcherConfig(offlineBs) + offlineIpldFetcher.PrototypeChooser = ipldFetcher.PrototypeChooser + offlineUnixFSFetcher := offlineIpldFetcher.WithReifier(unixfsnode.Reify) + + return FetchersOut{ + IPLDFetcher: ipldFetcher, + UnixfsFetcher: unixFSFetcher, + OfflineIPLDFetcher: offlineIpldFetcher, + OfflineUnixfsFetcher: offlineUnixFSFetcher, + } } // PathResolversOut allows injection of path resolvers type PathResolversOut struct { fx.Out - IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` - UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` + IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` + UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` + OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` + OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` } // PathResolverConfig creates path resolvers with the given fetchers. func PathResolverConfig(fetchers FetchersIn) PathResolversOut { return PathResolversOut{ - IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher), - UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher), + IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher), + UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher), + OfflineIPLDPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineIPLDFetcher), + OfflineUnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineUnixfsFetcher), } } diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 4a7241bb9116..674f5b4f0dc6 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -61,8 +61,8 @@ require ( github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huin/goupnp v1.2.0 // indirect - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 // indirect - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 // indirect + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c // indirect + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 39e3913ef5f2..c582628193a7 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -299,10 +299,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 h1:fGhrSi5CgMQxw0OdTKjrYXMxKjbmCFOFh4GtD8CK6dA= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 h1:O2fhQrtzG44UiGOH8YRVtAtfjCTu+/TubeFymhz5Brg= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c h1:17FO7HnKiFhO7iadu3zCgII+EblpdRmJt5qg9FqQo8Y= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7UynTbtdlt+w08ggb1UGLGaGjp1mMaZhoTZSctpn5Ak= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= diff --git a/go.mod b/go.mod index 6bf184ebd482..d2584cec03e5 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,8 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/google/uuid v1.3.1 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 diff --git a/go.sum b/go.sum index acde89518765..245c74e457ba 100644 --- a/go.sum +++ b/go.sum @@ -333,10 +333,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 h1:fGhrSi5CgMQxw0OdTKjrYXMxKjbmCFOFh4GtD8CK6dA= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 h1:O2fhQrtzG44UiGOH8YRVtAtfjCTu+/TubeFymhz5Brg= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c h1:17FO7HnKiFhO7iadu3zCgII+EblpdRmJt5qg9FqQo8Y= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7UynTbtdlt+w08ggb1UGLGaGjp1mMaZhoTZSctpn5Ak= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= diff --git a/plugin/plugins/nopfs/nopfs.go b/plugin/plugins/nopfs/nopfs.go index a04734187ba2..64350830f94b 100644 --- a/plugin/plugins/nopfs/nopfs.go +++ b/plugin/plugins/nopfs/nopfs.go @@ -62,8 +62,10 @@ func MakeBlocker() (*nopfs.Blocker, error) { func PathResolvers(fetchers node.FetchersIn, blocker *nopfs.Blocker) node.PathResolversOut { res := node.PathResolverConfig(fetchers) return node.PathResolversOut{ - IPLDPathResolver: ipfs.WrapResolver(res.IPLDPathResolver, blocker), - UnixFSPathResolver: ipfs.WrapResolver(res.UnixFSPathResolver, blocker), + IPLDPathResolver: ipfs.WrapResolver(res.IPLDPathResolver, blocker), + UnixFSPathResolver: ipfs.WrapResolver(res.UnixFSPathResolver, blocker), + OfflineIPLDPathResolver: ipfs.WrapResolver(res.OfflineIPLDPathResolver, blocker), + OfflineUnixFSPathResolver: ipfs.WrapResolver(res.OfflineUnixFSPathResolver, blocker), } }