From 682c21e4997904b84e6cc9bd90289ce6745f5e8e Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 7 Dec 2022 16:55:17 +0100 Subject: [PATCH] feat: Routing.Type=auto (DHT+IPNI) This changes the default routing to to be implicit DHT+IPNI Full context: https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084 --- cmd/ipfs/daemon.go | 29 +++---- config/init.go | 2 +- config/profile.go | 2 +- config/routing.go | 7 +- config/routing_test.go | 4 +- core/core_test.go | 2 +- core/node/libp2p/routingopt.go | 60 +++++++++++++ docs/config.md | 85 ++++++++++--------- .../migrations/ipfsfetcher/ipfsfetcher.go | 2 +- routing/delegated.go | 16 ++++ 10 files changed, 142 insertions(+), 67 deletions(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 1bfb9d6f1137..9ef4134140b5 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -61,6 +61,7 @@ const ( routingOptionNoneKwd = "none" routingOptionCustomKwd = "custom" routingOptionDefaultKwd = "default" + routingOptionAutoKwd = "auto" unencryptTransportKwd = "disable-transport-encryption" unrestrictedAPIAccessKwd = "unrestricted-api" writableKwd = "writable" @@ -89,7 +90,7 @@ For example, to change the 'Gateway' port: ipfs config Addresses.Gateway /ip4/127.0.0.1/tcp/8082 -The API address can be changed the same way: +The RPC API address can be changed the same way: ipfs config Addresses.API /ip4/127.0.0.1/tcp/5002 @@ -100,14 +101,14 @@ other computers in the network, use 0.0.0.0 as the ip address: ipfs config Addresses.Gateway /ip4/0.0.0.0/tcp/8080 -Be careful if you expose the API. It is a security risk, as anyone could +Be careful if you expose the RPC API. It is a security risk, as anyone could control your node remotely. If you need to control the node remotely, make sure to protect the port as you would other services or database (firewall, authenticated proxy, etc). HTTP Headers -ipfs supports passing arbitrary headers to the API and Gateway. You can +ipfs supports passing arbitrary headers to the RPC API and Gateway. You can do this by setting headers on the API.HTTPHeaders and Gateway.HTTPHeaders keys: @@ -141,18 +142,6 @@ environment variable: export IPFS_PATH=/path/to/ipfsrepo -Routing - -IPFS by default will use a DHT for content routing. There is an alternative -that operates the DHT in a 'client only' mode that can be enabled by -running the daemon as: - - ipfs daemon --routing=dhtclient - -Or you can set routing to dhtclient in the config: - - ipfs config Routing.Type dhtclient - DEPRECATION NOTICE Previously, ipfs used an environment variable as seen below: @@ -402,14 +391,20 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment routingOption, _ := req.Options[routingOptionKwd].(string) if routingOption == routingOptionDefaultKwd { - routingOption = cfg.Routing.Type + routingOption = cfg.Routing.Type.WithDefault(routingOptionAutoKwd) if routingOption == "" { - routingOption = routingOptionDHTKwd + routingOption = routingOptionAutoKwd } } switch routingOption { case routingOptionSupernodeKwd: return errors.New("supernode routing was never fully implemented and has been removed") + case routingOptionDefaultKwd, routingOptionAutoKwd: + ncfg.Routing = libp2p.ConstructDefaultRouting( + cfg.Identity.PeerID, + cfg.Addresses.Swarm, + cfg.Identity.PrivKey, + ) case routingOptionDHTClientKwd: ncfg.Routing = libp2p.DHTClientOption case routingOptionDHTKwd: diff --git a/config/init.go b/config/init.go index f86317369f55..4b936675796e 100644 --- a/config/init.go +++ b/config/init.go @@ -48,7 +48,7 @@ func InitWithIdentity(identity Identity) (*Config, error) { }, Routing: Routing{ - Type: "dht", + Type: nil, Methods: nil, Routers: nil, }, diff --git a/config/profile.go b/config/profile.go index 6748b5fb2b73..3e2766ce4762 100644 --- a/config/profile.go +++ b/config/profile.go @@ -174,7 +174,7 @@ functionality - performance of content discovery and data fetching may be degraded. `, Transform: func(c *Config) error { - c.Routing.Type = "dhtclient" + c.Routing.Type = NewOptionalString("dhtclient") // TODO: replace 'dhtclient' with 'autoclient' that prioritizes HTTP and uses dhtclient only after some delay as a fallback c.AutoNAT.ServiceMode = AutoNATServiceDisabled c.Reprovider.Interval = "0" diff --git a/config/routing.go b/config/routing.go index 90d91d28b1b0..983e8606db85 100644 --- a/config/routing.go +++ b/config/routing.go @@ -10,9 +10,10 @@ import ( type Routing struct { // Type sets default daemon routing mode. // - // Can be one of "dht", "dhtclient", "dhtserver", "none", or "custom". - // When "custom" is set, you can specify a list of Routers. - Type string + // Can be one of "auto", "dht", "dhtclient", "dhtserver", "none", or "custom". + // When unset or set to "auto", DHT and implicit routers are used. + // When "custom" is set, user-provided Routing.Routers is used. + Type *OptionalString `json:",omitempty"` Routers Routers diff --git a/config/routing_test.go b/config/routing_test.go index 013e285a5db8..49068f976d96 100644 --- a/config/routing_test.go +++ b/config/routing_test.go @@ -13,7 +13,7 @@ func TestRouterParameters(t *testing.T) { sec := time.Second min := time.Minute r := Routing{ - Type: "custom", + Type: NewOptionalString("custom"), Routers: map[string]RouterParser{ "router-dht": {Router{ Type: RouterTypeDHT, @@ -113,7 +113,7 @@ func TestRouterMissingParameters(t *testing.T) { require := require.New(t) r := Routing{ - Type: "custom", + Type: NewOptionalString("custom"), Routers: map[string]RouterParser{ "router-wrong-reframe": {Router{ Type: RouterTypeReframe, diff --git a/core/core_test.go b/core/core_test.go index 488eb8421ec1..57c9e338287e 100644 --- a/core/core_test.go +++ b/core/core_test.go @@ -221,7 +221,7 @@ func GetNode(t *testing.T, reframeURLs ...string) *IpfsNode { API: []string{"/ip4/127.0.0.1/tcp/0"}, }, Routing: config.Routing{ - Type: "custom", + Type: NewOptionalString("custom"), Routers: routers, Methods: config.Methods{ config.MethodNameFindPeers: config.Method{ diff --git a/core/node/libp2p/routingopt.go b/core/node/libp2p/routingopt.go index 9f3ae5c06e34..df553ace4f33 100644 --- a/core/node/libp2p/routingopt.go +++ b/core/node/libp2p/routingopt.go @@ -2,6 +2,7 @@ package libp2p import ( "context" + "time" "github.com/ipfs/go-datastore" "github.com/ipfs/kubo/config" @@ -23,6 +24,64 @@ type RoutingOption func( ...peer.AddrInfo, ) (routing.Routing, error) +// Default HTTP routers used in parallel to DHT when Routing.Type = "auto" +var defaultHTTPRouters = []string{ + "https://dev.cid.contact/routing/v1", // TODO: remove when cid.contact is ready + "https://cid.contact/routing/v1", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084 + // TODO: add an independent router from Cloudflare +} + +// ConstructDefaultRouting returns routers used when Routing.Type is unset or set to "auto" +func ConstructDefaultRouting(peerID string, addrs []string, privKey string) func( + ctx context.Context, + host host.Host, + dstore datastore.Batching, + validator record.Validator, + bootstrapPeers ...peer.AddrInfo, +) (routing.Routing, error) { + return func( + ctx context.Context, + host host.Host, + dstore datastore.Batching, + validator record.Validator, + bootstrapPeers ...peer.AddrInfo, + ) (routing.Routing, error) { + // Defined routers will be queried in parallel (optimizing for response speed) + // Different trade-offs can be made by setting Routing.Type = "custom" with own Routing.Routers + var routers []*routinghelpers.ParallelRouter + + // Run the default DHT routing (same as Routing.Type = "dht") + dhtRouting, err := DHTOption(ctx, host, dstore, validator, bootstrapPeers...) + if err != nil { + return nil, err + } + routers = append(routers, &routinghelpers.ParallelRouter{ + Router: dhtRouting, + IgnoreError: false, // TODO: confirm this is what we want + Timeout: 1 * time.Minute, // TODO: what value makes sense here? + ExecuteAfter: 0, + }) + + // Append HTTP routers for additional speed + for _, endpoint := range defaultHTTPRouters { + httpRouter, err := irouting.ConstructHTTPRouter(endpoint, peerID, addrs, privKey) + if err != nil { + return nil, err + } + routers = append(routers, &routinghelpers.ParallelRouter{ + Router: httpRouter, + IgnoreError: true, // TODO: confirm this is what we want + Timeout: 1 * time.Minute, // TODO: what value makes sense here? + ExecuteAfter: 0, + }) + } + + routing := routinghelpers.NewComposableParallel(routers) + return routing, nil + } +} + +// constructDHTRouting is used when Routing.Type = "dht" func constructDHTRouting(mode dht.ModeOpt) func( ctx context.Context, host host.Host, @@ -49,6 +108,7 @@ func constructDHTRouting(mode dht.ModeOpt) func( } } +// ConstructDelegatedRouting is used when Routing.Type = "custom" func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs []string, privKey string) func( ctx context.Context, host host.Host, diff --git a/docs/config.md b/docs/config.md index 1946d157d201..6af7667f662a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -105,11 +105,11 @@ config file at runtime. - [`Reprovider.Interval`](#reproviderinterval) - [`Reprovider.Strategy`](#reproviderstrategy) - [`Routing`](#routing) + - [`Routing.Type`](#routingtype) - [`Routing.Routers`](#routingrouters) - [`Routing.Routers: Type`](#routingrouters-type) - [`Routing.Routers: Parameters`](#routingrouters-parameters) - [`Routing: Methods`](#routing-methods) - - [`Routing.Type`](#routingtype) - [`Swarm`](#swarm) - [`Swarm.AddrFilters`](#swarmaddrfilters) - [`Swarm.DisableBandwidthMetrics`](#swarmdisablebandwidthmetrics) @@ -1322,6 +1322,49 @@ Type: `string` (or unset for the default, which is "all") Contains options for content, peer, and IPNS routing mechanisms. +### `Routing.Type` + +There are multiple routing options: "auto", "none", "dht" and "custom". + +* **DEFAULT:** If unset, or set to "auto", your node will use the IPFS DHT + and parallel HTTP routers listed below for additional speed. + +* If set to "none", your node will use _no_ routing system. You'll have to + explicitly connect to peers that have the content you're looking for. + +* If set to "dht" (or "dhtclient"/"dhtserver"), your node will ONLY use the IPFS DHT (no HTTP routers). + +* If set to "custom", all default routers are disabled, and only onles defined in `Routing.Routers` will be used. + +When the DHT is enabled, it can operate in two modes: client and server. + +* In server mode, your node will query other peers for DHT records, and will + respond to requests from other peers (both requests to store records and + requests to retrieve records). + +* In client mode, your node will query the DHT as a client but will not respond + to requests from other peers. This mode is less resource-intensive than server + mode. + +When `Routing.Type` is set to `auto` or `dht`, your node will start as a DHT client, and +switch to a DHT server when and if it determines that it's reachable from the +public internet (e.g., it's not behind a firewall). + +To force a specific DHT-only mode, client or server, set `Routing.Type` to +`dhtclient` or `dhtserver` respectively. Please do not set this to `dhtserver` +unless you're sure your node is reachable from the public network. + +When `Routing.Type` is set to `auto` your node will accelerate some types of routing +by leveraging HTTP endpoints compatible with [IPIP-337](https://github.com/ipfs/specs/pull/337) +in addition to the IPFS DHT. +By default, an instance of [IPNI](https://github.com/ipni/specs/blob/main/IPNI.md#readme) +at https://cid.contact is used. +Alternative routing rules can be configured in `Routing.Routers` after setting `Routing.Type` to `custom`. + +Default: `auto` (DHT + IPNI) + +Type: `optionalString` (`null`/missing means the default) + ### `Routing.Routers` **EXPERIMENTAL: `Routing.Routers` configuration may change in future release** @@ -1463,46 +1506,6 @@ ipfs config Routing.Methods --json '{ ``` -### `Routing.Type` - -There are three core routing options: "none", "dht" (default) and "custom". - -* If set to "none", your node will use _no_ routing system. You'll have to - explicitly connect to peers that have the content you're looking for. -* If set to "dht" (or "dhtclient"/"dhtserver"), your node will use the IPFS DHT. -* If set to "custom", `Routing.Routers` will be used. - -When the DHT is enabled, it can operate in two modes: client and server. - -* In server mode, your node will query other peers for DHT records, and will - respond to requests from other peers (both requests to store records and - requests to retrieve records). -* In client mode, your node will query the DHT as a client but will not respond - to requests from other peers. This mode is less resource-intensive than server - mode. - -When `Routing.Type` is set to `dht`, your node will start as a DHT client, and -switch to a DHT server when and if it determines that it's reachable from the -public internet (e.g., it's not behind a firewall). - -To force a specific DHT mode, client or server, set `Routing.Type` to -`dhtclient` or `dhtserver` respectively. Please do not set this to `dhtserver` -unless you're sure your node is reachable from the public network. - -**Example:** - -```json -{ - "Routing": { - "Type": "dhtclient" - } -} -``` - -Default: `dht` - -Type: `optionalString` (`null`/missing means the default) - ## `Swarm` Options for configuring the swarm. diff --git a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go index ce7b490efd20..d13eaf1484aa 100644 --- a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go +++ b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go @@ -188,7 +188,7 @@ func initTempNode(ctx context.Context, bootstrap []string, peers []peer.AddrInfo } // configure the temporary node - cfg.Routing.Type = "dhtclient" + cfg.Routing.Type = config.NewOptionalString("dhtclient") // Disable listening for inbound connections cfg.Addresses.Gateway = []string{} diff --git a/routing/delegated.go b/routing/delegated.go index 953cfac00e08..1ad061e6c44e 100644 --- a/routing/delegated.go +++ b/routing/delegated.go @@ -157,6 +157,22 @@ type ExtraHTTPParams struct { PrivKeyB64 string } +func ConstructHTTPRouter(endpoint string, peerID string, addrs []string, privKey string) (routing.Routing, error) { + return httpRoutingFromConfig( + config.Router{ + Type: "http", + Parameters: &config.HTTPRouterParams{ + Endpoint: endpoint, + }, + }, + &ExtraHTTPParams{ + PeerID: peerID, + Addrs: addrs, + PrivKeyB64: privKey, + }, + ) +} + func httpRoutingFromConfig(conf config.Router, extraHTTP *ExtraHTTPParams) (routing.Routing, error) { params := conf.Parameters.(*config.HTTPRouterParams) if params.Endpoint == "" {