From 76a3c2cfbca81ee1f3ba70635026c620fe6f3ccb Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 7 Dec 2022 16:55:17 +0100 Subject: [PATCH 01/10] feat: Routing.Type=auto (DHT+IPNI) This changes the default routing to to be implicit DHT+IPNI Full context: https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084 --- cmd/ipfs/daemon.go | 29 +++---- config/init.go | 2 +- config/profile.go | 2 +- config/routing.go | 7 +- config/routing_test.go | 4 +- core/core_test.go | 2 +- core/node/libp2p/routingopt.go | 60 +++++++++++++ docs/changelogs/v0.18.md | 19 ++++- docs/config.md | 85 ++++++++++--------- .../migrations/ipfsfetcher/ipfsfetcher.go | 2 +- routing/delegated.go | 16 ++++ test/sharness/t0041-ping.sh | 2 +- test/sharness/t0170-legacy-dht.sh | 4 + test/sharness/t0170-routing-dht.sh | 6 +- 14 files changed, 170 insertions(+), 70 deletions(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 1bfb9d6f113..9ef4134140b 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -61,6 +61,7 @@ const ( routingOptionNoneKwd = "none" routingOptionCustomKwd = "custom" routingOptionDefaultKwd = "default" + routingOptionAutoKwd = "auto" unencryptTransportKwd = "disable-transport-encryption" unrestrictedAPIAccessKwd = "unrestricted-api" writableKwd = "writable" @@ -89,7 +90,7 @@ For example, to change the 'Gateway' port: ipfs config Addresses.Gateway /ip4/127.0.0.1/tcp/8082 -The API address can be changed the same way: +The RPC API address can be changed the same way: ipfs config Addresses.API /ip4/127.0.0.1/tcp/5002 @@ -100,14 +101,14 @@ other computers in the network, use 0.0.0.0 as the ip address: ipfs config Addresses.Gateway /ip4/0.0.0.0/tcp/8080 -Be careful if you expose the API. It is a security risk, as anyone could +Be careful if you expose the RPC API. It is a security risk, as anyone could control your node remotely. If you need to control the node remotely, make sure to protect the port as you would other services or database (firewall, authenticated proxy, etc). HTTP Headers -ipfs supports passing arbitrary headers to the API and Gateway. You can +ipfs supports passing arbitrary headers to the RPC API and Gateway. You can do this by setting headers on the API.HTTPHeaders and Gateway.HTTPHeaders keys: @@ -141,18 +142,6 @@ environment variable: export IPFS_PATH=/path/to/ipfsrepo -Routing - -IPFS by default will use a DHT for content routing. There is an alternative -that operates the DHT in a 'client only' mode that can be enabled by -running the daemon as: - - ipfs daemon --routing=dhtclient - -Or you can set routing to dhtclient in the config: - - ipfs config Routing.Type dhtclient - DEPRECATION NOTICE Previously, ipfs used an environment variable as seen below: @@ -402,14 +391,20 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment routingOption, _ := req.Options[routingOptionKwd].(string) if routingOption == routingOptionDefaultKwd { - routingOption = cfg.Routing.Type + routingOption = cfg.Routing.Type.WithDefault(routingOptionAutoKwd) if routingOption == "" { - routingOption = routingOptionDHTKwd + routingOption = routingOptionAutoKwd } } switch routingOption { case routingOptionSupernodeKwd: return errors.New("supernode routing was never fully implemented and has been removed") + case routingOptionDefaultKwd, routingOptionAutoKwd: + ncfg.Routing = libp2p.ConstructDefaultRouting( + cfg.Identity.PeerID, + cfg.Addresses.Swarm, + cfg.Identity.PrivKey, + ) case routingOptionDHTClientKwd: ncfg.Routing = libp2p.DHTClientOption case routingOptionDHTKwd: diff --git a/config/init.go b/config/init.go index f86317369f5..4b936675796 100644 --- a/config/init.go +++ b/config/init.go @@ -48,7 +48,7 @@ func InitWithIdentity(identity Identity) (*Config, error) { }, Routing: Routing{ - Type: "dht", + Type: nil, Methods: nil, Routers: nil, }, diff --git a/config/profile.go b/config/profile.go index 6748b5fb2b7..3e2766ce476 100644 --- a/config/profile.go +++ b/config/profile.go @@ -174,7 +174,7 @@ functionality - performance of content discovery and data fetching may be degraded. `, Transform: func(c *Config) error { - c.Routing.Type = "dhtclient" + c.Routing.Type = NewOptionalString("dhtclient") // TODO: replace 'dhtclient' with 'autoclient' that prioritizes HTTP and uses dhtclient only after some delay as a fallback c.AutoNAT.ServiceMode = AutoNATServiceDisabled c.Reprovider.Interval = "0" diff --git a/config/routing.go b/config/routing.go index 90d91d28b1b..983e8606db8 100644 --- a/config/routing.go +++ b/config/routing.go @@ -10,9 +10,10 @@ import ( type Routing struct { // Type sets default daemon routing mode. // - // Can be one of "dht", "dhtclient", "dhtserver", "none", or "custom". - // When "custom" is set, you can specify a list of Routers. - Type string + // Can be one of "auto", "dht", "dhtclient", "dhtserver", "none", or "custom". + // When unset or set to "auto", DHT and implicit routers are used. + // When "custom" is set, user-provided Routing.Routers is used. + Type *OptionalString `json:",omitempty"` Routers Routers diff --git a/config/routing_test.go b/config/routing_test.go index 013e285a5db..49068f976d9 100644 --- a/config/routing_test.go +++ b/config/routing_test.go @@ -13,7 +13,7 @@ func TestRouterParameters(t *testing.T) { sec := time.Second min := time.Minute r := Routing{ - Type: "custom", + Type: NewOptionalString("custom"), Routers: map[string]RouterParser{ "router-dht": {Router{ Type: RouterTypeDHT, @@ -113,7 +113,7 @@ func TestRouterMissingParameters(t *testing.T) { require := require.New(t) r := Routing{ - Type: "custom", + Type: NewOptionalString("custom"), Routers: map[string]RouterParser{ "router-wrong-reframe": {Router{ Type: RouterTypeReframe, diff --git a/core/core_test.go b/core/core_test.go index 488eb8421ec..1d0703de076 100644 --- a/core/core_test.go +++ b/core/core_test.go @@ -221,7 +221,7 @@ func GetNode(t *testing.T, reframeURLs ...string) *IpfsNode { API: []string{"/ip4/127.0.0.1/tcp/0"}, }, Routing: config.Routing{ - Type: "custom", + Type: config.NewOptionalString("custom"), Routers: routers, Methods: config.Methods{ config.MethodNameFindPeers: config.Method{ diff --git a/core/node/libp2p/routingopt.go b/core/node/libp2p/routingopt.go index 9f3ae5c06e3..df553ace4f3 100644 --- a/core/node/libp2p/routingopt.go +++ b/core/node/libp2p/routingopt.go @@ -2,6 +2,7 @@ package libp2p import ( "context" + "time" "github.com/ipfs/go-datastore" "github.com/ipfs/kubo/config" @@ -23,6 +24,64 @@ type RoutingOption func( ...peer.AddrInfo, ) (routing.Routing, error) +// Default HTTP routers used in parallel to DHT when Routing.Type = "auto" +var defaultHTTPRouters = []string{ + "https://dev.cid.contact/routing/v1", // TODO: remove when cid.contact is ready + "https://cid.contact/routing/v1", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084 + // TODO: add an independent router from Cloudflare +} + +// ConstructDefaultRouting returns routers used when Routing.Type is unset or set to "auto" +func ConstructDefaultRouting(peerID string, addrs []string, privKey string) func( + ctx context.Context, + host host.Host, + dstore datastore.Batching, + validator record.Validator, + bootstrapPeers ...peer.AddrInfo, +) (routing.Routing, error) { + return func( + ctx context.Context, + host host.Host, + dstore datastore.Batching, + validator record.Validator, + bootstrapPeers ...peer.AddrInfo, + ) (routing.Routing, error) { + // Defined routers will be queried in parallel (optimizing for response speed) + // Different trade-offs can be made by setting Routing.Type = "custom" with own Routing.Routers + var routers []*routinghelpers.ParallelRouter + + // Run the default DHT routing (same as Routing.Type = "dht") + dhtRouting, err := DHTOption(ctx, host, dstore, validator, bootstrapPeers...) + if err != nil { + return nil, err + } + routers = append(routers, &routinghelpers.ParallelRouter{ + Router: dhtRouting, + IgnoreError: false, // TODO: confirm this is what we want + Timeout: 1 * time.Minute, // TODO: what value makes sense here? + ExecuteAfter: 0, + }) + + // Append HTTP routers for additional speed + for _, endpoint := range defaultHTTPRouters { + httpRouter, err := irouting.ConstructHTTPRouter(endpoint, peerID, addrs, privKey) + if err != nil { + return nil, err + } + routers = append(routers, &routinghelpers.ParallelRouter{ + Router: httpRouter, + IgnoreError: true, // TODO: confirm this is what we want + Timeout: 1 * time.Minute, // TODO: what value makes sense here? + ExecuteAfter: 0, + }) + } + + routing := routinghelpers.NewComposableParallel(routers) + return routing, nil + } +} + +// constructDHTRouting is used when Routing.Type = "dht" func constructDHTRouting(mode dht.ModeOpt) func( ctx context.Context, host host.Host, @@ -49,6 +108,7 @@ func constructDHTRouting(mode dht.ModeOpt) func( } } +// ConstructDelegatedRouting is used when Routing.Type = "custom" func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs []string, privKey string) func( ctx context.Context, host host.Host, diff --git a/docs/changelogs/v0.18.md b/docs/changelogs/v0.18.md index 4afa6793ea9..35f1489488f 100644 --- a/docs/changelogs/v0.18.md +++ b/docs/changelogs/v0.18.md @@ -11,6 +11,7 @@ Below is an outline of all that is in this release, so you get a sense of all th - [Overview](#overview) - [🔦 Highlights](#-highlights) - [(DAG-)JSON and (DAG-)CBOR Response Formats on Gateways](#dag-json-and-dag-cbor-response-formats-on-gateways) + - [Automated default `Routing.Type`](#automated-default-routingtype) - [Changelog](#changelog) - [Contributors](#contributors) @@ -21,7 +22,7 @@ Below is an outline of all that is in this release, so you get a sense of all th Implemented [IPIP-328](https://github.com/ipfs/specs/pull/328) which adds support to DAG-JSON and DAG-CBOR, as well as their non-DAG variants, to the gateway. Now, CIDs that encode JSON, CBOR, DAG-JSON and DAG-CBOR objects can be retrieved, and -traversed through IPLD Links. +traversed thanks to the [special meaning of CBOR Tag 42](https://github.com/ipld/cid-cbor/). HTTP clients can request JSON, CBOR, DAG-JSON, and DAG-CBOR responses by either passing the query parameter `?format` or setting the `Accept` HTTP header to the @@ -68,6 +69,22 @@ $ curl "http://127.0.0.1:8080/ipfs/$DIR_CID?format=dag-json" | jq } ``` +#### Automated default `Routing.Type` + +The default `Routing.Type` is changed from `dht` to `auto`. + +When `Routing.Type` is unset or set to `auto`, your node will accelerate some types of routing +by leveraging HTTP endpoints compatible with [IPIP-337](https://github.com/ipfs/specs/pull/337) +in addition to the IPFS DHT. +By default, an instance of IPNI ([InterPlanetary Network Indexer](https://github.com/ipni/specs/blob/main/IPNI.md#readme)) +at https://cid.contact is used. + +Old behavior can be restored by setting `Routing.Type` to `dht`. +Alternative routing rules, including alternative IPNI endpoints, can be configured in `Routing.Routers` after setting `Routing.Type` to `custom`. + +Learn more in [`Routing` docs](https://github.com/ipfs/kubo/blob/master/docs/config.md#routing). + + ### Changelog ### Contributors diff --git a/docs/config.md b/docs/config.md index 1946d157d20..6af7667f662 100644 --- a/docs/config.md +++ b/docs/config.md @@ -105,11 +105,11 @@ config file at runtime. - [`Reprovider.Interval`](#reproviderinterval) - [`Reprovider.Strategy`](#reproviderstrategy) - [`Routing`](#routing) + - [`Routing.Type`](#routingtype) - [`Routing.Routers`](#routingrouters) - [`Routing.Routers: Type`](#routingrouters-type) - [`Routing.Routers: Parameters`](#routingrouters-parameters) - [`Routing: Methods`](#routing-methods) - - [`Routing.Type`](#routingtype) - [`Swarm`](#swarm) - [`Swarm.AddrFilters`](#swarmaddrfilters) - [`Swarm.DisableBandwidthMetrics`](#swarmdisablebandwidthmetrics) @@ -1322,6 +1322,49 @@ Type: `string` (or unset for the default, which is "all") Contains options for content, peer, and IPNS routing mechanisms. +### `Routing.Type` + +There are multiple routing options: "auto", "none", "dht" and "custom". + +* **DEFAULT:** If unset, or set to "auto", your node will use the IPFS DHT + and parallel HTTP routers listed below for additional speed. + +* If set to "none", your node will use _no_ routing system. You'll have to + explicitly connect to peers that have the content you're looking for. + +* If set to "dht" (or "dhtclient"/"dhtserver"), your node will ONLY use the IPFS DHT (no HTTP routers). + +* If set to "custom", all default routers are disabled, and only onles defined in `Routing.Routers` will be used. + +When the DHT is enabled, it can operate in two modes: client and server. + +* In server mode, your node will query other peers for DHT records, and will + respond to requests from other peers (both requests to store records and + requests to retrieve records). + +* In client mode, your node will query the DHT as a client but will not respond + to requests from other peers. This mode is less resource-intensive than server + mode. + +When `Routing.Type` is set to `auto` or `dht`, your node will start as a DHT client, and +switch to a DHT server when and if it determines that it's reachable from the +public internet (e.g., it's not behind a firewall). + +To force a specific DHT-only mode, client or server, set `Routing.Type` to +`dhtclient` or `dhtserver` respectively. Please do not set this to `dhtserver` +unless you're sure your node is reachable from the public network. + +When `Routing.Type` is set to `auto` your node will accelerate some types of routing +by leveraging HTTP endpoints compatible with [IPIP-337](https://github.com/ipfs/specs/pull/337) +in addition to the IPFS DHT. +By default, an instance of [IPNI](https://github.com/ipni/specs/blob/main/IPNI.md#readme) +at https://cid.contact is used. +Alternative routing rules can be configured in `Routing.Routers` after setting `Routing.Type` to `custom`. + +Default: `auto` (DHT + IPNI) + +Type: `optionalString` (`null`/missing means the default) + ### `Routing.Routers` **EXPERIMENTAL: `Routing.Routers` configuration may change in future release** @@ -1463,46 +1506,6 @@ ipfs config Routing.Methods --json '{ ``` -### `Routing.Type` - -There are three core routing options: "none", "dht" (default) and "custom". - -* If set to "none", your node will use _no_ routing system. You'll have to - explicitly connect to peers that have the content you're looking for. -* If set to "dht" (or "dhtclient"/"dhtserver"), your node will use the IPFS DHT. -* If set to "custom", `Routing.Routers` will be used. - -When the DHT is enabled, it can operate in two modes: client and server. - -* In server mode, your node will query other peers for DHT records, and will - respond to requests from other peers (both requests to store records and - requests to retrieve records). -* In client mode, your node will query the DHT as a client but will not respond - to requests from other peers. This mode is less resource-intensive than server - mode. - -When `Routing.Type` is set to `dht`, your node will start as a DHT client, and -switch to a DHT server when and if it determines that it's reachable from the -public internet (e.g., it's not behind a firewall). - -To force a specific DHT mode, client or server, set `Routing.Type` to -`dhtclient` or `dhtserver` respectively. Please do not set this to `dhtserver` -unless you're sure your node is reachable from the public network. - -**Example:** - -```json -{ - "Routing": { - "Type": "dhtclient" - } -} -``` - -Default: `dht` - -Type: `optionalString` (`null`/missing means the default) - ## `Swarm` Options for configuring the swarm. diff --git a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go index ce7b490efd2..d13eaf1484a 100644 --- a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go +++ b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go @@ -188,7 +188,7 @@ func initTempNode(ctx context.Context, bootstrap []string, peers []peer.AddrInfo } // configure the temporary node - cfg.Routing.Type = "dhtclient" + cfg.Routing.Type = config.NewOptionalString("dhtclient") // Disable listening for inbound connections cfg.Addresses.Gateway = []string{} diff --git a/routing/delegated.go b/routing/delegated.go index 953cfac00e0..1ad061e6c44 100644 --- a/routing/delegated.go +++ b/routing/delegated.go @@ -157,6 +157,22 @@ type ExtraHTTPParams struct { PrivKeyB64 string } +func ConstructHTTPRouter(endpoint string, peerID string, addrs []string, privKey string) (routing.Routing, error) { + return httpRoutingFromConfig( + config.Router{ + Type: "http", + Parameters: &config.HTTPRouterParams{ + Endpoint: endpoint, + }, + }, + &ExtraHTTPParams{ + PeerID: peerID, + Addrs: addrs, + PrivKeyB64: privKey, + }, + ) +} + func httpRoutingFromConfig(conf config.Router, extraHTTP *ExtraHTTPParams) (routing.Routing, error) { params := conf.Parameters.(*config.HTTPRouterParams) if params.Endpoint == "" { diff --git a/test/sharness/t0041-ping.sh b/test/sharness/t0041-ping.sh index 8fdfe17975e..14268989d71 100755 --- a/test/sharness/t0041-ping.sh +++ b/test/sharness/t0041-ping.sh @@ -27,7 +27,7 @@ test_expect_success "test ping other" ' test_expect_success "test ping unreachable peer" ' printf "Looking up peer %s\n" "$BAD_PEER" > bad_ping_exp && - printf "PING QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJx.\nPing error: routing: not found\nError: ping failed\n" >> bad_ping_exp && + printf "PING QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJx.\nPing error: no addresses\nError: ping failed\n" >> bad_ping_exp && ! ipfsi 0 ping -n2 -- "$BAD_PEER" > bad_ping_actual 2>&1 && test_cmp bad_ping_exp bad_ping_actual ' diff --git a/test/sharness/t0170-legacy-dht.sh b/test/sharness/t0170-legacy-dht.sh index c18735b5b18..a4dffb34c6e 100755 --- a/test/sharness/t0170-legacy-dht.sh +++ b/test/sharness/t0170-legacy-dht.sh @@ -13,6 +13,10 @@ test_dht() { iptb testbed create -type localipfs -count $NUM_NODES -init ' + test_expect_success 'DHT-only routing' ' + iptb run -- ipfs config Routing.Type dht + ' + startup_cluster $NUM_NODES $@ test_expect_success 'peer ids' ' diff --git a/test/sharness/t0170-routing-dht.sh b/test/sharness/t0170-routing-dht.sh index 2eef692a8cb..2ef0a9cd1e2 100755 --- a/test/sharness/t0170-routing-dht.sh +++ b/test/sharness/t0170-routing-dht.sh @@ -2,7 +2,7 @@ # This file does the same tests as t0170-dht.sh but uses 'routing' commands instead # (only exception is query, which lives only under dht) -test_description="Test routing command" +test_description="Test routing command for DHT queries" . lib/test-lib.sh @@ -14,6 +14,10 @@ test_dht() { iptb testbed create -type localipfs -count $NUM_NODES -init ' + test_expect_success 'DHT-only routing' ' + iptb run -- ipfs config Routing.Type dht + ' + startup_cluster $NUM_NODES $@ test_expect_success 'peer ids' ' From e65885ae5ec6f09f5b4f1760c2ee1a0a63ce4a69 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 7 Dec 2022 19:22:04 +0100 Subject: [PATCH 02/10] docs: Routing.Type=auto timeouts for DHT/IPNI Based on feedback from https://github.com/ipfs/kubo/pull/9475 --- core/node/libp2p/routingopt.go | 8 ++++---- docs/changelogs/v0.18.md | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/core/node/libp2p/routingopt.go b/core/node/libp2p/routingopt.go index df553ace4f3..e28762ce4ad 100644 --- a/core/node/libp2p/routingopt.go +++ b/core/node/libp2p/routingopt.go @@ -57,8 +57,8 @@ func ConstructDefaultRouting(peerID string, addrs []string, privKey string) func } routers = append(routers, &routinghelpers.ParallelRouter{ Router: dhtRouting, - IgnoreError: false, // TODO: confirm this is what we want - Timeout: 1 * time.Minute, // TODO: what value makes sense here? + IgnoreError: false, + Timeout: 5 * time.Minute, // https://github.com/ipfs/kubo/pull/9475#discussion_r1042501333 ExecuteAfter: 0, }) @@ -70,8 +70,8 @@ func ConstructDefaultRouting(peerID string, addrs []string, privKey string) func } routers = append(routers, &routinghelpers.ParallelRouter{ Router: httpRouter, - IgnoreError: true, // TODO: confirm this is what we want - Timeout: 1 * time.Minute, // TODO: what value makes sense here? + IgnoreError: true, // https://github.com/ipfs/kubo/pull/9475#discussion_r1042507387 + Timeout: 15 * time.Second, // 5x server value from https://github.com/ipfs/kubo/pull/9475#discussion_r1042428529 ExecuteAfter: 0, }) } diff --git a/docs/changelogs/v0.18.md b/docs/changelogs/v0.18.md index 35f1489488f..147a4fb1d60 100644 --- a/docs/changelogs/v0.18.md +++ b/docs/changelogs/v0.18.md @@ -75,11 +75,13 @@ The default `Routing.Type` is changed from `dht` to `auto`. When `Routing.Type` is unset or set to `auto`, your node will accelerate some types of routing by leveraging HTTP endpoints compatible with [IPIP-337](https://github.com/ipfs/specs/pull/337) -in addition to the IPFS DHT. +in addition to the IPFS DHT. DHT response timeout in `auto` mode is 5 minutes, and IPNI is 3 seconds. By default, an instance of IPNI ([InterPlanetary Network Indexer](https://github.com/ipni/specs/blob/main/IPNI.md#readme)) at https://cid.contact is used. + Old behavior can be restored by setting `Routing.Type` to `dht`. + Alternative routing rules, including alternative IPNI endpoints, can be configured in `Routing.Routers` after setting `Routing.Type` to `custom`. Learn more in [`Routing` docs](https://github.com/ipfs/kubo/blob/master/docs/config.md#routing). From 5396526ce2bb73cafc257067b654fd998698511d Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 7 Dec 2022 20:15:36 +0100 Subject: [PATCH 03/10] chore: remove dev.cid.contact --- core/node/libp2p/routingopt.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/node/libp2p/routingopt.go b/core/node/libp2p/routingopt.go index e28762ce4ad..ed9048d3dd6 100644 --- a/core/node/libp2p/routingopt.go +++ b/core/node/libp2p/routingopt.go @@ -26,8 +26,7 @@ type RoutingOption func( // Default HTTP routers used in parallel to DHT when Routing.Type = "auto" var defaultHTTPRouters = []string{ - "https://dev.cid.contact/routing/v1", // TODO: remove when cid.contact is ready - "https://cid.contact/routing/v1", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084 + "https://cid.contact/routing/v1", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084 // TODO: add an independent router from Cloudflare } From e36db77484efc2c41565d0e75287c995e661b80c Mon Sep 17 00:00:00 2001 From: Gus Eggert Date: Wed, 7 Dec 2022 18:50:45 -0500 Subject: [PATCH 04/10] fix: remove routing API prefix from cid.contact URL --- core/node/libp2p/routingopt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/node/libp2p/routingopt.go b/core/node/libp2p/routingopt.go index ed9048d3dd6..2d2b7570c94 100644 --- a/core/node/libp2p/routingopt.go +++ b/core/node/libp2p/routingopt.go @@ -26,7 +26,7 @@ type RoutingOption func( // Default HTTP routers used in parallel to DHT when Routing.Type = "auto" var defaultHTTPRouters = []string{ - "https://cid.contact/routing/v1", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084 + "https://cid.contact", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084 // TODO: add an independent router from Cloudflare } From fbd19ea1c78ab0d599a954c52da451b796ec4ab1 Mon Sep 17 00:00:00 2001 From: Gus Eggert Date: Wed, 7 Dec 2022 19:54:56 -0500 Subject: [PATCH 05/10] fix: make changes to docs based on PR feedback --- docs/changelogs/v0.18.md | 11 ++++------- docs/config.md | 6 +----- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/docs/changelogs/v0.18.md b/docs/changelogs/v0.18.md index 147a4fb1d60..8134480b98a 100644 --- a/docs/changelogs/v0.18.md +++ b/docs/changelogs/v0.18.md @@ -69,16 +69,13 @@ $ curl "http://127.0.0.1:8080/ipfs/$DIR_CID?format=dag-json" | jq } ``` -#### Automated default `Routing.Type` +#### Content Routing -The default `Routing.Type` is changed from `dht` to `auto`. +Content routing is the process of discovering which peers provide a piece of content. Kubo has traditionally only supported the Kademlia DHT for content routing. -When `Routing.Type` is unset or set to `auto`, your node will accelerate some types of routing -by leveraging HTTP endpoints compatible with [IPIP-337](https://github.com/ipfs/specs/pull/337) -in addition to the IPFS DHT. DHT response timeout in `auto` mode is 5 minutes, and IPNI is 3 seconds. -By default, an instance of IPNI ([InterPlanetary Network Indexer](https://github.com/ipni/specs/blob/main/IPNI.md#readme)) -at https://cid.contact is used. +Kubo can now bridge networks by including support for the [delegated routing HTTP API](https://github.com/ipfs/specs/pull/337). Users can compose content routers using the `Routing.Routers` config, to pick content routers with different tradeoffs than a Kademlia DHT (for example, high-performance and high-capacity centralized endpoints, dedicated Kademlia DHT nodes, routers with unique provider records, privacy-focused content routers, etc.). One example is [InterPlanetary Network Indexers](https://github.com/ipni/specs/blob/main/IPNI.md#readme), which are HTTP endpoints that cache records from both the IPFS network and other sources such as web3.storage and Filecoin. This improves not only content availability by enabling Kubo to transparently fetch content directly from Filecoin storage providers, but also improves IPFS content routing latency by an order of magnitude and decreases resource consumption. +In this release, the default content router is changed from `dht` to `auto`. The `auto` router includes the IPFS DHT in addition to the [cid.contact](https://cid.contact) IPNI instance. In future releases, we plan to expand the functionality of `auto` to encompass automatic discovery of content routers, which will improve performance and content availability (for example, see [IPIP-342](https://github.com/ipfs/specs/pull/342)). Old behavior can be restored by setting `Routing.Type` to `dht`. diff --git a/docs/config.md b/docs/config.md index 6af7667f662..4827d888783 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1334,7 +1334,7 @@ There are multiple routing options: "auto", "none", "dht" and "custom". * If set to "dht" (or "dhtclient"/"dhtserver"), your node will ONLY use the IPFS DHT (no HTTP routers). -* If set to "custom", all default routers are disabled, and only onles defined in `Routing.Routers` will be used. +* If set to "custom", all default routers are disabled, and only ones defined in `Routing.Routers` will be used. When the DHT is enabled, it can operate in two modes: client and server. @@ -1388,7 +1388,6 @@ It specifies the routing type that will be created. Currently supported types: -- `reframe` **(DEPRECATED)** (delegated routing based on the [reframe protocol](https://github.com/ipfs/specs/tree/main/reframe#readme)) - `http` simple delegated routing based on HTTP protocol. - `dht` - `parallel` and `sequential`: Helpers that can be used to run several routers sequentially or in parallel. @@ -1401,9 +1400,6 @@ Type: `string` Parameters needed to create the specified router. Supported params per router type: -Reframe **(DEPRECATED)**: - - `Endpoint` (mandatory): URL that will be used to connect to a specified router. - HTTP: - `Endpoint` (mandatory): URL that will be used to connect to a specified router. - `MaxProvideBatchSize`: This number determines the maximum amount of CIDs sent per batch. Servers might not accept more than 100 elements per batch. 100 elements by default. From 7554df9f4598a4056b67bc33763c00e3b1ace96d Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 8 Dec 2022 17:28:17 +0100 Subject: [PATCH 06/10] docs(routing): reference IPIP-337 --- docs/changelogs/v0.18.md | 4 ++-- docs/config.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelogs/v0.18.md b/docs/changelogs/v0.18.md index 8134480b98a..aa80781d784 100644 --- a/docs/changelogs/v0.18.md +++ b/docs/changelogs/v0.18.md @@ -11,7 +11,7 @@ Below is an outline of all that is in this release, so you get a sense of all th - [Overview](#overview) - [🔦 Highlights](#-highlights) - [(DAG-)JSON and (DAG-)CBOR Response Formats on Gateways](#dag-json-and-dag-cbor-response-formats-on-gateways) - - [Automated default `Routing.Type`](#automated-default-routingtype) + - [Content Routing](#content-routing) - [Changelog](#changelog) - [Contributors](#contributors) @@ -71,7 +71,7 @@ $ curl "http://127.0.0.1:8080/ipfs/$DIR_CID?format=dag-json" | jq #### Content Routing -Content routing is the process of discovering which peers provide a piece of content. Kubo has traditionally only supported the Kademlia DHT for content routing. +Content routing is the process of discovering which peers provide a piece of content. Kubo has traditionally only supported the [libp2p's implementation of Kademlia DHT](https://github.com/libp2p/specs/tree/master/kad-dht) for content routing. Kubo can now bridge networks by including support for the [delegated routing HTTP API](https://github.com/ipfs/specs/pull/337). Users can compose content routers using the `Routing.Routers` config, to pick content routers with different tradeoffs than a Kademlia DHT (for example, high-performance and high-capacity centralized endpoints, dedicated Kademlia DHT nodes, routers with unique provider records, privacy-focused content routers, etc.). One example is [InterPlanetary Network Indexers](https://github.com/ipni/specs/blob/main/IPNI.md#readme), which are HTTP endpoints that cache records from both the IPFS network and other sources such as web3.storage and Filecoin. This improves not only content availability by enabling Kubo to transparently fetch content directly from Filecoin storage providers, but also improves IPFS content routing latency by an order of magnitude and decreases resource consumption. diff --git a/docs/config.md b/docs/config.md index 4827d888783..f8900f411a5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1388,8 +1388,8 @@ It specifies the routing type that will be created. Currently supported types: -- `http` simple delegated routing based on HTTP protocol. -- `dht` +- `http` simple delegated routing based on HTTP protocol from [IPIP-337](https://github.com/ipfs/specs/pull/337) +- `dht` provides decentralized routing based on [libp2p's kad-dht](https://github.com/libp2p/specs/tree/master/kad-dht) - `parallel` and `sequential`: Helpers that can be used to run several routers sequentially or in parallel. Type: `string` From 903fc25406b72850396862ba5061da075c5d1cfc Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 8 Dec 2022 17:55:03 +0100 Subject: [PATCH 07/10] docs(changelog): apply suggestion from review https://github.com/ipfs/kubo/pull/9475/commits/7554df9f4598a4056b67bc33763c00e3b1ace96d#r1042827850 --- docs/changelogs/v0.18.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelogs/v0.18.md b/docs/changelogs/v0.18.md index aa80781d784..7e714904923 100644 --- a/docs/changelogs/v0.18.md +++ b/docs/changelogs/v0.18.md @@ -73,7 +73,10 @@ $ curl "http://127.0.0.1:8080/ipfs/$DIR_CID?format=dag-json" | jq Content routing is the process of discovering which peers provide a piece of content. Kubo has traditionally only supported the [libp2p's implementation of Kademlia DHT](https://github.com/libp2p/specs/tree/master/kad-dht) for content routing. -Kubo can now bridge networks by including support for the [delegated routing HTTP API](https://github.com/ipfs/specs/pull/337). Users can compose content routers using the `Routing.Routers` config, to pick content routers with different tradeoffs than a Kademlia DHT (for example, high-performance and high-capacity centralized endpoints, dedicated Kademlia DHT nodes, routers with unique provider records, privacy-focused content routers, etc.). One example is [InterPlanetary Network Indexers](https://github.com/ipni/specs/blob/main/IPNI.md#readme), which are HTTP endpoints that cache records from both the IPFS network and other sources such as web3.storage and Filecoin. This improves not only content availability by enabling Kubo to transparently fetch content directly from Filecoin storage providers, but also improves IPFS content routing latency by an order of magnitude and decreases resource consumption. +Kubo can now bridge networks by including support for the [delegated routing HTTP API](https://github.com/ipfs/specs/pull/337). Users can compose content routers using the `Routing.Routers` config, to pick content routers with different tradeoffs than a Kademlia DHT (for example, high-performance and high-capacity centralized endpoints, dedicated Kademlia DHT nodes, routers with unique provider records, privacy-focused content routers, etc.). + +One example is [InterPlanetary Network Indexers](https://github.com/ipni/specs/blob/main/IPNI.md#readme), which are HTTP endpoints that cache records from both the IPFS network and other sources such as web3.storage and Filecoin. This improves not only content availability by enabling Kubo to transparently fetch content directly from Filecoin storage providers, but also improves IPFS content routing latency by an order of magnitude and decreases resource consumption. +*Note:* it's possible to retrieve content stored by Filecoin Storage Providers (SPs) from Kubo if the SPs service Bitswap requests. As of this release, some SPs are advertising Bitswap. You can follow the roadmap progress for IPNIs and Bitswap in SPs [here](https://www.starmaps.app/roadmap/github.com/protocol/bedrock/issues/1). In this release, the default content router is changed from `dht` to `auto`. The `auto` router includes the IPFS DHT in addition to the [cid.contact](https://cid.contact) IPNI instance. In future releases, we plan to expand the functionality of `auto` to encompass automatic discovery of content routers, which will improve performance and content availability (for example, see [IPIP-342](https://github.com/ipfs/specs/pull/342)). From 753b3da4b44a033b0adbfabe34f5390b463a81d3 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 8 Dec 2022 19:55:33 +0100 Subject: [PATCH 08/10] fix: switch pnet to dht routing https://github.com/ipfs/kubo/pull/9475#issuecomment-1343064066 --- cmd/ipfs/daemon.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 9ef4134140b..af105589db5 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -30,6 +30,7 @@ import ( fsrepo "github.com/ipfs/kubo/repo/fsrepo" "github.com/ipfs/kubo/repo/fsrepo/migrations" "github.com/ipfs/kubo/repo/fsrepo/migrations/ipfsfetcher" + pnet "github.com/libp2p/go-libp2p/core/pnet" sockets "github.com/libp2p/go-socket-activation" cmds "github.com/ipfs/go-ipfs-cmds" @@ -396,6 +397,16 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment routingOption = routingOptionAutoKwd } } + + // Private setups can't leverage peers returned by default IPNIs (Routing.Type=auto) + // To avoid breaking existing setups, switch them to DHT-only. + if routingOption == routingOptionAutoKwd { + if key, _ := repo.SwarmKey(); key != nil || pnet.ForcePrivateNetwork { + log.Error("Private networking (swarm.key / LIBP2P_FORCE_PNET) does not work with public HTTP IPNIs enabled by Routing.Type=auto. Kubo will use Routing.Type=dht instead. Update config to remove this message.") + routingOption = routingOptionDHTKwd + } + } + switch routingOption { case routingOptionSupernodeKwd: return errors.New("supernode routing was never fully implemented and has been removed") @@ -441,6 +452,11 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment fmt.Printf("Swarm key fingerprint: %x\n", node.PNetFingerprint) } + if (pnet.ForcePrivateNetwork || node.PNetFingerprint != nil) && routingOption == routingOptionAutoKwd { + // This should never happen, but better safe than sorry + log.Fatal("Private network does not work with Routing.Type=auto. Update your config to Routing.Type=dht (or none, and do manual peering)") + } + printSwarmAddrs(node) defer func() { From 942c4269afbbff7f7cb1768c6314a0900c3bd46d Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 8 Dec 2022 21:51:20 +0100 Subject: [PATCH 09/10] docs(changelog): apply suggestions from review Co-authored-by: Steve Loeppky --- docs/changelogs/v0.18.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelogs/v0.18.md b/docs/changelogs/v0.18.md index 7e714904923..285f00d2215 100644 --- a/docs/changelogs/v0.18.md +++ b/docs/changelogs/v0.18.md @@ -71,7 +71,7 @@ $ curl "http://127.0.0.1:8080/ipfs/$DIR_CID?format=dag-json" | jq #### Content Routing -Content routing is the process of discovering which peers provide a piece of content. Kubo has traditionally only supported the [libp2p's implementation of Kademlia DHT](https://github.com/libp2p/specs/tree/master/kad-dht) for content routing. +Content routing is the process of discovering which peers provide a piece of content. Kubo has traditionally only supported [libp2p's implementation of Kademlia DHT](https://github.com/libp2p/specs/tree/master/kad-dht) for content routing. Kubo can now bridge networks by including support for the [delegated routing HTTP API](https://github.com/ipfs/specs/pull/337). Users can compose content routers using the `Routing.Routers` config, to pick content routers with different tradeoffs than a Kademlia DHT (for example, high-performance and high-capacity centralized endpoints, dedicated Kademlia DHT nodes, routers with unique provider records, privacy-focused content routers, etc.). @@ -80,7 +80,7 @@ One example is [InterPlanetary Network Indexers](https://github.com/ipni/specs/b In this release, the default content router is changed from `dht` to `auto`. The `auto` router includes the IPFS DHT in addition to the [cid.contact](https://cid.contact) IPNI instance. In future releases, we plan to expand the functionality of `auto` to encompass automatic discovery of content routers, which will improve performance and content availability (for example, see [IPIP-342](https://github.com/ipfs/specs/pull/342)). -Old behavior can be restored by setting `Routing.Type` to `dht`. +Previous behavior can be restored by setting `Routing.Type` to `dht`. Alternative routing rules, including alternative IPNI endpoints, can be configured in `Routing.Routers` after setting `Routing.Type` to `custom`. From d273c8fc6a3aea456af4fae3e8abfe6ffaf4075a Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 8 Dec 2022 22:45:38 +0100 Subject: [PATCH 10/10] fix: use unique data for reprovider tests IPNI enabled by default made tests fail due to false-negative: Kubo was able to find provider for "foo" "bar" thanks to IPNI. This ensures data is unique enough to have no providers. --- config/profile.go | 2 +- config/reprovider.go | 5 +++++ core/node/groups.go | 8 ++++---- core/node/provider.go | 3 --- test/sharness/t0175-reprovider.sh | 14 +++++++------- test/sharness/t0175-strategic-provider.sh | 2 +- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/config/profile.go b/config/profile.go index 0349f4ba7f4..1a401391846 100644 --- a/config/profile.go +++ b/config/profile.go @@ -174,7 +174,7 @@ functionality - performance of content discovery and data fetching may be degraded. `, Transform: func(c *Config) error { - c.Routing.Type = NewOptionalString("dhtclient") // TODO: replace 'dhtclient' with 'autoclient' that prioritizes HTTP and uses dhtclient only after some delay as a fallback + c.Routing.Type = NewOptionalString("dhtclient") // TODO: https://github.com/ipfs/kubo/issues/9480 c.AutoNAT.ServiceMode = AutoNATServiceDisabled c.Reprovider.Interval = NewOptionalDuration(0) diff --git a/config/reprovider.go b/config/reprovider.go index aad3b33914c..19ee1c58d9c 100644 --- a/config/reprovider.go +++ b/config/reprovider.go @@ -1,5 +1,10 @@ package config +import "time" + +const DefaultReproviderInterval = time.Hour * 22 // https://github.com/ipfs/kubo/pull/9326 +const DefaultReproviderStrategy = "all" + type Reprovider struct { Interval *OptionalDuration `json:",omitempty"` // Time period to reprovide locally stored objects to the network Strategy *OptionalString `json:",omitempty"` // Which keys to announce diff --git a/core/node/groups.go b/core/node/groups.go index 79809263702..aa650ddf5d0 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -294,8 +294,8 @@ func Online(bcfg *BuildCfg, cfg *config.Config) fx.Option { OnlineProviders( cfg.Experimental.StrategicProviding, cfg.Experimental.AcceleratedDHTClient, - cfg.Reprovider.Strategy.WithDefault(DefaultReproviderStrategy), - cfg.Reprovider.Interval.WithDefault(DefaultReproviderInterval), + cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy), + cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval), ), ) } @@ -312,8 +312,8 @@ func Offline(cfg *config.Config) fx.Option { OfflineProviders( cfg.Experimental.StrategicProviding, cfg.Experimental.AcceleratedDHTClient, - cfg.Reprovider.Strategy.WithDefault(DefaultReproviderStrategy), - cfg.Reprovider.Interval.WithDefault(DefaultReproviderInterval), + cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy), + cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval), ), ) } diff --git a/core/node/provider.go b/core/node/provider.go index 43410573e9f..9110b41b26d 100644 --- a/core/node/provider.go +++ b/core/node/provider.go @@ -18,9 +18,6 @@ import ( irouting "github.com/ipfs/kubo/routing" ) -const DefaultReproviderInterval = time.Hour * 22 // https://github.com/ipfs/kubo/pull/9326 -const DefaultReproviderStrategy = "all" - // SIMPLE // ProviderQueue creates new datastore backed provider queue diff --git a/test/sharness/t0175-reprovider.sh b/test/sharness/t0175-reprovider.sh index 746aa6b0ab4..09535ecc4f5 100755 --- a/test/sharness/t0175-reprovider.sh +++ b/test/sharness/t0175-reprovider.sh @@ -34,7 +34,7 @@ reprovide() { init_strategy 'all' test_expect_success 'add test object' ' - HASH_0=$(echo "foo" | ipfsi 0 add -q --local) + HASH_0=$(date +"%FT%T.%N%z" | ipfsi 0 add -q --local) ' findprovs_empty '$HASH_0' @@ -49,8 +49,8 @@ test_expect_success 'Stop iptb' ' init_strategy 'pinned' test_expect_success 'prepare test files' ' - echo foo > f1 && - echo bar > f2 + date +"%FT%T.%N%z" > f1 && + date +"%FT%T.%N%z" > f2 ' test_expect_success 'add test objects' ' @@ -77,9 +77,9 @@ test_expect_success 'Stop iptb' ' init_strategy 'roots' test_expect_success 'prepare test files' ' - echo foo > f1 && - echo bar > f2 && - echo baz > f3 + date +"%FT%T.%N%z" > f1 && + date +"%FT%T.%N%z" > f2 && + date +"%FT%T.%N%z" > f3 ' test_expect_success 'add test objects' ' @@ -121,7 +121,7 @@ test_expect_success 'Disable reprovider ticking' ' startup_cluster ${NUM_NODES} test_expect_success 'add test object' ' - HASH_0=$(echo "foo" | ipfsi 0 add -q --offline) + HASH_0=$(date +"%FT%T.%N%z" | ipfsi 0 add -q --offline) ' findprovs_empty '$HASH_0' diff --git a/test/sharness/t0175-strategic-provider.sh b/test/sharness/t0175-strategic-provider.sh index 7bad38838b6..fafd6e5388c 100755 --- a/test/sharness/t0175-strategic-provider.sh +++ b/test/sharness/t0175-strategic-provider.sh @@ -22,7 +22,7 @@ test_expect_success 'use strategic providing' ' startup_cluster ${NUM_NODES} test_expect_success 'add test object' ' - HASH_0=$(echo "foo" | ipfsi 0 add -q) + HASH_0=$(date +"%FT%T.%N%z" | ipfsi 0 add -q) ' findprovs_empty '$HASH_0'