V5

.gitignore

@@ -2,3 +2,4 @@
 Cargo.lock
 reference 
 /docs/simulation/target
+examples/bootstrapping_nodes

CHANGELOG.md

@@ -2,6 +2,76 @@
 
 All notable changes to mainline dht will be documented in this file.
 
+## [Unreleased]
+
+### Added
+
+- Add `Id::from_ipv4()`.
+- Add `Id::is_valid_for_ipv4`.
+- Add `RoutingTable::nodes()` iterator.
+- Add `DhtBuilder::server_mode` to force server mode.
+- Add `DhtBuilder::public_ip` for manually setting the node's public ip to generate secure node `Id` from.
+- Add [adaptive mode](https://github.com/pubky/mainline?tab=readme-ov-file#adaptive-mode).
+- Add `DhtBuilder::extra_bootstrap()` to add more bootstrapping nodes from previous sessions.
+- Add `Dht::bootstrapped()` and `AsyncDht::bootstrapped()` to wait for the routing table to be bootstrapped.
+- Add `RoutingTable::to_bootstrap()`, `Dht::to_bootstrap()`, and `AsyncDht::to_bootstrap()` to export the addresses nodes in the routing table.
+- Add `Info::public_address()` which returns the best estimate for this node's public address.
+- Add `Info::firewalled()` which returns whether or not this node is firewalled, or publicly accessible.
+- Add `Info::server_mode()` which returns whether or not this node is running in server mode.
+- Add `DhtBuilder::info()` to export a thread safe and lightweight summary of the node's information and statistics.
+- Add `cache_bootstrap.rs` example to show how you can store your routing table to disk and use it for subsequent bootstrapping.
+- Add `Dht::get_mutable_most_recent()` and `AsyncDht::get_mutable_most_recent()` to get the most recent mutable item from the network.
+- Add `PutQueryError::Timeout` in case put query is terminated unsuccessfully, but no error responses.
+- Add `PutMutableError::Concurrrency(ConcurrrencyError)` for all cases where a `Lost Update Problem` may occur (read `Dht::put_mutable` documentation for more details).
+- Add `Dht::get_closest_nodes()` and `AsyncDht::get_closest_nodes()` to return the closest nodes (that support BEP_0044) with valid tokens.
+- Add `Dht::put()` and `AsyncDht::put()` to put a request to the closest nodes, and optionally to extra arbitrary nodes with valid tokens.
+- Add `Testnet::leak()` to keep the dht network running as a `'static`
+- Add `MutableError `.
+- Add `DecodeIdError`
+- Export `Dhtbuilder`.
+- Export `RoutingTable`.
+- Support `BEP_0042 DHT Security extension` when running in server mode. 
+
+### Removed
+
+- Remove `bytes` dependency.
+- Remove `ipv6` optionality and commit to `ipv4`.
+- Remove `Id::to_vec()`.
+- Exported `ClosestNodes`, you have to use it from `mainline::rpc`.
+- Removed `Node::unique()`, `Node::with_id()`, `Node::with_address()`, and `Node::with_token()`.
+- Removed `RoutingTable::default()`.
+- Removed exporting `rpc` module, and `Rpc` struct.
+- Removed `Dht::shutdown()` and `AsyncDht::shutdown()`.
+- Removed `DhtWasShutdown`
+
+### Changed
+
+- Rename `Settings` to `ClientBuilder`.
+- `Dht`, and `AsyncDht` is now behind a feature flag `node`, so you can include the `Rpc` only and build your own node.
+- All methods that were returning `Result<T, DhtWasShutdown>` now return `T`.
+- Enable calling `Dht::announce_peer()` and `Dht::put_immutable()` multiple times concurrently. 
+- Return `PutMutableError::Concurrrency(ConcurrrencyError)` from `Dht::put_mutable()`.
+- `Info::local_addr()` is infallible.
+- `MutableItem::seq()` returns `i64` instead of a refernece.
+- `Dht::put_immutable()` and `AsyncDh::put_immutable()` take `&[u8]` instead of `bytes::Bytes`.
+- `Dht::get_immutable()` and `AsyncDh::get_immutable()` return boxed slice `Box<[u8]>` instead of `bytes::Bytes`.
+- `Dht::put_immutable()` and `AsyncDh::put_immutable()` return `PutImmutableError`.
+- `Dht::announce_peer()` and `AsyncDh::announce_peer()` return `AnnouncePeerError`.
+- `Dht::put_mutable()` and `AsyncDh::put_mutable()` return `PutMutableError`.
+- All tracing logs are either `TRACE` (for krpcsocket), `DEBUG`, or `INFO` only for rare and singular events, 
+  like starting the node, updating the node Id, or switching to server mode (from adaptive mode).
+- Change `PutError` to contain transparent elements for generic `PutQueryError`, and more specialized `ConcurrrencyError`.
+- Remove `MutableItem::cas` field, and add optional `CAS` parameter to `Dht::put_mutable` and `AsyncDht::put_mutable`.
+- `Dht::find_node()` and `AsyncDht::find_node()` return `Box<[Node]>` instead of `Vec<Node>`.
+- `Node` is `Send` and `Sync`, and cheap to clone using an internal `Arc`.
+- `Node::new()` take `Id` and `SocketAddrV4`.
+- `RoutingTable::new()` takes an `Id`.
+- Return `GetIterator<T>` and `GetStream<T>` from `get_` methods from `Dht` and `AsyncDht` instead of exposing `flume`.
+- `Server::handle_request()` signature change, to avoid circular dependency on `Rpc`.
+- Make `DefaultServer` properties public.
+- Trait `Server` needs to implement `Clone`, but no longer needs to implement `Sync`.
+- `DhtBuilder` is not consuming, thanks to `Config` being `Clone`.
+
 ##  [4.2.0](https://github.com/pubky/mainline/compare/v4.1.0...v4.2.0) - 2024-12-13
 
 ### Added

Cargo.toml

@@ -9,39 +9,47 @@ license = "MIT"
 keywords = ["bittorrent", "torrent", "dht", "kademlia", "mainline"]
 categories = ["network-programming"]
 repository = "https://github.com/pubky/mainline"
+exclude = ["/docs/*", "/clippy.toml"]
 
 [dependencies]
 rand = "0.8.5"
 serde_bencode = "^0.2.4"
-serde = { version = "1.0.215", features = ["derive"] }
+serde = { version = "1.0.217", features = ["derive"] }
 serde_bytes = "0.11.15"
-thiserror = "2.0.3"
+thiserror = "2.0.9"
 crc = "3.2.1"
 sha1_smol = "1.0.1"
-flume = { version = "0.11.1", features = [], default-features = false }
 ed25519-dalek = "2.1.1"
-bytes = { version = "1.9.0", features = ["serde"] }
 tracing = "0.1"
 lru = { version = "0.12.5", default-features = false }
 document-features = "0.2.10"
+dyn-clone = "1.0.17"
+
+# `node` dependencies
+flume = { version = "0.11.1", features = [], default-features = false, optional = true }
+
+# `async` dependencies
+futures-lite = { version = "2.6.0", default-features = false, optional = true }
 
 [dev-dependencies]
-clap = { version = "4.5.21", features = ["derive"] }
+clap = { version = "4.5.23", features = ["derive"] }
 futures = "0.3.31"
 tracing-subscriber = "0.3"
 ctrlc = "3.4.5"
 histo = "1.0.0"
 rayon = "1.10"
 dashmap = "6.1"
+flume = "0.11.1"
 
 [features]
-## Enable [Dht::as_async()] to use [async_dht::AsyncDht]
-async = ["flume/async"]
+## Include [Dht] node.
+node = ["dep:flume"]
+## Enable [Dht::as_async()] to use [async_dht::AsyncDht].
+async = ["node", "flume/async", "dep:futures-lite"]
 
-default = []
+full = ["async"]
+
+default = ["full"]
 
 [package.metadata.docs.rs]
 all-features = true
-
-[lints.clippy]
-unwrap_used = "deny"

README.md

@@ -4,7 +4,8 @@ Simple, robust, BitTorrent's [Mainline](https://en.wikipedia.org/wiki/Mainline_D
 
 This library is focused on being the best and simplest Rust client for Mainline, especially focused on reliable and fast time-to-first-response.
 
-It should work as a routing / storing node as well, and has been running in production for many months without an issue. However if you are running your separate (read: small) DHT, or otherwise facing unusual DoS attack, you should consider implementing [rate limiting](#rate-limiting).
+It should work as a routing / storing node (server mode) as well, and has been running in production for many months without an issue. 
+However if you are concerned about spam or DoS, you should consider implementing [rate limiting](#rate-limiting).
 
 **[API Docs](https://docs.rs/mainline/latest/mainline/)**
 
@@ -21,14 +22,14 @@ Running as a client, means you can store and query for values on the DHT, but no
 ```rust
 use mainline::Dht;
 
-let dht = Dht::client(); // or `Dht::default();`
+let dht = Dht::client().unwrap();
 ```
 
 Supported BEPs:
-- [x] [BEP0005 DHT Protocol](https://www.bittorrent.org/beps/bep_0005.html)
-- [x] [BEP0042 DHT Security extension](https://www.bittorrent.org/beps/bep_0042.html)
-- [x] [BEP0043 Read-only DHT Nodes](https://www.bittorrent.org/beps/bep_0043.html)
-- [x] [BEP0044 Storing arbitrary data in the DHT](https://www.bittorrent.org/beps/bep_0044.html)
+- [x] [BEP_0005 DHT Protocol](https://www.bittorrent.org/beps/bep_0005.html)
+- [x] [BEP_0042 DHT Security extension](https://www.bittorrent.org/beps/bep_0042.html)
+- [x] [BEP_0043 Read-only DHT Nodes](https://www.bittorrent.org/beps/bep_0043.html)
+- [x] [BEP_0044 Storing arbitrary data in the DHT](https://www.bittorrent.org/beps/bep_0044.html)
 
 This implementation also includes [measures against Vertical Sybil Attacks](./docs/sybil-resistance.md).
 
@@ -39,19 +40,30 @@ Running as a server is the same as a client, but you also respond to incoming re
 ```rust
 use mainline::Dht;
 
-let dht = Dht::server(); // or `Dht::builder::server().build();` for more control.
+let dht = Dht::server().unwrap(); // or `Dht::builder::server_mode().build();` 
 ```
 
 Supported BEPs:
-- [x] [BEP0005 DHT Protocol](https://www.bittorrent.org/beps/bep_0005.html)
-- [ ] [BEP0042 DHT Security extension](https://www.bittorrent.org/beps/bep_0042.html)
-- [x] [BEP0043 Read-only DHT Nodes](https://www.bittorrent.org/beps/bep_0043.html)
-- [x] [BEP0044 Storing arbitrary data in the DHT](https://www.bittorrent.org/beps/bep_0044.html)
+- [x] [BEP_0005 DHT Protocol](https://www.bittorrent.org/beps/bep_0005.html)
+- [x] [BEP_0042 DHT Security extension](https://www.bittorrent.org/beps/bep_0042.html)
+- [x] [BEP_0043 Read-only DHT Nodes](https://www.bittorrent.org/beps/bep_0043.html)
+- [x] [BEP_0044 Storing arbitrary data in the DHT](https://www.bittorrent.org/beps/bep_0044.html)
 
 #### Rate limiting
 
-The default server implementation has no rate-limiting, you can run your own [custom server](./examples/custom_server.rs) and apply your custom rate-limiting. However, that limit/block will only apply _after_ parsing incoming messages, and it won't affect handling incoming responses.
+The default server implementation has no rate-limiting, you can run your own [custom server](./examples/custom_server.rs) and apply your custom rate-limiting. 
+However, that limit/block will only apply _after_ parsing incoming messages, and it won't affect handling incoming responses.
+
+### Adaptive mode
+
+The default Adaptive mode will start the node in client mode, and after 15 minutes of running with a publicly accessible address,
+it will switch to server mode. This way nodes that can serve as routing nodes (accessible and less likely to churn), serve as such.
+
+If you want to explicitly start in Server mode, because you know you are not running behind firewall,
+you can call `Dht::builder().server_mode().build()`, and you can optionally add your known public ip so the node doesn't have to depend on,
+votes from responding nodes: `Dht::builder().server_mode().public_ip().build()`.
 
 ## Acknowledgment
 
-This implementation was possible thanks to [Webtorrent's Bittorrent-dht](https://github.com/webtorrent/bittorrent-dht) as a reference, and [Rustydht-lib](https://github.com/raptorswing/rustydht-lib) that saved me a lot of time, especially at the serialization and deserialization of Bencode messages.
+This implementation was possible thanks to [Webtorrent's Bittorrent-dht](https://github.com/webtorrent/bittorrent-dht) as a reference, 
+and [Rustydht-lib](https://github.com/raptorswing/rustydht-lib) that saved me a lot of time, especially at the serialization and deserialization of Bencode messages.

docs/simulation/Cargo.toml

@@ -6,7 +6,7 @@ edition = "2021"
 [dependencies]
 clap = { version = "4.5.20", features = ["derive"] }
 ctrlc = "3.4.5"
-mainline = { version = "3.0.1", path = "../.." }
+mainline = { path = "../.." }
 num_cpus = "1.16.0"
 plotters = "0.3.7"
 statrs = "0.17.1"

docs/simulation/src/main.rs

@@ -5,6 +5,8 @@ use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::thread;
 
+use std::net::SocketAddrV4;
+
 use clap::Parser;
 use full_palette::GREY;
 use mainline::{ClosestNodes, Id, Node};
@@ -88,7 +90,7 @@ fn main() {
 fn build_dht(size: usize) -> Arc<BTreeMap<Id, Node>> {
     let mut dht = BTreeMap::new();
     for i in 0..size {
-        let node = Node::unique(i);
+        let node = Node::new(Id::random(), SocketAddrV4::new((i as u32).into(), i as u16));
         dht.insert(*node.id(), node);
     }
 

examples/bootstrap.rs

@@ -1,5 +1,3 @@
-use std::{thread, time::Duration};
-
 use mainline::Dht;
 
 use tracing::Level;
@@ -10,7 +8,11 @@ fn main() {
         .with_max_level(Level::DEBUG)
         .init();
 
-    Dht::client().unwrap();
+    let client = Dht::client().unwrap();
+
+    client.bootstrapped();
+
+    let info = client.info();
 
-    thread::sleep(Duration::from_secs(5));
+    println!("{:?}", info);
 }

examples/cache_bootstrap.rs

@@ -0,0 +1,53 @@
+//! Demonstrates caching and reusing bootstrapping nodes from the running
+//! node's routing table.
+//!
+//! Saves the bootstrapping nodes in `examples/bootstrapping_nodes.toml` relative to
+//! the script's directory, regardless of where the script is run from.
+
+use std::fs;
+use std::io::{Read, Write};
+use std::path::PathBuf;
+
+use mainline::Dht;
+
+use tracing::Level;
+use tracing_subscriber;
+
+fn main() {
+    tracing_subscriber::fmt()
+        .with_max_level(Level::DEBUG)
+        .init();
+
+    let mut builder = Dht::builder();
+
+    let examples_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("examples");
+    let nodes_file = examples_dir.join("bootstrapping_nodes");
+    if nodes_file.exists() {
+        let mut file =
+            fs::File::open(&nodes_file).expect("Failed to open bootstrapping nodes file");
+        let mut content = String::new();
+        file.read_to_string(&mut content)
+            .expect("Failed to read bootstrapping nodes file");
+
+        let cached_nodes = content
+            .lines()
+            .map(|line| line.to_string())
+            .collect::<Vec<_>>();
+
+        // To confirm that these old nodes are still viable,
+        // try `builder.bootstrap(&cached_nodes)` instead,
+        // this way you don't rely on default bootstrap nodes.
+        builder.extra_bootstrap(&cached_nodes);
+    };
+
+    let client = builder.build().unwrap();
+
+    client.bootstrapped();
+
+    let bootstrap = client.to_bootstrap();
+
+    let bootstrap_content = bootstrap.join("\n");
+    let mut file = fs::File::create(&nodes_file).expect("Failed to save bootstrapping nodes");
+    file.write(bootstrap_content.as_bytes())
+        .expect("Failed to write bootstrapping nodes");
+}