Review full codebase

.github/workflows/rust.yaml

@@ -0,0 +1,34 @@
+name: Rust CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        profile: minimal
+        toolchain: stable
+        components: rustfmt, clippy
+        override: true
+
+    - name: Check formatting
+      run: cargo fmt -- --check
+
+    - name: Lint with Clippy
+      run: cargo clippy -- -D warnings
+
+    - name: Build
+      run: cargo build --verbose
+
+    - name: Run tests
+      run: cargo test --verbose

.gitignore

@@ -1,2 +1,3 @@
 /target
 Cargo.lock
+reference 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mainline"
-version = "0.1.0"
+version = "0.6.0"
 authors = ["nuh.dev"]
 edition = "2018"
 description = "Simple, robust, BitTorrent's Mainline DHT implementation"
@@ -10,4 +10,22 @@ keywords = ["bittorrent", "torrent", "dht", "kademlia", "mainline"]
 categories = ["network-programming"]
 repository = "https://github.com/nuhvi/mainline"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[dependencies]
+rand = "0.8.5"
+serde_bencode = "^0.2.4"
+serde = { version = "1.0.133", features = ["derive"] }
+serde_bytes = "0.11.5"
+thiserror = "1.0.49"
+crc = "3.0.1"
+sha1_smol = "1.0.0"
+flume = { version = "0.11.0", features = ["select", "eventual-fairness"], default-features = false }
+ed25519-dalek = "2.1.0"
+bytes = "1.5.0"
+
+[dev-dependencies]
+clap = { version = "4.4.8", features = ["derive"] }
+futures = "0.3.29"
+
+[features]
+async = ["flume/async"]
+default = ["async"]

README.md

@@ -1,3 +1,50 @@
 # Mainline
 
-WIP mainline rust implementation.
+Simple, robust, BitTorrent's [Mainline](https://en.wikipedia.org/wiki/Mainline_DHT) DHT implementation.
+
+This library is focused on being the best and simplest Rust client for Mainline, especially focused on reliable and fast time-to-first-response.
+
+It should work as a routing / storing node as well, but if you want to run a reliable node to support the network, you might be better off running [libtorrent](https://libtorrent.org/) for now.
+
+## Get started
+
+Check the [Examples](./examples).
+
+## Features
+
+### Client
+
+Running as a client, means you can store and query for values on the DHT, but not accept any incoming requests.
+
+```rust
+use mainline::Dht;
+
+let dht = Dht::client(); // or Dht::default();
+```
+
+Supported BEPs:
+- [x] [BEP0005 DHT Protocol](https://www.bittorrent.org/beps/bep_0005.html)
+- [X] [BEP0042 DHT Security extension](https://www.bittorrent.org/beps/bep_0042.html)
+- [x] [BEP0043 Read-only DHT Nodes](https://www.bittorrent.org/beps/bep_0043.html)
+- [x] [BEP0044 Storing arbitrary data in the DHT](https://www.bittorrent.org/beps/bep_0044.html)
+
+### Server
+
+Running as a server is the same as a client, but you also respond to incoming requests and serve as a routing and storing node, supporting the general routing of the DHT, and contributing to the storage capacity of the DHT.
+
+```rust
+use mainline::Dht;
+
+let dht = Dht::server(); // or Dht::builder::as_server().build();
+```
+
+Supported BEPs:
+- [x] [BEP0005 DHT Protocol](https://www.bittorrent.org/beps/bep_0005.html)
+- [ ] [BEP0042 DHT Security extension](https://www.bittorrent.org/beps/bep_0042.html)
+- [x] [BEP0043 Read-only DHT Nodes](https://www.bittorrent.org/beps/bep_0043.html)
+- [ ] [BEP0044 Storing arbitrary data in the DHT](https://www.bittorrent.org/beps/bep_0044.html)
+
+
+## Acknowledgment
+
+This implementation was possible thanks to [Webtorrent's Bittorrent-dht](https://github.com/webtorrent/bittorrent-dht) as a reference, and [Rustydht-lib](https://github.com/raptorswing/rustydht-lib) that saved me a lot of time, especially at the serialization and deserialization of Bencode messages.

examples/README.md

@@ -0,0 +1,37 @@
+# Examples
+
+## Announce peer
+
+```sh
+cargo run --example announce_peer <40 bytes hex info_hash>
+```
+
+## Get peers
+
+```sh
+cargo run --example get_peers <40 bytes hex info_hash>
+```
+
+## Put Immutable
+
+```sh
+cargo run --example put_immutable <string>
+```
+
+## Get Immutable
+
+```sh
+cargo run --example get_immutable <40 bytes hex target from put_immutable>
+```
+
+## Put Mutable
+
+```sh
+cargo run --example put_mutable <64 bytes hex secret_key> <string>
+```
+
+## Get Mutable
+
+```sh
+cargo run --example get_mutable <40 bytes hex target from put_mutable>
+`````

examples/announce_peer.rs

@@ -0,0 +1,70 @@
+use std::{str::FromStr, time::Instant};
+
+use mainline::{Dht, Id};
+
+use clap::Parser;
+
+#[derive(Parser)]
+#[command(author, version, about, long_about = None)]
+struct Cli {
+    /// info_hash to annouce a peer on
+    infohash: String,
+}
+
+fn main() {
+    let cli = Cli::parse();
+
+    match Id::from_str(cli.infohash.as_str()) {
+        Ok(infohash) => {
+            let dht = Dht::default();
+
+            let start = Instant::now();
+
+            println!("\nAnnouncing peer on an infohash: {} ...\n", cli.infohash);
+
+            let metadata = dht
+                .announce_peer(infohash, Some(6991))
+                .expect("announce_peer fialed");
+            println!(
+                "Announced peer in {:?} seconds",
+                start.elapsed().as_secs_f32()
+            );
+            let stored_at = metadata.stored_at();
+            println!("Stored at: {:?} nodes", stored_at.len());
+            for node in stored_at {
+                println!("   {:?}", node);
+            }
+
+            // You can now reannounce to the same closest nodes
+            // skipping the the lookup step.
+            //
+            // This time we choose to not sepcify the port, effectively
+            // making the port implicit to be detected by the storing node
+            // from the source address of the announce_peer request
+            //
+            // Uncomment the following lines to try it out:
+
+            // println!(
+            //     "Announcing again to {:?} closest_nodes ...",
+            //     metadata.closest_nodes().len()
+            // );
+            //
+            // let again = Instant::now();
+            // match dht.announce_peer_to(infohash, metadata.closest_nodes(), None) {
+            //     Ok(metadata) => {
+            //         println!(
+            //             "Announced again to {:?} nodes in {:?} seconds",
+            //             metadata.stored_at().len(),
+            //             again.elapsed().as_secs()
+            //         );
+            //     }
+            //     Err(err) => {
+            //         println!("Error: {:?}", err);
+            //     }
+            // }
+        }
+        Err(err) => {
+            println!("Error: {:?}", err)
+        }
+    };
+}

examples/get_immutable.rs

@@ -0,0 +1,63 @@
+use std::{str::FromStr, time::Instant};
+
+use mainline::{Dht, Id};
+
+use clap::Parser;
+
+#[derive(Parser)]
+#[command(author, version, about, long_about = None)]
+struct Cli {
+    /// Immutable data sha1 hash to lookup.
+    target: String,
+}
+
+fn main() {
+    let cli = Cli::parse();
+
+    let target_parse_result: Result<Id, _> = Id::from_str(cli.target.as_str());
+
+    match target_parse_result {
+        Ok(infohash) => {
+            let dht = Dht::default();
+
+            let start = Instant::now();
+
+            println!("\nLooking up immutable data: {} ...\n", cli.target);
+
+            let mut response = &mut dht.get_immutable(infohash);
+
+            if let Some(res) = response.next() {
+                println!(
+                    "Got result in {:?} seconds\n",
+                    start.elapsed().as_secs_f32()
+                );
+
+                // No need to stream responses, just print the first result, since
+                // all immutable data items are guaranteed to be the same.
+
+                match String::from_utf8(res.value.to_vec()) {
+                    Ok(string) => {
+                        println!("Got immutable data: {:?} | from: {:?}", string, res.from);
+                    }
+                    Err(_) => {
+                        println!("Got immutable data: {:?} | from: {:?}", res.value, res.from);
+                    }
+                };
+            }
+
+            println!(
+                "\nVisited {:?} nodes, found {:?} closest nodes",
+                response.visited,
+                &response.closest_nodes.len()
+            );
+
+            println!(
+                "\nQuery exhausted in {:?} seconds",
+                start.elapsed().as_secs_f32(),
+            );
+        }
+        Err(err) => {
+            println!("Error: {}", err)
+        }
+    };
+}

examples/get_mutable.rs

@@ -0,0 +1,92 @@
+use ed25519_dalek::VerifyingKey;
+use std::convert::TryFrom;
+
+use std::time::Instant;
+
+use mainline::Dht;
+
+use clap::Parser;
+
+#[derive(Parser)]
+#[command(author, version, about, long_about = None)]
+struct Cli {
+    /// Mutable data public key.
+    public_key: String,
+}
+
+fn main() {
+    let cli = Cli::parse();
+
+    let public_key = from_hex(cli.public_key.clone());
+    let dht = Dht::default();
+
+    let start = Instant::now();
+    let mut first = false;
+
+    println!("\nLooking up mutable item: {} ...\n", cli.public_key);
+
+    let mut count = 0;
+
+    let mut response = &mut dht.get_mutable(public_key.as_bytes(), None);
+
+    for res in &mut response {
+        if !first {
+            first = true;
+            println!(
+                "Got first result in {:?} seconds\n",
+                start.elapsed().as_secs_f32()
+            );
+
+            println!("Streaming mutable items:\n");
+        }
+
+        count += 1;
+
+        match String::from_utf8(res.item.value().to_vec()) {
+            Ok(string) => {
+                println!(
+                    "Got mutable item: {:?}, seq: {:?} | from: {:?}",
+                    string,
+                    res.item.seq(),
+                    res.from
+                );
+            }
+            Err(_) => {
+                println!(
+                    "Got mutable item: {:?}, seq: {:?} | from: {:?}",
+                    res.item.value(),
+                    res.item.seq(),
+                    res.from
+                );
+            }
+        };
+    }
+
+    println!(
+        "Visited {:?} nodes, found {:?} closest nodes",
+        response.visited,
+        &response.closest_nodes.len()
+    );
+
+    println!(
+        "\nQuery exhausted in {:?} seconds, got {:?} peers.",
+        start.elapsed().as_secs_f32(),
+        count
+    );
+}
+
+fn from_hex(s: String) -> VerifyingKey {
+    if s.len() % 2 != 0 {
+        panic!("Number of Hex characters should be even");
+    }
+
+    let mut bytes = Vec::with_capacity(s.len() / 2);
+
+    for i in 0..s.len() / 2 {
+        let byte_str = &s[i * 2..(i * 2) + 2];
+        let byte = u8::from_str_radix(byte_str, 16).expect("Invalid hex character");
+        bytes.push(byte);
+    }
+
+    VerifyingKey::try_from(bytes.as_slice()).expect("Invalid mutable key")
+}