#12 network module (#56)

* added a checkstyle and filled it with rules

Signed-off-by: Georgi Georgiev <[email protected]>

* updated checkstyle rules and location

Signed-off-by: Georgi Georgiev <[email protected]>

* Add nabu jar

* Add initial Kademlia bootstrap logic

* Replace existing Nabu jar with Nabu jar that has dependencies built into it

* Add network module

* Add network module

* Update nabu

* Refactor

* Remove finished TOTO

* Refactor long line

* Remove unused import

* Refactor time period to constant

* Update cli logic

* Add comma separator

* Rename constant

* Remove unused variable

* Refactor network module

* Change bootnode

* Remove westend-local

* Refactor DB

* Add loading local genesis only from json

* Add javadocs

* Remove unused import

* Refactor

---------

Signed-off-by: Georgi Georgiev <[email protected]>
Co-authored-by: Georgi Georgiev <[email protected]>
Co-authored-by: Boris Velkovski <[email protected]>
Co-authored-by: vikinatora <[email protected]>

.gitignore

@@ -23,3 +23,6 @@ out/
 ### Rocks DB ###
 /rocks-db
 /test
+
+### Local Chain Spec
+/genesis/westend-local.json

README.md

@@ -5,7 +5,18 @@ Repository is under active development. Goal for first phase is to have a functi
 # Build & run
 
 1. Run smoldot light client.
-    - See (https://github.com/smol-dot/smoldot#wasm-light-node)[here] for information how to do that
+   - See (https://github.com/smol-dot/smoldot#wasm-light-node)[here] for information how to do that
 2. ```./gradlew build```
 3. ```java -jar build/libs/java-host-1.0-SNAPSHOT.jar -n {network}```
-    - `network` can be `westend`, `polkadot` or `kusama`
+   - `network` can be `westend`, `polkadot`, `kusama` or `local`
+
+### Local development
+
+1. Create a local chain specification file:
+   1. Make a copy of `genesis/westend-local-example.json` in the same folder.
+   2. Rename the copied file to `westend-local.json`
+   3. Add your local boot nodes and
+     other information if necessary.
+2. Run smoldot light client as mentioned above
+3. ```./gradlew build```
+4. ```java -jar build/libs/java-host-1.0-SNAPSHOT.jar -n local```

build.gradle.kts

@@ -40,10 +40,10 @@ dependencies {
     // CLI
     implementation("commons-cli:commons-cli:1.3.1")
 
-    // JSON-RPC dependencies
     // TODO: Publish imported packages to mvnrepository and import them
     implementation(fileTree(mapOf("dir" to "libs", "include" to listOf("*.jar"))))
 
+    // JSON-RPC dependencies
     // Needed for strange error when starting up json rpc server
     // See: https://github.com/briandilley/jsonrpc4j/issues/280
     implementation("javax.jws:javax.jws-api:1.1")
@@ -55,6 +55,7 @@ dependencies {
     implementation("org.springframework.boot:spring-boot-starter-websocket")
 
     implementation("org.java-websocket:Java-WebSocket:1.5.3")
+
 }
 
 tasks.getByName<Test>("test") {

src/main/java/com/limechain/chain/Chain.java

@@ -9,6 +9,7 @@
 public enum Chain {
     POLKADOT("polkadot"),
     KUSAMA("kusama"),
+    LOCAL("local"),
     WESTEND("westend");
 
     /**

src/main/java/com/limechain/chain/ChainService.java

@@ -38,8 +38,18 @@ public ChainService(HostConfig hostConfig, KVRepository<String, Object> reposito
     }
 
     protected void initialize(HostConfig hostConfig) {
-        Optional<Object> genesis = repository.find(this.getGenesisKey());
-        if (genesis.isPresent()) {
+        Optional<Object> genesis = repository.find(genesisKey);
+        /*
+            WORKAROUND
+            The inLocalDevelopment variable and its usage below are only to aid in development.
+            It is expected that the local genesis file will change rather frequently while the all official
+                chain specifications will never change. To improve performance we are loading the chain
+                specifications only once, and then they are saved to the database.
+            Saving the local genesis file is not suitable in this early phase of development.
+            This might be removed in the future.
+         */
+        boolean inLocalDevelopment = hostConfig.getChain() == Chain.LOCAL;
+        if (genesis.isPresent() && !inLocalDevelopment) {
             this.setGenesis((ChainSpec) genesis.get());
             log.log(Level.INFO, "✅️Loaded chain spec from DB");
             return;

src/main/java/com/limechain/cli/Cli.java

@@ -1,5 +1,6 @@
 package com.limechain.cli;
 
+import com.limechain.chain.Chain;
 import com.limechain.storage.DBInitializer;
 import lombok.Getter;
 import lombok.extern.java.Log;
@@ -11,7 +12,11 @@
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
 import java.util.logging.Level;
+import java.util.stream.Collectors;
 
 /**
  * Abstraction class around apache.commons.cli used to set arguments rules and parse node arguments
@@ -23,7 +28,7 @@ public class Cli {
      * Holds CLI options
      */
     private final Options options;
-
+    private final List<String> validChains = List.of(Chain.values()).stream().map(Chain::getValue).toList();
     private final HelpFormatter formatter = new HelpFormatter();
 
     public Cli() {
@@ -42,12 +47,13 @@ public CliArguments parseArgs(String[] args) {
             CommandLineParser parser = new DefaultParser();
             CommandLine cmd = parser.parse(options, args);
             String network = cmd.getOptionValue("network", "").toLowerCase();
+            if (!validChains.contains(network) && !network.isEmpty()) throw new ParseException("Invalid network");
             String dbPath = cmd.getOptionValue("db-path", DBInitializer.DEFAULT_DIRECTORY);
 
             return new CliArguments(network, dbPath);
         } catch (ParseException e) {
             log.log(Level.SEVERE, "Failed to parse cli arguments", e);
-            formatter.printHelp("Specify the network name - polkadot, kusama, westend", options);
+            formatter.printHelp("Specify the network name - " + String.join(", ", validChains), options);
             throw new RuntimeException();
         }
     }

src/main/java/com/limechain/config/HostConfig.java

@@ -28,13 +28,14 @@ public class HostConfig {
      * Chain the Host is running on
      */
     private Chain chain;
-
     @Value("${genesis.path.polkadot}")
     private String polkadotGenesisPath;
     @Value("${genesis.path.kusama}")
     private String kusamaGenesisPath;
     @Value("${genesis.path.westend}")
     private String westendGenesisPath;
+    @Value("${genesis.path.local}")
+    private String localGenesisPath;
     @Value("${helper.node.address}")
     private String helperNodeAddress;
 
@@ -66,7 +67,12 @@ public String getGenesisPath() {
             case WESTEND -> {
                 return westendGenesisPath;
             }
+            case LOCAL ->{
+                return localGenesisPath;
+            }
+            default -> {
+                throw new RuntimeException("Invalid Chain in host configuration");
+            }
         }
-        throw new RuntimeException("Invalid Chain in host configuration");
     }
 }

src/main/java/com/limechain/network/Network.java

@@ -0,0 +1,94 @@
+package com.limechain.network;
+
+import com.limechain.chain.Chain;
+import com.limechain.chain.ChainService;
+import com.limechain.config.HostConfig;
+import com.limechain.network.kad.KademliaService;
+import io.ipfs.multihash.Multihash;
+import io.libp2p.core.Host;
+import io.libp2p.protocol.Ping;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.java.Log;
+import org.peergos.HostBuilder;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.logging.Level;
+
+/**
+ * A Singleton Network class that handles all peer connections and Kademlia
+ */
+@Component
+@Log
+@Getter
+@Setter
+public class Network {
+    /**
+     * Interval between periodic peer searches
+     */
+    private static final int TEN_SECONDS_IN_MS = 10000;
+    private static final int HOST_PORT = 1001;
+    private static Network network;
+    public static KademliaService kademliaService;
+    private HostBuilder hostBuilder;
+    private Host host;
+
+    /**
+     * Initializes a host for the peer connection,
+     * Initializes the Kademlia service
+     * Manages if nodes running locally are going to be allowed
+     * Connects Kademlia to boot nodes
+     *
+     * @param chainService chain specification information containing boot nodes
+     * @param hostConfig host configuration containing current network
+     */
+    private Network(ChainService chainService, HostConfig hostConfig) {
+        boolean isLocalEnabled = hostConfig.getChain() == Chain.LOCAL;
+        hostBuilder = (new HostBuilder()).generateIdentity().listenLocalhost(HOST_PORT);
+        Multihash hostId = Multihash.deserialize(hostBuilder.getPeerId().getBytes());
+        kademliaService = new KademliaService("/dot/kad", hostId, isLocalEnabled);
+        hostBuilder.addProtocols(List.of(new Ping(), kademliaService.getDht()));
+
+        host = hostBuilder.build();
+        kademliaService.setHost(host);
+
+        kademliaService.connectBootNodes(chainService.getGenesis().getBootNodes());
+    }
+
+    /**
+     * @return Network class instance
+     */
+    public static Network getInstance() {
+        if (network != null) {
+            return network;
+        }
+        throw new AssertionError("Network not initialized.");
+    }
+
+    /**
+     * Initializes singleton Network instance
+     * This is used two times on startup
+     *
+     * @return Network instance saved in class or if not found returns new Network instance
+     */
+    public static Network initialize(ChainService chainService, HostConfig hostConfig) {
+        if (network != null) {
+            log.log(Level.WARNING, "Network module already initialized.");
+            return network;
+        }
+        network = new Network(chainService, hostConfig);
+        log.log(Level.INFO, "Initialized network module!");
+        return network;
+    }
+
+    /**
+     * Periodically searched for new peers
+     */
+    @Scheduled(fixedDelay = TEN_SECONDS_IN_MS)
+    public void findPeers() {
+        log.log(Level.INFO, "Searching for nodes...");
+        kademliaService.findNewPeers();
+    }
+}

src/main/java/com/limechain/network/kad/KademliaService.java

@@ -0,0 +1,71 @@
+package com.limechain.network.kad;
+
+import io.ipfs.multiaddr.MultiAddress;
+import io.ipfs.multihash.Multihash;
+import io.libp2p.core.Host;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.java.Log;
+import org.peergos.protocol.dht.Kademlia;
+import org.peergos.protocol.dht.KademliaEngine;
+import org.peergos.protocol.dht.RamProviderStore;
+import org.peergos.protocol.dht.RamRecordStore;
+
+import java.net.ConnectException;
+import java.util.List;
+import java.util.Random;
+import java.util.logging.Level;
+import java.util.stream.Collectors;
+
+/**
+ * Service used for operating the Kademlia distributed hash table.
+ */
+@Log
+@Getter
+@Setter
+public class KademliaService {
+    public static final int REPLICATION = 20;
+    public static final int ALPHA = 3;
+    private Kademlia dht;
+    private Host host;
+
+    public KademliaService(String protocolId, Multihash hostId, boolean localDht) {
+        this.initialize(protocolId, hostId, localDht);
+    }
+
+    /**
+     * Initializes Kademlia dht with replication=20 and alpha=3
+     *
+     * @param protocolId
+     * @param hostId
+     * @param localEnabled
+     * @return Kademlia dht
+     */
+    private void initialize(String protocolId, Multihash hostId, boolean localEnabled) {
+        dht = new Kademlia(new KademliaEngine(hostId, new RamProviderStore(), new RamRecordStore()),
+                protocolId, REPLICATION, ALPHA, localEnabled);
+    }
+
+    /**
+     * Connects to boot nodes to the Kademlia dht
+     *
+     * @param bootNodes boot nodes set in ChainService
+     */
+    public void connectBootNodes(String[] bootNodes) {
+        var bootstrapMultiAddress = List.of(bootNodes).stream()
+                .map(MultiAddress::new)
+                .collect(Collectors.toList());
+        int successfulBootNodes = dht.bootstrapRoutingTable(host, bootstrapMultiAddress, addr -> !addr.contains("wss"));
+        log.log(Level.INFO, "Successfully connected to " + successfulBootNodes + " boot nodes");
+    }
+
+    /**
+     * Populates Kademlia dht with peers closest in distance to a random id
+     */
+    public void findNewPeers() {
+        byte[] hash = new byte[32];
+        (new Random()).nextBytes(hash);
+        Multihash randomPeerId = new Multihash(Multihash.Type.sha2_256, hash);
+        dht.findClosestPeers(randomPeerId, REPLICATION, host);
+    }
+}

src/main/java/com/limechain/rpc/config/CommonConfig.java

@@ -6,6 +6,7 @@
 import com.limechain.cli.CliArguments;
 import com.limechain.config.HostConfig;
 import com.limechain.config.SystemInfo;
+import com.limechain.network.Network;
 import com.limechain.storage.DBInitializer;
 import com.limechain.storage.KVRepository;
 import org.springframework.boot.ApplicationArguments;
@@ -37,7 +38,7 @@ public HostConfig hostConfig(ApplicationArguments arguments) {
 
     @Bean
     public KVRepository<String, Object> repository(HostConfig hostConfig) {
-        return DBInitializer.initialize(hostConfig.getRocksDbPath());
+        return DBInitializer.initialize(hostConfig.getRocksDbPath(), hostConfig.getChain());
     }
 
     @Bean
@@ -50,4 +51,9 @@ public SystemInfo systemInfo(HostConfig hostConfig) {
         return new SystemInfo();
     }
 
+    @Bean
+    public Network network(ChainService chainService, HostConfig hostConfig) {
+        return Network.initialize(chainService, hostConfig);
+    }
+
 }

src/main/java/com/limechain/rpc/http/server/HttpRpc.java

@@ -6,6 +6,7 @@
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.annotation.ComponentScan;
+import org.springframework.scheduling.annotation.EnableScheduling;
 
 import java.util.Collections;
 
@@ -23,6 +24,7 @@
         "com.limechain.rpc.http.server",
         "com.limechain.storage"
 })
+@EnableScheduling
 public class HttpRpc {
 
     /**

src/main/java/com/limechain/storage/DBInitializer.java

@@ -1,5 +1,6 @@
 package com.limechain.storage;
 
+import com.limechain.chain.Chain;
 import lombok.extern.java.Log;
 
 import java.util.HashMap;
@@ -27,14 +28,15 @@ public class DBInitializer {
      * Initializes the connection if it doesn't exist and returns it
      *
      * @param path path where the DB should write to
+     * @param chain current network used for prefix
      * @return connection to the DB
      */
-    public static DBRepository initialize(String path) {
+    public static DBRepository initialize(String path, Chain chain) {
         if (instances.containsKey(path)) {
             return instances.get(path);
         }
 
-        DBRepository repo = new DBRepository(path);
+        DBRepository repo = new DBRepository(path, chain.getValue());
 
         instances.put(path, repo);
         return repo;