lndmanaged: implement lndmanage daemon

A daemon is implemented that is capable of running several service in
an asynchronous way. This commit adds async grpc channels to the node
interface. As a first application we implement a channel acceptor and
watching of failed/succeeded HTLCs. Adds distinguished logging.

examples/example_network_info.py

@@ -3,7 +3,7 @@
 from lndmanage import settings
 
 import logging.config
-logging.config.dictConfig(settings.logger_config)
+logging.config.dictConfig(settings.lndm_logger_config)
 logger = logging.getLogger(__name__)
 
 if __name__ == '__main__':

examples/example_node_info.py

@@ -3,7 +3,7 @@
 from lndmanage import settings
 
 import logging.config
-logging.config.dictConfig(settings.logger_config)
+logging.config.dictConfig(settings.lndm_logger_config)
 
 if __name__ == '__main__':
     node = LndNode()

lndmanage/lib/chan_acceptor.py

@@ -0,0 +1,104 @@
+"""Implements logic for accepting channels dynamically."""
+import asyncio
+from typing import TYPE_CHECKING
+
+from lndmanage.grpc_compiled import rpc_pb2 as lnd
+from lndmanage.lib.node import LndNode
+from lndmanage import settings
+from lndmanage.lib.network_info import NetworkAnalysis
+
+import logging
+logger = logging.getLogger('CHANAC')
+logger.setLevel(logging.DEBUG)
+
+if TYPE_CHECKING:
+    from configparser import ConfigParser
+
+
+class ChanAcceptor(object):
+    min_size_private: int
+    max_size_private: int
+    min_size_public: int
+    max_size_public: int
+
+    def __init__(self, node: LndNode, config: 'ConfigParser'):
+        self.node = node
+        self.config = config
+        self.network_analysis = NetworkAnalysis(self.node)
+        self.configure()
+
+    def configure(self):
+        # configuration is designed to also accept an empty configuration,
+        # in which case we take default values as fallback
+        self.min_size_private = int(
+            self.config.get(
+                'channel_acceptor',
+                'min_channel_size_private',
+                fallback=settings.CHACC_MIN_CHANNEL_SIZE_PRIVATE
+            )
+        )
+        self.max_size_private = int(
+            self.config.get(
+                'channel_acceptor',
+                'max_channel_size_private',
+                fallback=settings.CHACC_MAX_CHANNEL_SIZE_PRIVATE
+            )
+        )
+        self.min_size_public = int(
+            self.config.get(
+                'channel_acceptor',
+                'min_channel_size_public',
+                fallback=settings.CHACC_MIN_CHANNEL_SIZE_PUBLIC
+            )
+        )
+        self.max_size_public = int(
+            self.config.get(
+                'channel_acceptor',
+                'max_channel_size_public',
+                fallback=settings.CHACC_MAX_CHANNEL_SIZE_PUBLIC
+            )
+        )
+
+    async def manage_channel_openings(self):
+        response_queue = asyncio.queues.Queue()
+        try:
+            # async way to use a bidirectional streaming grpc endpoint
+            # with an async iterator
+            async for r in self.node.async_rpc.ChannelAcceptor(
+                    self.request_iterator(response_queue)):
+                await response_queue.put(r)
+        except asyncio.CancelledError:
+            logger.info("channel acceptor cancelled")
+            return
+
+    async def request_iterator(self, channel_details: asyncio.Queue):
+        logger.info("channel acceptor started")
+        while True:
+            channel_detail = await channel_details.get()
+            if self.accept_channel(channel_detail):
+                yield lnd.ChannelAcceptResponse(
+                    accept=True, pending_chan_id=channel_detail.pending_chan_id
+                )
+            else:
+                yield lnd.ChannelAcceptResponse(
+                    accept=False, pending_chan_id=channel_detail.pending_chan_id
+                )
+
+    def accept_channel(self, channel_detail) -> bool:
+        # be careful, exceptions from here seem to not get raised up to the main
+        # loop
+        # TODO: raise exceptions from here
+        logger.info(
+            f"about to make a decision about channel:\n{channel_detail}")
+        node_pubkey = channel_detail.node_pubkey.hex()
+        is_private = self.network_analysis.is_private(node_pubkey)
+        logger.info(f"is private {is_private}")
+        if is_private:
+            if (self.min_size_private < channel_detail.funding_amt
+                    < self.max_size_private):
+                return True
+        else:
+            if (self.min_size_public < channel_detail.funding_amt
+                    < self.max_size_public):
+                return True
+        return False

lndmanage/lib/configure.py

@@ -1,16 +1,14 @@
 import os
 import configparser
 
-from lndmanage.lib.user import yes_no_question, get_user_input
-
 import logging
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('CONFIG')
 logger.addHandler(logging.NullHandler())
 
 this_file_path = os.path.dirname(os.path.realpath(__file__))
 
 
-def valid_path(path):
+def valid_path(path: str):
     path = os.path.expanduser(path)
     if os.path.exists(path):
         return path
@@ -19,16 +17,29 @@ def valid_path(path):
         return False
 
 
-def valid_host(host):
+def valid_host(host: str):
     try:
-        ip, port = host.split(':')
+        _, _ = host.split(':')
     except ValueError:
         print("Error: Host is not of format '127.0.0.1:10009'")
         return False
     return host
 
 
-def check_or_create_configuration(home_dir):
+def create_lndmanaged_config(home_dir: str):
+    config = configparser.ConfigParser()
+    config_template_path = os.path.join(
+        this_file_path, '../templates/lndmanaged_sample.ini')
+    config.read(config_template_path)
+
+    lndmanaged_config_path = os.path.join(home_dir, 'lndmanaged.ini')
+
+    with open(lndmanaged_config_path, 'w') as configfile:
+        config.write(configfile)
+    print(f'Lndmanage daemon config was written to {lndmanaged_config_path}.')
+
+
+def check_or_create_configuration(home_dir: str):
     """
     Checks if lndmanage configuration exists, otherwise creates configuration.
 
@@ -54,9 +65,10 @@ def check_or_create_configuration(home_dir):
             print("Will use admin.macaroon and tls.cert from this directory.")
         else:
             remote = True
-            print(f"IF LND RUNS ON A REMOTE HOST, CONFIGURE {home_dir}/config.ini.")
+            print(f"IF LND RUNS ON A REMOTE HOST, CONFIGURE "
+                  f"{home_dir}/config.ini.")
 
-        # build config file
+        # create lndmanage config file
         config = configparser.ConfigParser()
         os.mkdir(home_dir)
         config_template_path = os.path.join(
@@ -67,17 +79,28 @@ def check_or_create_configuration(home_dir):
         config['network']['admin_macaroon_file'] = str(admin_macaroon_path)
         config['network']['tls_cert_file'] = str(tls_cert_path)
 
-        config_path = os.path.join(home_dir, 'config.ini')
+        lndmanage_config_path = os.path.join(home_dir, 'config.ini')
 
-        with open(config_path, 'w') as configfile:
+        with open(lndmanage_config_path, 'w') as configfile:
             config.write(configfile)
-        print(f'Config file was written to {config_path}.')
+        print(f'Config file was written to {lndmanage_config_path}.')
+
+        # lndmanaged config
+        create_lndmanaged_config(home_dir)
+
+        # leave the interactive mode for the user to update configuration
         if remote:
             exit(0)
 
     else:
-        config_path = os.path.join(home_dir, 'config.ini')
-        if not os.path.isfile(config_path):
+        lndmanage_config_path = os.path.join(home_dir, 'config.ini')
+        if not os.path.isfile(lndmanage_config_path):
             raise FileNotFoundError(
-                f"Configuration file does not exist. Filename: {config_path}. "
-                f"Delete .lndmanage folder and run again.")
+                f"Configuration file does not exist. Filename: "
+                f"{lndmanage_config_path}. Delete .lndmanage folder and "
+                f"run again.")
+
+        lndmanaged_config_path = os.path.join(home_dir, 'lndmanaged.ini')
+        if not os.path.isfile(lndmanaged_config_path):
+            # lndmanaged config
+            create_lndmanaged_config(home_dir)

lndmanage/lib/fee_setting.py

@@ -1,7 +1,7 @@
 import lndmanage.grpc_compiled.rpc_pb2 as ln
 
 import logging
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('FEESET')
 logger.addHandler(logging.NullHandler())
 
 CLTV = 40
@@ -52,19 +52,3 @@ def set_fees_by_balancedness(
             time_lock_delta=CLTV,
         )
         logger.debug(update_request)
-        # node._stub.UpdateChannelPolicy(request=update_request)
-
-
-if __name__ == '__main__':
-    from lndmanage.lib.node import LndNode
-    import logging.config
-    from lndmanage import settings
-
-    logging.config.dictConfig(settings.logger_config)
-
-    nd = LndNode()
-
-    set_fees_by_balancedness(
-        nd, base_unbalanced_msat=0, rate_unbalanced_decimal=0.000001,
-        base_balanced_msat=40, rate_balanced_decimal=0.000050)
-

lndmanage/lib/forwardings.py

@@ -3,10 +3,9 @@
 
 import numpy as np
 
-from lndmanage.lib.node import LndNode
 from lndmanage import settings
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('FWDING')
 logger.addHandler(logging.NullHandler())
 
 np.warnings.filterwarnings('ignore')
@@ -614,15 +613,3 @@ def get_forwarding_statistics_channels(node, time_interval_start,
             if c['bandwidth_demand'] > 0.5:
                 c['action_required'] = True
     return channels
-
-
-if __name__ == '__main__':
-    import time
-    import logging.config
-    logging.config.dictConfig(settings.logger_config)
-    logger = logging.getLogger()
-
-    nd = LndNode()
-    fa = ForwardingAnalyzer(nd)
-    fa.initialize_forwarding_data(time_start=0, time_end=time.time())
-    print(fa.simple_flow_analysis())

lndmanage/lib/info.py

@@ -3,10 +3,9 @@
 
 from lndmanage.lib.network_info import NetworkAnalysis
 from lndmanage.lib import ln_utilities
-from lndmanage import settings
 
 import logging
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('INFORM')
 logger.addHandler(logging.NullHandler())
 
 # width of a column in the ouptput in characters

lndmanage/lib/listchannels.py

@@ -3,14 +3,14 @@
 """
 
 import math
-import logging
 import time
 from collections import OrderedDict
 
 from lndmanage.lib.forwardings import get_forwarding_statistics_channels
 from lndmanage import settings
 
-logger = logging.getLogger(__name__)
+import logging
+logger = logging.getLogger('LSTCHN')
 logger.addHandler(logging.NullHandler())
 
 # define symbols for bool to string conversion

lndmanage/lib/lncli.py

@@ -9,7 +9,7 @@
 from lndmanage import settings
 
 import logging
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('LNCLI')
 logger.addHandler(logging.NullHandler())
 
 

lndmanage/lib/network.py

@@ -8,7 +8,7 @@
 from lndmanage import settings
 
 import logging
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('NETWRK')
 logger.addHandler(logging.NullHandler())
 
 
@@ -247,13 +247,3 @@ def nodes_in_neighborhood_of_nodes(self, nodes, blacklist_nodes, nnodes=100):
 
         sorted_neighboring_nodes = sorted(neighboring_nodes, key=lambda x: x[1], reverse=True)
         return sorted_neighboring_nodes[:nnodes]
-
-
-if __name__ == '__main__':
-    import logging.config
-    logging.config.dictConfig(settings.logger_config)
-
-    from lndmanage.lib.node import LndNode
-    nd = LndNode()
-    print(f"Graph size: {nd.network.graph.size()}")
-    print(f"Number of channels: {len(nd.network.edges.keys())}")