Merge pull request #86 from TogetherCrew/feat/use-neo4jops-singleton

Feat/use neo4jops singleton

analyzer_init.py

@@ -1,7 +1,7 @@
 from typing import Any
 
 from discord_analyzer import RnDaoAnalyzer
-from utils.credentials import get_mongo_credentials, get_neo4j_credentials
+from utils.credentials import get_mongo_credentials
 
 
 class AnalyzerInit:
@@ -23,17 +23,14 @@ def get_analyzer(self) -> RnDaoAnalyzer:
 
         # credentials
         mongo_creds = get_mongo_credentials()
-        neo4j_creds = get_neo4j_credentials()
 
         analyzer.set_mongo_database_info(
             mongo_db_host=mongo_creds["host"],
             mongo_db_password=mongo_creds["password"],
             mongo_db_port=mongo_creds["port"],
             mongo_db_user=mongo_creds["user"],
         )
-        analyzer.set_neo4j_database_info(neo4j_creds=neo4j_creds)
         analyzer.database_connect()
-        analyzer.setup_neo4j_metrics()
 
         return analyzer
 

discord_analyzer/DB_operations/mongo_neo4j_ops.py

@@ -2,7 +2,7 @@
 
 from discord_analyzer.DB_operations.mongodb_interaction import MongoDBOps
 from discord_analyzer.DB_operations.network_graph import make_neo4j_networkx_query_dict
-from tc_neo4j_lib.neo4j_ops import Neo4jOps
+from tc_neo4j_lib.neo4j_ops import Neo4jOps, Query
 
 
 class MongoNeo4jDB:
@@ -11,34 +11,10 @@ def __init__(self, testing=False):
         having both databases in one class
 
         """
-        self.neo4j_ops = None
+        self.neo4j_ops = Neo4jOps.get_instance()
         self.mongoOps = None
         self.testing = testing
 
-    def set_neo4j_utils(
-        self,
-        db_name: str,
-        host: str,
-        port: str,
-        protocol: str,
-        user: str,
-        password: str,
-    ):
-        """
-        store the neo4j utils instance
-        """
-        self.neo4j_ops = Neo4jOps()
-        self.neo4j_ops.set_neo4j_db_info(
-            neo4j_db_name=db_name,
-            neo4j_protocol=protocol,
-            neo4j_user=user,
-            neo4j_password=password,
-            neo4j_host=host,
-            neo4j_port=port,
-        )
-        self.neo4j_ops.neo4j_database_connect()
-        logging.info("Neo4j Connected Successfully!")
-
     def set_mongo_db_ops(
         self, mongo_user: str, mongo_pass: str, mongo_host: str, mongo_port: str
     ):
@@ -115,6 +91,7 @@ def store_analytics_data(
                     guildId=guild_id,
                     community_id=community_id,
                 )
+                print(queries_list[0])
                 self.run_operations_transaction(
                     guildId=guild_id,
                     queries_list=queries_list,
@@ -124,8 +101,8 @@ def store_analytics_data(
             logging.warning("Testing mode enabled! Not saving any data")
 
     def run_operations_transaction(
-        self, guildId, queries_list, remove_memberactivities
-    ):
+        self, guildId: str, queries_list: list[Query], remove_memberactivities: bool
+    ) -> None:
         """
         do the deletion and insertion operations inside a transaction
 
@@ -142,7 +119,7 @@ def run_operations_transaction(
         """
         self.guild_msg = f"GUILDID: {guildId}:"
 
-        transaction_queries = []
+        transaction_queries: list[Query] = []
         if remove_memberactivities:
             logging.info(
                 f"{self.guild_msg} Neo4J GuildId accounts relation will be removed!"
@@ -152,14 +129,13 @@ def run_operations_transaction(
             )
             transaction_queries.append(delete_relationship_query)
 
-        # logging.info(queries_list)
         transaction_queries.extend(queries_list)
 
-        self.neo4j_ops.store_data_neo4j(transaction_queries, message=self.guild_msg)
+        self.neo4j_ops.run_queries_in_batch(transaction_queries, message=self.guild_msg)
 
     def _create_guild_rel_deletion_query(
         self, guildId: str, relation_name: str = "INTERACTED_WITH"
-    ):
+    ) -> Query:
         """
         create a query to delete the relationships
         between DiscordAccount users in a specific guild
@@ -176,11 +152,19 @@ def _create_guild_rel_deletion_query(
         final_query : str
             the final query to remove the relationships
         """
-
-        delete_relationship_query = f"""
+        query_str = f"""
           MATCH
             (:DiscordAccount)
                 -[r:{relation_name} {{guildId: '{guildId}'}}]-(:DiscordAccount)
             DETACH DELETE r"""
 
-        return delete_relationship_query
+        parameters = {
+            "relation_name": relation_name,
+            "guild_id": guildId,
+        }
+
+        query = Query(
+            query=query_str,
+            parameters=parameters,
+        )
+        return query

discord_analyzer/DB_operations/network_graph.py

@@ -3,13 +3,14 @@
 import datetime
 
 import networkx
+from tc_neo4j_lib import Query
 
 
 def make_neo4j_networkx_query_dict(
     networkx_graphs: dict[datetime.datetime, networkx.classes.graph.Graph],
     guildId: str,
     community_id: str,
-):
+) -> list[Query]:
     """
     make a list of queries to store networkx graphs into the neo4j
 
@@ -26,7 +27,7 @@ def make_neo4j_networkx_query_dict(
 
     Returns:
     -----------
-    queries_list : list
+    queries_list : list[Query]
         list of string queries to store data into neo4j
     """
     # extract the graphs and their corresponding interaction dates
@@ -53,7 +54,7 @@ def make_graph_list_query(
     guildId: str,
     community_id: str,
     toGuildRelation: str = "IS_MEMBER",
-):
+) -> list[Query]:
     """
     Make a list of queries for each graph to save their results
 
@@ -75,10 +76,10 @@ def make_graph_list_query(
 
     Returns:
     ---------
-    final_queries : list of str
+    final_queries : list[Query]
         list of strings, each is a query for an interaction graph to be created
     """
-    final_queries = []
+    final_queries: list[Query] = []
 
     for graph, date in zip(networkx_graphs, networkx_dates):
         nodes_dict = graph.nodes.data()
@@ -104,7 +105,7 @@ def create_community_node_query(
     community_id: str,
     guild_id: str,
     community_node: str = "Community",
-) -> str:
+) -> Query:
     """
     create the community node
 
@@ -114,20 +115,32 @@ def create_community_node_query(
         the community id to create its node
     guild_id : str
         the guild node to attach to community
+
+    Returns
+    ---------
+    query : Query
+        the query to run on neo4j to create community node
     """
     date_now_timestamp = get_timestamp()
 
-    query = f"""
-        MERGE (g:Guild {{guildId: '{guild_id}'}})
-        ON CREATE SET g.createdAt = {int(date_now_timestamp)}
+    query_str = f"""
+        MERGE (g:Guild {{guildId: $guild_id}})
+        ON CREATE SET g.createdAt = $date_now
         WITH g
-        MERGE (c:{community_node} {{id: '{community_id}'}})
-        ON CREATE SET c.createdAt = {int(date_now_timestamp)}
+        MERGE (c:{community_node} {{id: $community_id}})
+        ON CREATE SET c.createdAt = $date_now
         WITH g, c
         MERGE (g) -[r:IS_WITHIN]-> (c)
-        ON CREATE SET r.createdAt = {int(date_now_timestamp)}
+        ON CREATE SET r.createdAt = $date_now
     """
 
+    parameters = {
+        "guild_id": guild_id,
+        "date_now": int(date_now_timestamp),
+        "community_id": community_id,
+    }
+    query = Query(query_str, parameters)
+
     return query
 
 
@@ -139,7 +152,7 @@ def create_network_query(
     nodes_type: str = "DiscordAccount",
     rel_type: str = "INTERACTED_WITH",
     toGuildRelation: str = "IS_MEMBER",
-):
+) -> tuple[list[Query], list[Query]]:
     """
     make string query to save the accounts with their
      account_name and relationships with their relation from **a graph**.
@@ -164,18 +177,18 @@ def create_network_query(
 
     Returns:
     ----------
-    node_queries : list of str
+    node_queries : list[Query]
         the list of MERGE queries for creating all nodes
-    rel_queries : list of str
+    rel_queries : list[Query]
         the list of MERGE queries for creating all relationships
     """
     # getting the timestamp `date`
     graph_date_timestamp = get_timestamp(graph_date)
     date_now_timestamp = get_timestamp()
 
     # initializiation of queries
-    rel_queries = []
-    node_queries = []
+    rel_queries: list[Query] = []
+    node_queries: list[Query] = []
 
     for node in nodes_dict:
         node_str_query = ""
@@ -186,27 +199,33 @@ def create_network_query(
         node_acc_name = node[1]["acc_name"]
         # creating the query
         node_str_query += (
-            f"MERGE (a{node_num}:{nodes_type} {{userId: '{node_acc_name}'}})   "
+            f"MERGE (a{node_num}:{nodes_type} {{userId: $node_acc_name}})   "
         )
         node_str_query += f"""ON CREATE SET a{node_num}.createdAt =
-                                    {int(date_now_timestamp)}
+                                    $date_now_timestamp
                             """
 
         # relationship query between users and guilds
         if guildId is not None:
             # creating the guilds if they weren't created before
             node_str_query += f"""MERGE (g:Guild {{guildId: '{guildId}'}})
-                                ON CREATE SET g.createdAt = {int(date_now_timestamp)}
+                                ON CREATE SET g.createdAt = $date_now_timestamp
                             """
 
             node_str_query += f"""
                 MERGE (a{node_num})
                         -[rel_guild{node_num}:{toGuildRelation}]-> (g)
                     ON CREATE SET
-                        rel_guild{node_num}.createdAt = {int(date_now_timestamp)}
+                        rel_guild{node_num}.createdAt = $date_now_timestamp
             """
 
-        node_queries.append(node_str_query + ";")
+        parameters = {
+            "node_acc_name": node_acc_name,
+            "date_now_timestamp": int(date_now_timestamp),
+        }
+        query_str = node_str_query + ";"
+
+        node_queries.append(Query(query_str, parameters))
 
     for idx, edge in enumerate(edge_dict):
         rel_str_query = ""
@@ -225,19 +244,27 @@ def create_network_query(
         interaction_count = edge[2]["weight"]
 
         rel_str_query += f"""MATCH (a{starting_acc_num}:{nodes_type}
-                            {{userId: \'{starting_node_acc_name}\'}})
+                            {{userId: $starting_node_acc_name}})
                                 MATCH (a{ending_acc_num}:{nodes_type}
-                                  {{userId: \'{ending_node_acc_name}\'}})
+                                  {{userId: $ending_node_acc_name}})
                                 MERGE
                                 (a{starting_acc_num}) -[rel{idx}:{rel_type}
                                     {{
-                                        date: {int(graph_date_timestamp)},
-                                        weight: {int(interaction_count)},
-                                        guildId: '{guildId}'
+                                        date: $date,
+                                        weight: $weight,
+                                        guildId: $guild_id
                                     }}
                                 ]-> (a{ending_acc_num})
                                     """
-        rel_queries.append(rel_str_query + ";")
+        query_str = rel_str_query + ";"
+        parameters = {
+            "starting_node_acc_name": starting_node_acc_name,
+            "ending_node_acc_name": ending_node_acc_name,
+            "date": int(graph_date_timestamp),
+            "weight": int(interaction_count),
+            "guild_id": guildId,
+        }
+        rel_queries.append(Query(query_str, parameters))
 
     return node_queries, rel_queries