Merge pull request #70 from TogetherCrew/development

merging development to main branch. The latest features.

analyzer_init.py

@@ -1,50 +1,38 @@
 from typing import Any
 
 from discord_analyzer import RnDaoAnalyzer
-from utils.daolytics_uitls import (
-    get_mongo_credentials,
-    get_neo4j_credentials,
-    get_saga_db_location,
-)
+from utils.credentials import get_mongo_credentials
 
 
 class AnalyzerInit:
     """
     initialize the analyzer with its configs
     """
 
-    def __init__(self, community_id: str) -> None:
-        self.community_id = community_id
+    # TODO: update to platform_id as input
+    def __init__(self, guild_id: str) -> None:
+        self.guild_id = guild_id
 
-    def get_analyzer(self) -> tuple[RnDaoAnalyzer, dict[str, Any]]:
+    def get_analyzer(self) -> RnDaoAnalyzer:
         """
         Returns:
         ---------
         analyzer : RnDaoAnalyzer
-        mongo_creds : dict[str, Any]
         """
-        analyzer = RnDaoAnalyzer(self.community_id)
+        analyzer = RnDaoAnalyzer(self.guild_id)
 
         # credentials
         mongo_creds = get_mongo_credentials()
-        neo4j_creds = get_neo4j_credentials()
-        saga_mongo_location = get_saga_db_location()
-
-        mongo_creds["db_name"] = saga_mongo_location["db_name"]
-        mongo_creds["collection_name"] = saga_mongo_location["collection_name"]
-        mongo_creds["connection_str"] = self._get_mongo_connection(mongo_creds)
 
         analyzer.set_mongo_database_info(
             mongo_db_host=mongo_creds["host"],
             mongo_db_password=mongo_creds["password"],
             mongo_db_port=mongo_creds["port"],
             mongo_db_user=mongo_creds["user"],
         )
-        analyzer.set_neo4j_database_info(neo4j_creds=neo4j_creds)
         analyzer.database_connect()
-        analyzer.setup_neo4j_metrics()
 
-        return analyzer, mongo_creds
+        return analyzer
 
     def _get_mongo_connection(self, mongo_creds: dict[str, Any]):
         user = mongo_creds["user"]

automation/utils/automation_base.py

@@ -2,18 +2,17 @@
 from typing import Any
 from uuid import uuid1
 
-from utils.get_mongo_client import MongoSingleton
-from utils.get_rabbitmq import prepare_rabbit_mq
+from utils.mongo import MongoSingleton
+from utils.rabbitmq import RabbitMQSingleton
 
 
 class AutomationBase:
     def __init__(self) -> None:
         """
         utilities for automation workflow
         """
-        mongo_singleton = MongoSingleton.get_instance()
-        self.mongo_client = mongo_singleton.get_client()
-        self.rabbitmq = prepare_rabbit_mq()
+        self.mongo_client = MongoSingleton.get_instance().get_client()
+        self.rabbitmq = RabbitMQSingleton.get_instance().get_client()
 
     def _get_users_from_guildmembers(
         self, guild_id: str, user_ids: list[str], strategy: str = "ngu"

automation/utils/model.py

@@ -1,5 +1,5 @@
 from utils.get_automation_env import get_automations_env
-from utils.get_mongo_client import MongoSingleton
+from utils.mongo import MongoSingleton
 
 from .interfaces import Automation
 

discord_analyzer/DB_operations/mongo_neo4j_ops.py

@@ -2,7 +2,7 @@
 
 from discord_analyzer.DB_operations.mongodb_interaction import MongoDBOps
 from discord_analyzer.DB_operations.network_graph import make_neo4j_networkx_query_dict
-from tc_neo4j_lib.neo4j_ops import Neo4jOps
+from tc_neo4j_lib.neo4j_ops import Neo4jOps, Query
 
 
 class MongoNeo4jDB:
@@ -11,34 +11,10 @@ def __init__(self, testing=False):
         having both databases in one class
 
         """
-        self.neo4j_ops = None
+        self.neo4j_ops = Neo4jOps.get_instance()
         self.mongoOps = None
         self.testing = testing
 
-    def set_neo4j_utils(
-        self,
-        db_name: str,
-        host: str,
-        port: str,
-        protocol: str,
-        user: str,
-        password: str,
-    ):
-        """
-        store the neo4j utils instance
-        """
-        self.neo4j_ops = Neo4jOps()
-        self.neo4j_ops.set_neo4j_db_info(
-            neo4j_db_name=db_name,
-            neo4j_protocol=protocol,
-            neo4j_user=user,
-            neo4j_password=password,
-            neo4j_host=host,
-            neo4j_port=port,
-        )
-        self.neo4j_ops.neo4j_database_connect()
-        logging.info("Neo4j Connected Successfully!")
-
     def set_mongo_db_ops(
         self, mongo_user: str, mongo_pass: str, mongo_host: str, mongo_port: str
     ):
@@ -59,6 +35,7 @@ def set_mongo_db_ops(
     def store_analytics_data(
         self,
         analytics_data: dict,
+        guild_id: str,
         community_id: str,
         remove_memberactivities: bool = False,
         remove_heatmaps: bool = False,
@@ -69,12 +46,13 @@ def store_analytics_data(
 
         Parameters:
         -------------
-        analytics_data : dictionary
-            a nested dictinoary with keys as guildId
-            and values as heatmaps and memberactivities data
-            heatmaps is also a list of dictinoaries
+        analytics_data : dict
+            a nested dictinoary with keys as `heatmaps`, and `memberactivities`
+            values of the heatmaps is a list of dictinoaries
             and memberactivities is a tuple of memberactivities dictionary list
-             and memebractivities networkx object dictionary list
+            and memebractivities networkx object dictionary list
+        guild_id: str
+            what the data is related to
         community_id : str
             the community id to save the data for
         remove_memberactivities : bool
@@ -88,43 +66,43 @@ def store_analytics_data(
         ----------
         `None`
         """
-        for guildId in analytics_data.keys():
-            heatmaps_data = analytics_data[guildId]["heatmaps"]
-            (memberactivities_data, memberactivities_networkx_data) = analytics_data[
-                guildId
-            ]["memberactivities"]
-
-            if not self.testing:
-                # mongodb transactions
-                self.mongoOps._do_analytics_write_transaction(
-                    guildId=guildId,
-                    delete_heatmaps=remove_heatmaps,
-                    delete_member_acitivities=remove_memberactivities,
-                    acitivties_list=memberactivities_data,
-                    heatmaps_list=heatmaps_data,
-                )
+        heatmaps_data = analytics_data["heatmaps"]
+        (memberactivities_data, memberactivities_networkx_data) = analytics_data[
+            "memberactivities"
+        ]
+
+        if not self.testing:
+            # mongodb transactions
+            self.mongoOps._do_analytics_write_transaction(
+                guildId=guild_id,
+                delete_heatmaps=remove_heatmaps,
+                delete_member_acitivities=remove_memberactivities,
+                acitivties_list=memberactivities_data,
+                heatmaps_list=heatmaps_data,
+            )
 
-                # neo4j transactions
-                if (
-                    memberactivities_networkx_data is not None
-                    and memberactivities_networkx_data != []
-                ):
-                    queries_list = make_neo4j_networkx_query_dict(
-                        networkx_graphs=memberactivities_networkx_data,
-                        guildId=guildId,
-                        community_id=community_id,
-                    )
-                    self.run_operations_transaction(
-                        guildId=guildId,
-                        queries_list=queries_list,
-                        remove_memberactivities=remove_memberactivities,
-                    )
-            else:
-                logging.warning("Testing mode enabled! Not saving any data")
+            # neo4j transactions
+            if (
+                memberactivities_networkx_data is not None
+                and memberactivities_networkx_data != []
+            ):
+                queries_list = make_neo4j_networkx_query_dict(
+                    networkx_graphs=memberactivities_networkx_data,
+                    guildId=guild_id,
+                    community_id=community_id,
+                )
+                print(queries_list[0])
+                self.run_operations_transaction(
+                    guildId=guild_id,
+                    queries_list=queries_list,
+                    remove_memberactivities=remove_memberactivities,
+                )
+        else:
+            logging.warning("Testing mode enabled! Not saving any data")
 
     def run_operations_transaction(
-        self, guildId, queries_list, remove_memberactivities
-    ):
+        self, guildId: str, queries_list: list[Query], remove_memberactivities: bool
+    ) -> None:
         """
         do the deletion and insertion operations inside a transaction
 
@@ -141,7 +119,7 @@ def run_operations_transaction(
         """
         self.guild_msg = f"GUILDID: {guildId}:"
 
-        transaction_queries = []
+        transaction_queries: list[Query] = []
         if remove_memberactivities:
             logging.info(
                 f"{self.guild_msg} Neo4J GuildId accounts relation will be removed!"
@@ -151,14 +129,13 @@ def run_operations_transaction(
             )
             transaction_queries.append(delete_relationship_query)
 
-        # logging.info(queries_list)
         transaction_queries.extend(queries_list)
 
-        self.neo4j_ops.store_data_neo4j(transaction_queries, message=self.guild_msg)
+        self.neo4j_ops.run_queries_in_batch(transaction_queries, message=self.guild_msg)
 
     def _create_guild_rel_deletion_query(
         self, guildId: str, relation_name: str = "INTERACTED_WITH"
-    ):
+    ) -> Query:
         """
         create a query to delete the relationships
         between DiscordAccount users in a specific guild
@@ -175,11 +152,19 @@ def _create_guild_rel_deletion_query(
         final_query : str
             the final query to remove the relationships
         """
-
-        delete_relationship_query = f"""
+        query_str = f"""
           MATCH
             (:DiscordAccount)
                 -[r:{relation_name} {{guildId: '{guildId}'}}]-(:DiscordAccount)
             DETACH DELETE r"""
 
-        return delete_relationship_query
+        parameters = {
+            "relation_name": relation_name,
+            "guild_id": guildId,
+        }
+
+        query = Query(
+            query=query_str,
+            parameters=parameters,
+        )
+        return query

discord_analyzer/DB_operations/mongodb_interaction.py

@@ -188,38 +188,6 @@ def _batch_insertion(self, collection, data, message, batch_size):
             logging.info(f"{message}: Batch {loop_idx + 1}/{batch_count}")
             collection.insert_many(data[batch_idx : batch_idx + batch_size])
 
-    def check_heatmaps(self, guildId, selectedChannels, heatmap_model):
-        """
-        check whether all the channels are in heatmaps or not
-
-        Parameters:
-        -------------
-        guildId : str
-            the guildId to remove its collection data
-        selectedChannels : list
-            list of `channelId`s
-        heatmap_model : HeatMapModel
-            the heatmaps model to access it
-
-        Returns:
-        ---------
-        is_available : bool
-            is all the selectedChannels available in heatmap collection or not
-        """
-        heatmap_c = heatmap_model(self.mongo_db_access.db_mongo_client[guildId])
-        channels = heatmap_c.get_channels_disctinct()
-
-        if channels is not None:
-            # check if all the selected channels are available in heatmaps
-            is_available = all(element in selectedChannels for element in channels)
-        else:
-            log_msg = "MongoDB heatmaps table check raised an exception,"
-            log_msg += " the heatmaps analysis wouldn't be done!"
-            logging.info(log_msg)
-            is_available = True
-
-        return is_available
-
     def empty_collection(self, session, guildId, activity):
         """
         empty a specified collection