From 27719733504c09dccdb678e67896968f3b18083a Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 22 May 2024 18:13:32 +0330 Subject: [PATCH 1/5] feat: updating usage of Neo4jOps! Many more fixes TODO. --- .../DB_operations/mongo_neo4j_ops.py | 53 ++++-------- .../DB_operations/network_graph.py | 83 ++++++++++++------- .../neo4j_analysis/analyzer_node_stats.py | 8 +- .../analysis/neo4j_analysis/centrality.py | 24 +++--- .../local_clustering_coefficient.py | 6 +- .../analysis/neo4j_analysis/louvain.py | 29 ++++--- discord_analyzer/analysis/neo4j_metrics.py | 10 +-- .../analysis/neo4j_utils/projection_utils.py | 13 +-- discord_analyzer/analyzer/neo4j_analytics.py | 10 +-- .../analyzer/utils/analyzer_db_manager.py | 9 -- discord_analyzer/rn_analyzer.py | 3 +- .../test_decentralization_score.py | 5 +- .../test_degree_centrality_multiple_guilds.py | 5 +- ...ality_multiple_guilds_preserve_parallel.py | 5 +- ...degree_centrality_parallel_preservation.py | 5 +- tests/integration/test_fragmentation_score.py | 5 +- .../test_fragmentation_score_exclude_past.py | 5 +- .../test_fragmentation_score_from_start.py | 5 +- .../test_fragmentation_score_rescaling.py | 5 +- ..._generated_graph_period_1_year_run_once.py | 4 +- .../test_generated_graph_period_1year.py | 4 +- .../test_generated_graph_period_35_days.py | 5 +- ...generated_graph_period_35_days_run_once.py | 5 +- .../test_generated_graph_period_3_months.py | 5 +- ...enerated_graph_period_3_months_run_once.py | 5 +- .../test_generated_graph_period_6_months.py | 5 +- ...enerated_graph_period_6_months_run_once.py | 5 +- .../test_interacted_in_deletion.py | 4 +- tests/integration/test_lcc_all_connected.py | 4 +- .../test_lcc_partially_connected.py | 4 +- .../test_louvain_algorithm_computation.py | 6 +- ...st_louvain_algorithm_get_computed_dates.py | 8 +- .../integration/test_neo4j_compute_metrics.py | 4 +- .../test_neo4j_compute_metrics_from_start.py | 4 +- ...t_neo4j_projection_utils_computed_dates.py | 13 +-- .../test_network_graph_creation.py | 4 +- tests/integration/test_node_stats.py | 4 +- tests/integration/utils/neo4j_conn.py | 1 - 38 files changed, 193 insertions(+), 189 deletions(-) diff --git a/discord_analyzer/DB_operations/mongo_neo4j_ops.py b/discord_analyzer/DB_operations/mongo_neo4j_ops.py index 79da7cd..6cf781c 100644 --- a/discord_analyzer/DB_operations/mongo_neo4j_ops.py +++ b/discord_analyzer/DB_operations/mongo_neo4j_ops.py @@ -2,7 +2,7 @@ from discord_analyzer.DB_operations.mongodb_interaction import MongoDBOps from discord_analyzer.DB_operations.network_graph import make_neo4j_networkx_query_dict -from tc_neo4j_lib.neo4j_ops import Neo4jOps +from tc_neo4j_lib.neo4j_ops import Neo4jOps, Query class MongoNeo4jDB: @@ -11,34 +11,10 @@ def __init__(self, testing=False): having both databases in one class """ - self.neo4j_ops = None + self.neo4j_ops = Neo4jOps.get_instance() self.mongoOps = None self.testing = testing - def set_neo4j_utils( - self, - db_name: str, - host: str, - port: str, - protocol: str, - user: str, - password: str, - ): - """ - store the neo4j utils instance - """ - self.neo4j_ops = Neo4jOps() - self.neo4j_ops.set_neo4j_db_info( - neo4j_db_name=db_name, - neo4j_protocol=protocol, - neo4j_user=user, - neo4j_password=password, - neo4j_host=host, - neo4j_port=port, - ) - self.neo4j_ops.neo4j_database_connect() - logging.info("Neo4j Connected Successfully!") - def set_mongo_db_ops( self, mongo_user: str, mongo_pass: str, mongo_host: str, mongo_port: str ): @@ -124,8 +100,8 @@ def store_analytics_data( logging.warning("Testing mode enabled! Not saving any data") def run_operations_transaction( - self, guildId, queries_list, remove_memberactivities - ): + self, guildId: str, queries_list: list[Query], remove_memberactivities: bool + ) -> None: """ do the deletion and insertion operations inside a transaction @@ -142,7 +118,7 @@ def run_operations_transaction( """ self.guild_msg = f"GUILDID: {guildId}:" - transaction_queries = [] + transaction_queries: list[Query] = [] if remove_memberactivities: logging.info( f"{self.guild_msg} Neo4J GuildId accounts relation will be removed!" @@ -152,14 +128,13 @@ def run_operations_transaction( ) transaction_queries.append(delete_relationship_query) - # logging.info(queries_list) transaction_queries.extend(queries_list) - self.neo4j_ops.store_data_neo4j(transaction_queries, message=self.guild_msg) + self.neo4j_ops.run_queries_in_batch(transaction_queries, message=self.guild_msg) def _create_guild_rel_deletion_query( self, guildId: str, relation_name: str = "INTERACTED_WITH" - ): + ) -> Query: """ create a query to delete the relationships between DiscordAccount users in a specific guild @@ -176,11 +151,19 @@ def _create_guild_rel_deletion_query( final_query : str the final query to remove the relationships """ - - delete_relationship_query = f""" + query_str = f""" MATCH (:DiscordAccount) -[r:{relation_name} {{guildId: '{guildId}'}}]-(:DiscordAccount) DETACH DELETE r""" - return delete_relationship_query + parameters = { + "relation_name": relation_name, + "guild_id": guildId, + } + + query = Query( + query=query_str, + parameters=parameters, + ) + return query diff --git a/discord_analyzer/DB_operations/network_graph.py b/discord_analyzer/DB_operations/network_graph.py index b2dfec6..cf131c1 100644 --- a/discord_analyzer/DB_operations/network_graph.py +++ b/discord_analyzer/DB_operations/network_graph.py @@ -3,13 +3,14 @@ import datetime import networkx +from tc_neo4j_lib import Query def make_neo4j_networkx_query_dict( networkx_graphs: dict[datetime.datetime, networkx.classes.graph.Graph], guildId: str, community_id: str, -): +) -> list[Query]: """ make a list of queries to store networkx graphs into the neo4j @@ -26,7 +27,7 @@ def make_neo4j_networkx_query_dict( Returns: ----------- - queries_list : list + queries_list : list[Query] list of string queries to store data into neo4j """ # extract the graphs and their corresponding interaction dates @@ -53,7 +54,7 @@ def make_graph_list_query( guildId: str, community_id: str, toGuildRelation: str = "IS_MEMBER", -): +) -> list[Query]: """ Make a list of queries for each graph to save their results @@ -75,10 +76,10 @@ def make_graph_list_query( Returns: --------- - final_queries : list of str + final_queries : list[Query] list of strings, each is a query for an interaction graph to be created """ - final_queries = [] + final_queries: list[Query] = [] for graph, date in zip(networkx_graphs, networkx_dates): nodes_dict = graph.nodes.data() @@ -104,7 +105,7 @@ def create_community_node_query( community_id: str, guild_id: str, community_node: str = "Community", -) -> str: +) -> Query: """ create the community node @@ -114,20 +115,32 @@ def create_community_node_query( the community id to create its node guild_id : str the guild node to attach to community + + Returns + --------- + query : Query + the query to run on neo4j to create community node """ date_now_timestamp = get_timestamp() - query = f""" - MERGE (g:Guild {{guildId: '{guild_id}'}}) - ON CREATE SET g.createdAt = {int(date_now_timestamp)} + query_str = f""" + MERGE (g:Guild {{guildId: $guild_id}}) + ON CREATE SET g.createdAt = $date_now WITH g - MERGE (c:{community_node} {{id: '{community_id}'}}) - ON CREATE SET c.createdAt = {int(date_now_timestamp)} + MERGE (c:{community_node} {{id: $community_id}}) + ON CREATE SET c.createdAt = $date_now WITH g, c MERGE (g) -[r:IS_WITHIN]-> (c) - ON CREATE SET r.createdAt = {int(date_now_timestamp)} + ON CREATE SET r.createdAt = $date_now """ + parameters = { + "guild_id": guild_id, + "date_now": int(date_now_timestamp), + "community_id": community_id, + } + query = Query(query_str, parameters) + return query @@ -139,7 +152,7 @@ def create_network_query( nodes_type: str = "DiscordAccount", rel_type: str = "INTERACTED_WITH", toGuildRelation: str = "IS_MEMBER", -): +) -> tuple[list[Query], list[Query]]: """ make string query to save the accounts with their account_name and relationships with their relation from **a graph**. @@ -164,9 +177,9 @@ def create_network_query( Returns: ---------- - node_queries : list of str + node_queries : list[Query] the list of MERGE queries for creating all nodes - rel_queries : list of str + rel_queries : list[Query] the list of MERGE queries for creating all relationships """ # getting the timestamp `date` @@ -174,8 +187,8 @@ def create_network_query( date_now_timestamp = get_timestamp() # initializiation of queries - rel_queries = [] - node_queries = [] + rel_queries: list[Query] = [] + node_queries: list[Query] = [] for node in nodes_dict: node_str_query = "" @@ -186,27 +199,33 @@ def create_network_query( node_acc_name = node[1]["acc_name"] # creating the query node_str_query += ( - f"MERGE (a{node_num}:{nodes_type} {{userId: '{node_acc_name}'}}) " + f"MERGE (a{node_num}:{nodes_type} {{userId: $node_acc_name}}) " ) node_str_query += f"""ON CREATE SET a{node_num}.createdAt = - {int(date_now_timestamp)} + $date_now_timestamp """ # relationship query between users and guilds if guildId is not None: # creating the guilds if they weren't created before node_str_query += f"""MERGE (g:Guild {{guildId: '{guildId}'}}) - ON CREATE SET g.createdAt = {int(date_now_timestamp)} + ON CREATE SET g.createdAt = $date_now_timestamp """ node_str_query += f""" MERGE (a{node_num}) -[rel_guild{node_num}:{toGuildRelation}]-> (g) ON CREATE SET - rel_guild{node_num}.createdAt = {int(date_now_timestamp)} + rel_guild{node_num}.createdAt = $date_now_timestamp """ - node_queries.append(node_str_query + ";") + parameters = { + "node_acc_name": node_acc_name, + date_now_timestamp: int(date_now_timestamp), + } + query_str = node_str_query + ";" + + node_queries.append(Query(query_str, parameters)) for idx, edge in enumerate(edge_dict): rel_str_query = "" @@ -225,19 +244,27 @@ def create_network_query( interaction_count = edge[2]["weight"] rel_str_query += f"""MATCH (a{starting_acc_num}:{nodes_type} - {{userId: \'{starting_node_acc_name}\'}}) + {{userId: $starting_node_acc_name}}) MATCH (a{ending_acc_num}:{nodes_type} - {{userId: \'{ending_node_acc_name}\'}}) + {{userId: $ending_node_acc_name}}) MERGE (a{starting_acc_num}) -[rel{idx}:{rel_type} {{ - date: {int(graph_date_timestamp)}, - weight: {int(interaction_count)}, - guildId: '{guildId}' + date: $date, + weight: $weight, + guildId: $guild_id }} ]-> (a{ending_acc_num}) """ - rel_queries.append(rel_str_query + ";") + query_str = rel_str_query + ";" + parameters = { + "starting_node_acc_name": starting_node_acc_name, + "ending_node_acc_name": ending_node_acc_name, + "date": int(graph_date_timestamp), + "weight": int(interaction_count), + "guild_id": guildId, + } + rel_queries.append(Query(query_str, parameters)) return node_queries, rel_queries diff --git a/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py b/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py index 5741669..14c76b2 100644 --- a/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py +++ b/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py @@ -123,13 +123,15 @@ def get_computed_dates( """ get the computed dates of our guild """ - query = f""" + query = """ MATCH (:DiscordAccount) - -[r:INTERACTED_IN]->(g:Guild {{guildId: '{guildId}'}}) + -[r:INTERACTED_IN]->(g:Guild {guildId: $guild_id}) WHERE r.status IS NOT NULL RETURN r.date as computed_dates """ - computed_dates = projection_utils.get_computed_dates(query=query) + computed_dates = projection_utils.get_computed_dates( + query=query, guild_id=guildId + ) return computed_dates diff --git a/discord_analyzer/analysis/neo4j_analysis/centrality.py b/discord_analyzer/analysis/neo4j_analysis/centrality.py index bf7bf17..5bf03b1 100644 --- a/discord_analyzer/analysis/neo4j_analysis/centrality.py +++ b/discord_analyzer/analysis/neo4j_analysis/centrality.py @@ -4,7 +4,7 @@ import pandas as pd from discord_analyzer.analysis.neo4j_metrics import Neo4JMetrics from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils -from tc_neo4j_lib.neo4j_ops import Neo4jOps +from tc_neo4j_lib.neo4j_ops import Neo4jOps, Query class Centerality: @@ -143,13 +143,13 @@ def _get_dates_to_compute( guildId : str the guildId to get computations date """ - query = f""" - MATCH (g:Guild {{guildId: '{guildId}'}}) + query = """ + MATCH (g:Guild {guildId: $guild_id}) -[r:HAVE_METRICS] -> (g) WHERE r.decentralizationScore IS NOT NULL RETURN r.date as computed_dates """ - computed_dates = projection_utils.get_computed_dates(query) + computed_dates = projection_utils.get_computed_dates(query, guild_id=guildId) dates_to_compute = user_interaction_dates - computed_dates @@ -348,18 +348,18 @@ def save_decentralization_score( the network decentrality scores over time """ # preparing the queries - queries = [] + queries: list[Query] = [] for date in decentrality_score.keys(): - query = f""" - MATCH (g: Guild {{guildId: '{guildId}'}}) - MERGE (g) -[r:HAVE_METRICS {{ - date: {date} - }}]-> (g) - SET r.decentralizationScore = {decentrality_score[date]} + query_str = """ + MATCH (g: Guild {guildId: $guild_id}) + MERGE (g) -[r:HAVE_METRICS {date: $date}]-> (g) + SET r.decentralizationScore = $score """ + parameters = {"guild_id": guildId, "score": decentrality_score[date]} + query = Query(query=query_str, parameters=parameters) queries.append(query) - self.neo4j_ops.store_data_neo4j( + self.neo4j_ops.run_queries_in_batch( queries, message=f"GUILDID: {guildId}: Saving Network Decentrality:", ) diff --git a/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py b/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py index 869cd8f..49fabfd 100644 --- a/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py +++ b/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py @@ -118,13 +118,13 @@ def get_computed_dates( the computation dates """ # getting the dates computed before - query = f""" + query = """ MATCH (:DiscordAccount) - -[r:INTERACTED_IN]->(g:Guild {{guildId: '{guildId}'}}) + -[r:INTERACTED_IN]->(g:Guild {guildId: $guild_id}) WHERE r.localClusteringCoefficient IS NOT NULL RETURN r.date as computed_dates """ - computed_dates = projection_utils.get_computed_dates(query) + computed_dates = projection_utils.get_computed_dates(query, guild_id=guildId) return computed_dates diff --git a/discord_analyzer/analysis/neo4j_analysis/louvain.py b/discord_analyzer/analysis/neo4j_analysis/louvain.py index 82eb060..94c8073 100644 --- a/discord_analyzer/analysis/neo4j_analysis/louvain.py +++ b/discord_analyzer/analysis/neo4j_analysis/louvain.py @@ -6,11 +6,11 @@ class Louvain: - def __init__(self, neo4j_ops: Neo4jOps) -> None: + def __init__(self) -> None: """ louvain algorithm wrapper to compute """ - self.neo4j_ops = neo4j_ops + self.neo4j_ops = Neo4jOps.get_instance() def compute(self, guild_id: str, from_start: bool = False) -> None: """ @@ -25,7 +25,7 @@ def compute(self, guild_id: str, from_start: bool = False) -> None: if True, then would compute from start default is False """ - projection_utils = ProjectionUtils(gds=self.neo4j_ops.gds, guildId=guild_id) + projection_utils = ProjectionUtils(guildId=guild_id) computable_dates = projection_utils.get_dates(guildId=guild_id) @@ -105,13 +105,13 @@ def get_computed_dates( the computation dates """ # getting the dates computed before - query = f""" - MATCH (g:Guild {{guildId: '{guildId}'}}) + query = """ + MATCH (g:Guild {guildId: $guild_id}) -[r:HAVE_METRICS]->(g) WHERE r.louvainModularityScore IS NOT NULL RETURN r.date as computed_dates """ - computed_dates = projection_utils.get_computed_dates(query) + computed_dates = projection_utils.get_computed_dates(query, guild_id=guildId) return computed_dates @@ -134,16 +134,19 @@ def compute_graph_louvain( msg = f"GUILDID: {guild_id}" try: _ = self.neo4j_ops.gds.run_cypher( - f""" - CALL gds.louvain.stats("{graph_name}") + """ + CALL gds.louvain.stats($graph_name) YIELD modularity WITH modularity - MATCH (g:Guild {{guildId: '{guild_id}'}}) - MERGE (g) -[r:HAVE_METRICS {{ - date: {date} - }}]-> (g) + MATCH (g:Guild {guildId: $guild_id}) + MERGE (g) -[r:HAVE_METRICS { + date: $date + }]-> (g) SET r.louvainModularityScore = modularity - """ + """, + graph_name=graph_name, + guild_id=guild_id, + date=date, ) except Exception as exp: logging.error( diff --git a/discord_analyzer/analysis/neo4j_metrics.py b/discord_analyzer/analysis/neo4j_metrics.py index b23d6b3..bbc0d67 100644 --- a/discord_analyzer/analysis/neo4j_metrics.py +++ b/discord_analyzer/analysis/neo4j_metrics.py @@ -1,8 +1,7 @@ import os from discord_analyzer.analysis.neo4j_utils.compute_metrics import Neo4JMetrics -from dotenv import load_dotenv -from tc_neo4j_lib.neo4j_ops import Neo4jOps +from dotenv import load_dotenvfrom tc_neo4j_lib.neo4j_ops import Neo4jOps def degree_centrality( @@ -206,10 +205,9 @@ def decenterialization_score(neo4j_analytics, centrality_scores): results_degreeCenterality["score_undirected"] = results_degreeCenterality["score"] # normalizing undirected scores - results_degreeCenterality[ - "normalized_score_undirected" - ] = results_degreeCenterality["score"] / sum( - results_degreeCenterality["score"].values > 0 + results_degreeCenterality["normalized_score_undirected"] = ( + results_degreeCenterality["score"] + / sum(results_degreeCenterality["score"].values > 0) ) # the normalization over positive score_out results_degreeCenterality["normalized_score_out"] = results_degreeCenterality[ diff --git a/discord_analyzer/analysis/neo4j_utils/projection_utils.py b/discord_analyzer/analysis/neo4j_utils/projection_utils.py index 070714d..3201071 100644 --- a/discord_analyzer/analysis/neo4j_utils/projection_utils.py +++ b/discord_analyzer/analysis/neo4j_utils/projection_utils.py @@ -1,11 +1,10 @@ import logging - -from graphdatascience import GraphDataScience +from tc_neo4j_lib.neo4j_ops import Neo4jOps class ProjectionUtils: - def __init__(self, gds: GraphDataScience, guildId: str) -> None: - self.gds = gds + def __init__(self, guildId: str) -> None: + self.gds = Neo4jOps.get_instance().gds self.guildId = guildId def project_temp_graph( @@ -123,7 +122,7 @@ def get_dates(self, guildId: str) -> set[float]: return computable_dates_set - def get_computed_dates(self, query: str) -> set[float]: + def get_computed_dates(self, query: str, **params) -> set[float]: """ get the computed metric dates for that specific query @@ -133,8 +132,10 @@ def get_computed_dates(self, query: str) -> set[float]: the query to get the computed dates of a metric must have one return results with label of computed_dates first one is date + params: Dict[str, Any] + parameters to the query """ - dates = self.gds.run_cypher(query) + dates = self.gds.run_cypher(query, params) computed_dates = set(dates["computed_dates"].values) return computed_dates diff --git a/discord_analyzer/analyzer/neo4j_analytics.py b/discord_analyzer/analyzer/neo4j_analytics.py index 3672dae..3063662 100644 --- a/discord_analyzer/analyzer/neo4j_analytics.py +++ b/discord_analyzer/analyzer/neo4j_analytics.py @@ -11,12 +11,12 @@ class Neo4JAnalytics: - def __init__(self, neo4j_ops: Neo4jOps) -> None: + def __init__(self) -> None: """ neo4j metrics to be compute input variables are all the neo4j credentials """ - self.neo4j_ops = neo4j_ops + self.neo4j_ops = Neo4jOps.get_instance() def compute_metrics(self, guildId: str, from_start: bool) -> None: """ @@ -116,7 +116,7 @@ def compute_network_decentrality(self, guildId: str, from_start: bool): """ msg = f"GUILDID: {guildId}:" try: - centrality = Centerality(self.neo4j_ops) + centrality = Centerality() # degree decentrality _ = centrality.compute_network_decentrality( guildId=guildId, from_start=from_start @@ -137,7 +137,7 @@ def compute_node_stats(self, guildId: str, from_start: bool): msg = f"GUILDID: {guildId}:" try: logging.info(f"{msg}: computing node stats") - node_stats = NodeStats(self.neo4j_ops, threshold=2) + node_stats = NodeStats(threshold=2) node_stats.compute_stats(guildId, from_start) except Exception as exp: logging.error(f"{msg} Exception occured in node stats computation, {exp}") @@ -170,6 +170,6 @@ def compute_louvain_algorithm(self, guild_id: str, from_start: bool) -> None: from_start : bool compute from the start of the data available or continue the previous """ - louvain = Louvain(self.neo4j_ops) + louvain = Louvain() louvain.compute(guild_id, from_start) diff --git a/discord_analyzer/analyzer/utils/analyzer_db_manager.py b/discord_analyzer/analyzer/utils/analyzer_db_manager.py index 7f7ad22..53abee6 100644 --- a/discord_analyzer/analyzer/utils/analyzer_db_manager.py +++ b/discord_analyzer/analyzer/utils/analyzer_db_manager.py @@ -63,12 +63,3 @@ def database_connect(self): mongo_host=self.mongo_host, mongo_port=self.mongo_port, ) - - self.DB_connections.set_neo4j_utils( - db_name=self.neo4j_db_name, - host=self.neo4j_host, - port=self.neo4j_port, - protocol=self.neo4j_protocol, - user=self.neo4j_user, - password=self.neo4j_password, - ) diff --git a/discord_analyzer/rn_analyzer.py b/discord_analyzer/rn_analyzer.py index f4a519f..9a76629 100644 --- a/discord_analyzer/rn_analyzer.py +++ b/discord_analyzer/rn_analyzer.py @@ -30,7 +30,8 @@ def setup_neo4j_metrics(self) -> None: """ setup the neo4j analytics wrapper """ - self.neo4j_analytics = Neo4JAnalytics(neo4j_ops=self.DB_connections.neo4j_ops) + + self.neo4j_analytics = Neo4JAnalytics() def run_once(self): """Run analysis once (Wrapper)""" diff --git a/tests/integration/test_decentralization_score.py b/tests/integration/test_decentralization_score.py index 80a1c93..25b4732 100644 --- a/tests/integration/test_decentralization_score.py +++ b/tests/integration/test_decentralization_score.py @@ -1,7 +1,6 @@ # the nodes of the graph are partially connected from discord_analyzer.analysis.neo4j_analysis.centrality import Centerality - -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_decentralization_score(): @@ -13,7 +12,7 @@ def test_decentralization_score(): https://miro.com/app/board/uXjVM7GdYqo=/?moveToWidget=3458764558210553321&cot=14 """ guildId = "1234" - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() centrality = Centerality(neo4j_ops) # deleting all data diff --git a/tests/integration/test_degree_centrality_multiple_guilds.py b/tests/integration/test_degree_centrality_multiple_guilds.py index 432cee9..141b93b 100644 --- a/tests/integration/test_degree_centrality_multiple_guilds.py +++ b/tests/integration/test_degree_centrality_multiple_guilds.py @@ -1,8 +1,7 @@ # we have nodes of a community is connected to another one # meaning we have nodes available in more than one community from discord_analyzer.analysis.neo4j_analysis.centrality import Centerality - -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_multiple_guilds(): @@ -16,7 +15,7 @@ def test_multiple_guilds(): https://miro.com/app/board/uXjVM7GdYqo=/?share_link_id=105382864070 """ guildId = "1234" - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_degree_centrality_multiple_guilds_preserve_parallel.py b/tests/integration/test_degree_centrality_multiple_guilds_preserve_parallel.py index 2de197d..0434914 100644 --- a/tests/integration/test_degree_centrality_multiple_guilds_preserve_parallel.py +++ b/tests/integration/test_degree_centrality_multiple_guilds_preserve_parallel.py @@ -1,8 +1,7 @@ # we have nodes of a community is connected to another one # meaning we have nodes available in more than one community from discord_analyzer.analysis.neo4j_analysis.centrality import Centerality - -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_multiple_guilds_preserve_parallel(): @@ -16,7 +15,7 @@ def test_multiple_guilds_preserve_parallel(): https://miro.com/app/board/uXjVM7GdYqo=/?share_link_id=105382864070 """ guildId = "1234" - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() centrality = Centerality(neo4j_ops) # deleting all data diff --git a/tests/integration/test_degree_centrality_parallel_preservation.py b/tests/integration/test_degree_centrality_parallel_preservation.py index af93399..1c3e0e9 100644 --- a/tests/integration/test_degree_centrality_parallel_preservation.py +++ b/tests/integration/test_degree_centrality_parallel_preservation.py @@ -1,7 +1,6 @@ # the nodes of the graph are partially connected from discord_analyzer.analysis.neo4j_analysis.centrality import Centerality - -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_partially_connected_coeffs(): @@ -13,7 +12,7 @@ def test_partially_connected_coeffs(): https://miro.com/app/board/uXjVM7GdYqo=/?share_link_id=105382864070 """ guildId = "1234" - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_fragmentation_score.py b/tests/integration/test_fragmentation_score.py index a81b135..1708e68 100644 --- a/tests/integration/test_fragmentation_score.py +++ b/tests/integration/test_fragmentation_score.py @@ -1,15 +1,14 @@ from datetime import datetime, timedelta from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics - -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_avg_clustering_coeff(): """ test scaling of the avgClusteringCoefficient (a.k.a fragmentation score) """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() neo4j_analytics = Neo4JAnalytics(neo4j_ops) # deleting all data diff --git a/tests/integration/test_fragmentation_score_exclude_past.py b/tests/integration/test_fragmentation_score_exclude_past.py index 312328e..40ba8f9 100644 --- a/tests/integration/test_fragmentation_score_exclude_past.py +++ b/tests/integration/test_fragmentation_score_exclude_past.py @@ -1,15 +1,14 @@ from datetime import datetime, timedelta from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics - -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_avg_clustering_exclude_past(): """ test scaling of the avgClusteringCoefficient (a.k.a fragmentation score) """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() neo4j_analytics = Neo4JAnalytics(neo4j_ops) # deleting all data diff --git a/tests/integration/test_fragmentation_score_from_start.py b/tests/integration/test_fragmentation_score_from_start.py index 700f385..404bce0 100644 --- a/tests/integration/test_fragmentation_score_from_start.py +++ b/tests/integration/test_fragmentation_score_from_start.py @@ -1,15 +1,14 @@ from datetime import datetime, timedelta from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics - -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_avg_clustering_coeff_from_start(): """ test scaling of the avgClusteringCoefficient (a.k.a fragmentation score) """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() neo4j_analytics = Neo4JAnalytics(neo4j_ops) # deleting all data diff --git a/tests/integration/test_fragmentation_score_rescaling.py b/tests/integration/test_fragmentation_score_rescaling.py index 40c8975..607df8f 100644 --- a/tests/integration/test_fragmentation_score_rescaling.py +++ b/tests/integration/test_fragmentation_score_rescaling.py @@ -1,15 +1,14 @@ from datetime import datetime, timedelta from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics - -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_avg_clustering_coeff_scaling(): """ test scaling of the avgClusteringCoefficient (a.k.a fragmentation score) """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() neo4j_analytics = Neo4JAnalytics(neo4j_ops) # deleting all data diff --git a/tests/integration/test_generated_graph_period_1_year_run_once.py b/tests/integration/test_generated_graph_period_1_year_run_once.py index 875616c..8ba7556 100644 --- a/tests/integration/test_generated_graph_period_1_year_run_once.py +++ b/tests/integration/test_generated_graph_period_1_year_run_once.py @@ -5,7 +5,7 @@ from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild @@ -20,7 +20,7 @@ def test_networkgraph_one_year_period_run_once_available_analytics(): community_id = "aabbccddeeff001122334455" platform_id = "515151515151515151515151" db_access = launch_db_access(guildId) - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() neo4j_ops.gds.run_cypher( """ diff --git a/tests/integration/test_generated_graph_period_1year.py b/tests/integration/test_generated_graph_period_1year.py index db62ac2..3404075 100644 --- a/tests/integration/test_generated_graph_period_1year.py +++ b/tests/integration/test_generated_graph_period_1year.py @@ -5,7 +5,7 @@ from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild @@ -20,7 +20,7 @@ def test_networkgraph_one_year_period_recompute_available_analytics(): community_id = "aabbccddeeff001122334455" platform_id = "515151515151515151515151" db_access = launch_db_access(guildId) - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() neo4j_ops.gds.run_cypher( """ diff --git a/tests/integration/test_generated_graph_period_35_days.py b/tests/integration/test_generated_graph_period_35_days.py index f05709c..0d20b41 100644 --- a/tests/integration/test_generated_graph_period_35_days.py +++ b/tests/integration/test_generated_graph_period_35_days.py @@ -5,7 +5,8 @@ from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data -from .utils.neo4j_conn import neo4j_setup + +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild @@ -20,7 +21,7 @@ def test_networkgraph_35_days_period_recompute_available_analytics(): platform_id = "515151515151515151515151" community_id = "aabbccddeeff001122334455" db_access = launch_db_access(guildId) - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() neo4j_ops.gds.run_cypher( """ diff --git a/tests/integration/test_generated_graph_period_35_days_run_once.py b/tests/integration/test_generated_graph_period_35_days_run_once.py index 41b8f62..8e06333 100644 --- a/tests/integration/test_generated_graph_period_35_days_run_once.py +++ b/tests/integration/test_generated_graph_period_35_days_run_once.py @@ -5,7 +5,8 @@ from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data -from .utils.neo4j_conn import neo4j_setup + +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild @@ -20,7 +21,7 @@ def test_networkgraph_35_days_period_run_once_available_analytics(): platform_id = "515151515151515151515151" community_id = "aabbccddeeff001122334455" db_access = launch_db_access(guildId) - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() neo4j_ops.gds.run_cypher( """ diff --git a/tests/integration/test_generated_graph_period_3_months.py b/tests/integration/test_generated_graph_period_3_months.py index bd170e7..ddae3dc 100644 --- a/tests/integration/test_generated_graph_period_3_months.py +++ b/tests/integration/test_generated_graph_period_3_months.py @@ -5,7 +5,8 @@ from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data -from .utils.neo4j_conn import neo4j_setup + +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild @@ -20,7 +21,7 @@ def test_networkgraph_three_months_period_recompute_available_analytics(): community_id = "aabbccddeeff001122334455" platform_id = "515151515151515151515151" db_access = launch_db_access(guildId) - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() neo4j_ops.gds.run_cypher( """ diff --git a/tests/integration/test_generated_graph_period_3_months_run_once.py b/tests/integration/test_generated_graph_period_3_months_run_once.py index 5e4b134..9d71b55 100644 --- a/tests/integration/test_generated_graph_period_3_months_run_once.py +++ b/tests/integration/test_generated_graph_period_3_months_run_once.py @@ -5,7 +5,8 @@ from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data -from .utils.neo4j_conn import neo4j_setup + +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild @@ -20,7 +21,7 @@ def test_networkgraph_three_months_period_run_once_available_analytics(): platform_id = "515151515151515151515151" community_id = "aabbccddeeff001122334455" db_access = launch_db_access(guildId) - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() neo4j_ops.gds.run_cypher( """ diff --git a/tests/integration/test_generated_graph_period_6_months.py b/tests/integration/test_generated_graph_period_6_months.py index 01ee33c..834f615 100644 --- a/tests/integration/test_generated_graph_period_6_months.py +++ b/tests/integration/test_generated_graph_period_6_months.py @@ -5,7 +5,8 @@ from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data -from .utils.neo4j_conn import neo4j_setup + +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild @@ -20,7 +21,7 @@ def test_networkgraph_six_months_period_recompute_available_analytics(): platform_id = "515151515151515151515151" community_id = "aabbccddeeff001122334455" db_access = launch_db_access(guildId) - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() neo4j_ops.gds.run_cypher( """ diff --git a/tests/integration/test_generated_graph_period_6_months_run_once.py b/tests/integration/test_generated_graph_period_6_months_run_once.py index e76f635..a655fac 100644 --- a/tests/integration/test_generated_graph_period_6_months_run_once.py +++ b/tests/integration/test_generated_graph_period_6_months_run_once.py @@ -5,7 +5,8 @@ from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data -from .utils.neo4j_conn import neo4j_setup + +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild @@ -21,7 +22,7 @@ def test_networkgraph_six_months_period_run_once_available_analytics(): platform_id = "515151515151515151515151" db_access = launch_db_access(guildId) - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() neo4j_ops.gds.run_cypher( """ diff --git a/tests/integration/test_interacted_in_deletion.py b/tests/integration/test_interacted_in_deletion.py index 01a854d..df09d1f 100644 --- a/tests/integration/test_interacted_in_deletion.py +++ b/tests/integration/test_interacted_in_deletion.py @@ -1,6 +1,6 @@ from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_interacted_in_deletion(): @@ -8,7 +8,7 @@ def test_interacted_in_deletion(): test whether we're deleting the INTERACTED_IN relations or not """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() neo4j_analytics = Neo4JAnalytics(neo4j_ops) neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_lcc_all_connected.py b/tests/integration/test_lcc_all_connected.py index f580c0a..0955f91 100644 --- a/tests/integration/test_lcc_all_connected.py +++ b/tests/integration/test_lcc_all_connected.py @@ -3,7 +3,7 @@ LocalClusteringCoeff, ) -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_all_connected_coeffs(): @@ -16,7 +16,7 @@ def test_all_connected_coeffs(): To see more info for this test: https://miro.com/app/board/uXjVM7GdYqo=/?share_link_id=105382864070 """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_lcc_partially_connected.py b/tests/integration/test_lcc_partially_connected.py index 6ed6f78..5cd81e7 100644 --- a/tests/integration/test_lcc_partially_connected.py +++ b/tests/integration/test_lcc_partially_connected.py @@ -3,7 +3,7 @@ LocalClusteringCoeff, ) -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_partially_connected_coeffs(): @@ -14,7 +14,7 @@ def test_partially_connected_coeffs(): To see more info for this test: https://miro.com/app/board/uXjVM7GdYqo=/?share_link_id=105382864070 """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_louvain_algorithm_computation.py b/tests/integration/test_louvain_algorithm_computation.py index b08c224..94141d3 100644 --- a/tests/integration/test_louvain_algorithm_computation.py +++ b/tests/integration/test_louvain_algorithm_computation.py @@ -1,13 +1,13 @@ from discord_analyzer.analysis.neo4j_analysis.louvain import Louvain -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_louvain_algorithm_available_data(): """ test the louvain algorithm with some nodes connected """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") @@ -55,7 +55,7 @@ def test_louvain_algorithm_more_available_data(): """ test the louvain algorithm with some more data available """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_louvain_algorithm_get_computed_dates.py b/tests/integration/test_louvain_algorithm_get_computed_dates.py index c2a1a13..c1c6cbe 100644 --- a/tests/integration/test_louvain_algorithm_get_computed_dates.py +++ b/tests/integration/test_louvain_algorithm_get_computed_dates.py @@ -1,14 +1,14 @@ from discord_analyzer.analysis.neo4j_analysis.louvain import Louvain from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_louvain_get_computed_dates_empty_data(): """ test with empty data for getting the computed dates """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") @@ -50,7 +50,7 @@ def test_louvain_get_computed_dates_empty_data_with_have_metrics_relation(): """ test with empty data for getting the computed dates """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") @@ -93,7 +93,7 @@ def test_louvain_get_computed_dates_one_data(): """ test with empty data for getting the computed dates """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_neo4j_compute_metrics.py b/tests/integration/test_neo4j_compute_metrics.py index e2d0295..62e4c8b 100644 --- a/tests/integration/test_neo4j_compute_metrics.py +++ b/tests/integration/test_neo4j_compute_metrics.py @@ -1,7 +1,7 @@ import numpy as np from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_guild_results_available(): @@ -12,7 +12,7 @@ def test_guild_results_available(): and decentralization scores are available in guild node and localClustetingCoefficient is available in DiscordAccount nodes """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_neo4j_compute_metrics_from_start.py b/tests/integration/test_neo4j_compute_metrics_from_start.py index 33d0d7d..1233232 100644 --- a/tests/integration/test_neo4j_compute_metrics_from_start.py +++ b/tests/integration/test_neo4j_compute_metrics_from_start.py @@ -1,7 +1,7 @@ import numpy as np from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_neo4j_compute_metrics_from_start(): @@ -12,7 +12,7 @@ def test_neo4j_compute_metrics_from_start(): and decentralization scores are available in guild node and localClustetingCoefficient is available in DiscordAccount nodes """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_neo4j_projection_utils_computed_dates.py b/tests/integration/test_neo4j_projection_utils_computed_dates.py index ecbf3e3..9804859 100644 --- a/tests/integration/test_neo4j_projection_utils_computed_dates.py +++ b/tests/integration/test_neo4j_projection_utils_computed_dates.py @@ -1,13 +1,13 @@ from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils -from .utils.neo4j_conn import neo4j_setup +# from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_neo4j_projection_utils_get_computed_dates(): """ testing the projection utils get_computed_dates """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") @@ -65,13 +65,14 @@ def test_neo4j_projection_utils_get_computed_dates(): SET r12.guildId = '{guildId}' """ ) - projection_utils = ProjectionUtils(neo4j_ops.gds, guildId=guildId) + projection_utils = ProjectionUtils(guildId=guildId) computed_dates = projection_utils.get_computed_dates( - f""" - MATCH (:DiscordAccount)-[r:INTERACTED_IN]->(g:Guild {{guildId: '{guildId}'}}) + """ + MATCH (:DiscordAccount)-[r:INTERACTED_IN]->(g:Guild {guildId: $guild_id}) WHERE r.localClusteringCoefficient is NOT NULL RETURN r.date as computed_dates - """ + """, + guild_id=guildId, ) print(computed_dates) diff --git a/tests/integration/test_network_graph_creation.py b/tests/integration/test_network_graph_creation.py index e2cc15e..6464282 100644 --- a/tests/integration/test_network_graph_creation.py +++ b/tests/integration/test_network_graph_creation.py @@ -6,12 +6,12 @@ from discord_analyzer.analysis.utils.activity import Activity from .utils.mock_graph import generate_mock_graph, store_mock_data_in_neo4j -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_network_graph_create(): community_id = "4321" - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_node_stats.py b/tests/integration/test_node_stats.py index 430cf18..5bda02b 100644 --- a/tests/integration/test_node_stats.py +++ b/tests/integration/test_node_stats.py @@ -1,7 +1,7 @@ # test out local clustering coefficient with all nodes connected from discord_analyzer.analysis.neo4j_analysis.analyzer_node_stats import NodeStats -from .utils.neo4j_conn import neo4j_setup +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_node_stats(): @@ -12,7 +12,7 @@ def test_node_stats(): To see the graph for this test: https://miro.com/app/board/uXjVM7GdYqo=/?share_link_id=105382864070 """ - neo4j_ops = neo4j_setup() + neo4j_ops = Neo4jOps.get_instance() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/utils/neo4j_conn.py b/tests/integration/utils/neo4j_conn.py index 8a534be..7a7c163 100644 --- a/tests/integration/utils/neo4j_conn.py +++ b/tests/integration/utils/neo4j_conn.py @@ -1,7 +1,6 @@ import os from dotenv import load_dotenv -from tc_neo4j_lib.neo4j_ops import Neo4jOps def neo4j_setup() -> Neo4jOps: From 0f677d3da37b13d93be0452a0cc52ac38b549272 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 22 May 2024 18:15:31 +0330 Subject: [PATCH 2/5] fix: import syntax error! --- discord_analyzer/analysis/neo4j_metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/discord_analyzer/analysis/neo4j_metrics.py b/discord_analyzer/analysis/neo4j_metrics.py index bbc0d67..8750476 100644 --- a/discord_analyzer/analysis/neo4j_metrics.py +++ b/discord_analyzer/analysis/neo4j_metrics.py @@ -1,7 +1,8 @@ import os from discord_analyzer.analysis.neo4j_utils.compute_metrics import Neo4JMetrics -from dotenv import load_dotenvfrom tc_neo4j_lib.neo4j_ops import Neo4jOps +from dotenv import load_dotenv +from tc_neo4j_lib.neo4j_ops import Neo4jOps def degree_centrality( From 7e629ecd9e9ec2d7eda80f4da8934214de8cd11c Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 23 May 2024 10:08:52 +0330 Subject: [PATCH 3/5] fix: removing the manually creation of neo4j instance! --- analyzer_init.py | 5 +- .../DB_operations/mongo_neo4j_ops.py | 1 + .../DB_operations/network_graph.py | 2 +- .../neo4j_analysis/analyzer_node_stats.py | 40 +-- .../analysis/neo4j_analysis/centrality.py | 23 +- .../local_clustering_coefficient.py | 40 +-- .../analysis/neo4j_analysis/louvain.py | 15 +- discord_analyzer/analysis/neo4j_metrics.py | 242 ------------------ .../{compute_metrics.py => neo4j_metrics.py} | 6 +- discord_analyzer/analyzer/neo4j_analytics.py | 2 +- .../analyzer/utils/analyzer_db_manager.py | 23 -- discord_analyzer/rn_analyzer.py | 8 +- .../test_assess_engagement_mention.py | 4 +- .../test_assess_engagement_reactions.py | 4 +- .../test_assess_engagement_replies.py | 4 +- .../test_decentralization_score.py | 2 +- .../test_degree_centrality_multiple_guilds.py | 2 +- ...ality_multiple_guilds_preserve_parallel.py | 2 +- ...degree_centrality_parallel_preservation.py | 2 +- tests/integration/test_fragmentation_score.py | 2 +- .../test_fragmentation_score_exclude_past.py | 2 +- .../test_fragmentation_score_from_start.py | 2 +- .../test_fragmentation_score_rescaling.py | 2 +- .../test_interacted_in_deletion.py | 2 +- tests/integration/test_lcc_all_connected.py | 2 +- .../test_lcc_partially_connected.py | 2 +- .../test_louvain_algorithm_computation.py | 4 +- ...st_louvain_algorithm_get_computed_dates.py | 18 +- .../integration/test_neo4j_compute_metrics.py | 2 +- .../test_neo4j_compute_metrics_from_start.py | 2 +- ...t_neo4j_projection_utils_computed_dates.py | 2 +- tests/integration/test_node_stats.py | 2 +- tests/integration/utils/analyzer_setup.py | 10 - tests/integration/utils/mock_graph.py | 9 - tests/unit/test_creds_loading.py | 25 -- utils/credentials.py | 28 -- 36 files changed, 101 insertions(+), 442 deletions(-) delete mode 100644 discord_analyzer/analysis/neo4j_metrics.py rename discord_analyzer/analysis/neo4j_utils/{compute_metrics.py => neo4j_metrics.py} (98%) diff --git a/analyzer_init.py b/analyzer_init.py index 52a9555..9cfdcaf 100644 --- a/analyzer_init.py +++ b/analyzer_init.py @@ -1,7 +1,7 @@ from typing import Any from discord_analyzer import RnDaoAnalyzer -from utils.credentials import get_mongo_credentials, get_neo4j_credentials +from utils.credentials import get_mongo_credentials class AnalyzerInit: @@ -23,7 +23,6 @@ def get_analyzer(self) -> RnDaoAnalyzer: # credentials mongo_creds = get_mongo_credentials() - neo4j_creds = get_neo4j_credentials() analyzer.set_mongo_database_info( mongo_db_host=mongo_creds["host"], @@ -31,9 +30,7 @@ def get_analyzer(self) -> RnDaoAnalyzer: mongo_db_port=mongo_creds["port"], mongo_db_user=mongo_creds["user"], ) - analyzer.set_neo4j_database_info(neo4j_creds=neo4j_creds) analyzer.database_connect() - analyzer.setup_neo4j_metrics() return analyzer diff --git a/discord_analyzer/DB_operations/mongo_neo4j_ops.py b/discord_analyzer/DB_operations/mongo_neo4j_ops.py index 6cf781c..35f53ab 100644 --- a/discord_analyzer/DB_operations/mongo_neo4j_ops.py +++ b/discord_analyzer/DB_operations/mongo_neo4j_ops.py @@ -91,6 +91,7 @@ def store_analytics_data( guildId=guild_id, community_id=community_id, ) + print(queries_list[0]) self.run_operations_transaction( guildId=guild_id, queries_list=queries_list, diff --git a/discord_analyzer/DB_operations/network_graph.py b/discord_analyzer/DB_operations/network_graph.py index cf131c1..466d653 100644 --- a/discord_analyzer/DB_operations/network_graph.py +++ b/discord_analyzer/DB_operations/network_graph.py @@ -221,7 +221,7 @@ def create_network_query( parameters = { "node_acc_name": node_acc_name, - date_now_timestamp: int(date_now_timestamp), + "date_now_timestamp": int(date_now_timestamp), } query_str = node_str_query + ";" diff --git a/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py b/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py index 14c76b2..bfd8956 100644 --- a/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py +++ b/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py @@ -8,7 +8,7 @@ class NodeStats: - def __init__(self, neo4j_ops: Neo4jOps, threshold: int = 2) -> None: + def __init__(self, threshold: int = 2) -> None: """ initialize the Node status computations object the status could be either one of `Sender`, `Receiver`, `Balanced` @@ -27,12 +27,13 @@ def __init__(self, neo4j_ops: Neo4jOps, threshold: int = 2) -> None: - else it is balanced """ + neo4j_ops = Neo4jOps.get_instance() self.gds = neo4j_ops.gds self.driver = neo4j_ops.neo4j_driver self.threshold = threshold def compute_stats(self, guildId: str, from_start: bool) -> None: - projection_utils = ProjectionUtils(gds=self.gds, guildId=guildId) + projection_utils = ProjectionUtils(guildId=guildId) # possible dates to do the computations possible_dates = projection_utils.get_dates(guildId=guildId) @@ -80,41 +81,46 @@ def compute_node_stats_wrapper( date=date, ) natural_dc = self.gds.run_cypher( - f""" + """ CALL gds.degree.stream( - '{graph_name}', - {{ + $graph_name, + { relationshipWeightProperty: 'weight' - }} + } ) YIELD nodeId, score RETURN gds.util.asNode(nodeId).userId AS userId, score - """ + """, + { + "graph_name": graph_name, + }, ) reverse_dc = self.gds.run_cypher( - f""" + """ CALL gds.degree.stream( - '{graph_name}', - {{ + $graph_name, + { orientation: 'REVERSE', relationshipWeightProperty: 'weight' - }} + } ) YIELD nodeId, score RETURN gds.util.asNode(nodeId).userId AS userId, score - """ + """, + { + "graph_name": graph_name, + }, ) df = self.get_date_stats(natural_dc, reverse_dc, threshold=self.threshold) self.save_properties_db(guildId, df, date) _ = self.gds.run_cypher( - f""" - CALL gds.graph.drop( - "{graph_name}" - ) - """ + "CALL gds.graph.drop($graph_name)", + { + "graph_name": graph_name, + }, ) def get_computed_dates( diff --git a/discord_analyzer/analysis/neo4j_analysis/centrality.py b/discord_analyzer/analysis/neo4j_analysis/centrality.py index 5bf03b1..103897d 100644 --- a/discord_analyzer/analysis/neo4j_analysis/centrality.py +++ b/discord_analyzer/analysis/neo4j_analysis/centrality.py @@ -2,17 +2,17 @@ from typing import Literal import pandas as pd -from discord_analyzer.analysis.neo4j_metrics import Neo4JMetrics +from discord_analyzer.analysis.neo4j_utils.neo4j_metrics import Neo4JMetrics from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils from tc_neo4j_lib.neo4j_ops import Neo4jOps, Query class Centerality: - def __init__(self, neo4j_ops: Neo4jOps) -> None: + def __init__(self) -> None: """ centerality algorithms """ - self.neo4j_ops = neo4j_ops + self.neo4j_ops = Neo4jOps.get_instance() def compute_degree_centerality( self, @@ -31,8 +31,6 @@ def compute_degree_centerality( ------------ guildId : str the user nodes of guildId - gds : GraphDataScience - the gds instance to interact with DB direction : str the direction of relation could be `in_degree`, `out_degree`, `undirected` @@ -98,18 +96,19 @@ def compute_degree_centerality( results = self.neo4j_ops.gds.run_cypher( f""" {query} - WHERE r.guildId = '{guildId}' + WHERE r.guildId = $guild_id RETURN a.userId as a_userId, r.date as date, r.weight as weight, b.userId as b_userId - """ + """, + params={"guild_id": guildId}, ) dates_to_compute = set(results["date"].value_counts().index) if not from_start: - projection_utils = ProjectionUtils(gds=self.neo4j_ops.gds, guildId=guildId) + projection_utils = ProjectionUtils(guildId=guildId) dates_to_compute = self._get_dates_to_compute( projection_utils, dates_to_compute, guildId @@ -317,7 +316,7 @@ def compute_network_decentrality( from_start=from_start, ) - neo4j_metrics = Neo4JMetrics(self.neo4j_ops.gds) + neo4j_metrics = Neo4JMetrics() # saving each date network decentrality network_decentrality: dict[float, float | Literal[-1]] = {} @@ -355,7 +354,11 @@ def save_decentralization_score( MERGE (g) -[r:HAVE_METRICS {date: $date}]-> (g) SET r.decentralizationScore = $score """ - parameters = {"guild_id": guildId, "score": decentrality_score[date]} + parameters = { + "guild_id": guildId, + "score": decentrality_score[date], + "date": date, + } query = Query(query=query_str, parameters=parameters) queries.append(query) diff --git a/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py b/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py index 49fabfd..9fe309d 100644 --- a/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py +++ b/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py @@ -2,12 +2,12 @@ from uuid import uuid1 from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils -from graphdatascience import GraphDataScience +from tc_neo4j_lib import Neo4jOps class LocalClusteringCoeff: - def __init__(self, gds: GraphDataScience) -> None: - self.gds = gds + def __init__(self) -> None: + self.gds = Neo4jOps.get_instance().gds def compute(self, guildId: str, from_start: bool = False) -> None: """ @@ -17,14 +17,8 @@ def compute(self, guildId: str, from_start: bool = False) -> None: Parameters: ------------ - gds : GraphDataScience - the python GraphDataScience instance - neo4j_analytics : Neo4JMetrics object - our written Neo4JMetrics class instance - use_names : bool - whether to add user names to results - if True, the userId will be added alongside nodeId in output - default is False + guildId : str + the guild to compute the analytics for from_start : bool whether to compute the metric from the first day or not if True, then would compute from start @@ -34,7 +28,7 @@ def compute(self, guildId: str, from_start: bool = False) -> None: --------- `None` """ - projection_utils = ProjectionUtils(gds=self.gds, guildId=guildId) + projection_utils = ProjectionUtils(guildId=guildId) # Getting all possible dates computable_dates = projection_utils.get_dates(guildId=guildId) @@ -93,9 +87,12 @@ def local_clustering_computation_wrapper( # dropping the computed date _ = self.gds.run_cypher( - f""" - CALL gds.graph.drop("{graph_projected_name}") """ + CALL gds.graph.drop($graph_projected_name) + """, + { + "graph_projected_name": graph_projected_name, + }, ) def get_computed_dates( @@ -145,17 +142,22 @@ def compute_graph_lcc(self, date: float, graph_name: str, guildId: str) -> None: msg = f"GUILDID: {guildId}" try: _ = self.gds.run_cypher( - f""" + """ CALL gds.localClusteringCoefficient.stream( - "{graph_name}" + $graph_name ) YIELD nodeId, localClusteringCoefficient WITH gds.util.asNode(nodeId) as userNode, localClusteringCoefficient - MATCH (g:Guild {{guildId: '{guildId}'}}) - MERGE (userNode) -[r:INTERACTED_IN {{date: {date}}}]-> (g) + MATCH (g:Guild {guildId: $guild_id}) + MERGE (userNode) -[r:INTERACTED_IN {date: $date}]-> (g) SET r.localClusteringCoefficient = localClusteringCoefficient - """ + """, + { + "graph_name": graph_name, + "guild_id": guildId, + "date": date, + }, ) except Exception as exp: logging.error(f"{msg} error in computing localClusteringCoefficient, {exp}") diff --git a/discord_analyzer/analysis/neo4j_analysis/louvain.py b/discord_analyzer/analysis/neo4j_analysis/louvain.py index 94c8073..ada52ff 100644 --- a/discord_analyzer/analysis/neo4j_analysis/louvain.py +++ b/discord_analyzer/analysis/neo4j_analysis/louvain.py @@ -80,9 +80,12 @@ def louvain_computation_wrapper( # dropping the computed date _ = self.neo4j_ops.gds.run_cypher( - f""" - CALL gds.graph.drop("{graph_projected_name}") """ + CALL gds.graph.drop($graph_projected_name) + """, + { + "graph_projected_name": graph_projected_name, + }, ) def get_computed_dates( @@ -144,9 +147,11 @@ def compute_graph_louvain( }]-> (g) SET r.louvainModularityScore = modularity """, - graph_name=graph_name, - guild_id=guild_id, - date=date, + { + "graph_name": graph_name, + "guild_id": guild_id, + "date": date, + }, ) except Exception as exp: logging.error( diff --git a/discord_analyzer/analysis/neo4j_metrics.py b/discord_analyzer/analysis/neo4j_metrics.py deleted file mode 100644 index 8750476..0000000 --- a/discord_analyzer/analysis/neo4j_metrics.py +++ /dev/null @@ -1,242 +0,0 @@ -import os - -from discord_analyzer.analysis.neo4j_utils.compute_metrics import Neo4JMetrics -from dotenv import load_dotenv -from tc_neo4j_lib.neo4j_ops import Neo4jOps - - -def degree_centrality( - gds, - neo4j_analytics, - use_names=False, - drop_projection=True, - method="stream", - node="DiscordAccount", - relationship="INTERACTED", - relationship_orientation="NATURAL", - parallel_relationship=False, -): - """ - a sample function to show how to compute DegreeCenterality using neo4j_ops - Note: this function does not assume the relation over time - - - Parameters: - ------------ - gds : GraphDataScience - the python GraphDataScience instance - neo4j_analytics : Neo4JMetrics object - our written Neo4JMetrics class instance - use_names : bool - whether to add user names to results - if True, the userId will be added alongside nodeId in output - default is False - drop_projection : bool - drop the graph projection - default is True, which means the graph projections - will be dropped after metric computation - **Note:** Must drop the projection to be able to update results, - make it False if you want do something experimental. - method : str - whether `stream`, `stats`, `Mutate`, or `write`, default is `stream` - each has a special effect on the database, - see: https://neo4j.com/docs/graph-data-science/current/graph-catalog-node-ops/ - node : str - the node name we're computing the degree centrality for - NOTE: Important to have the node exactly like it is saved in DB. - relationship : str - the relationship name we're computing the degree centrality for - relationship_orientation : str - the relationship orientation to be assumed - either `NATURAL`, `REVERSE`, or `UNDIRECTED` - parallel_relationship : bool - whether to assume parallel relationship as one or the real count - if False, then for relationship like A -> B - and B->A the degree centrality of A and B will be 2 - else the degree centrality of A and B will be 1 - - Returns: - --------- - results : pandas dataframe - the results of metrics in pandas dataframe format - """ - - if relationship_orientation not in ["NATURAL", "REVERSE", "UNDIRECTED"]: - msg_prefix = "Wrong relationship orientation given" - msg_prefix += "should be either `NATURAL`, `REVERSE`, or `UNDIRECTED`!" - raise ValueError(f"{msg_prefix} Entered: {relationship_orientation}") - - # compute the total weight of each INTERACTED relationship - gds.run_cypher( - """MATCH (a:DiscordAccount) -[r:INTERACTED]-(:DiscordAccount) - SET r.total_weight= REDUCE(total=0, weight in r.weights | total + weight);""" - ) - - # make the relationship projection configs - relationship_projection = {} - - if parallel_relationship: - relationship_projection[f"{relationship}"] = { - "properties": {"total_weight": {"aggregation": "SUM"}}, - "orientation": f"{relationship_orientation}", - } - else: - relationship_projection[f"{relationship}"] = { - "orientation": f"{relationship_orientation}", - "properties": ["total_weight"], - } - - # first we have to apply the projection (will be saved in server memory) - G, _ = gds.graph.project("MyGraph", node, relationship_projection) - - configuration = None - if method == "write": - configuration = {"relationshipWeightProperty": "total_weight"} - - # get the results as pandas dataframe - results = neo4j_analytics.compute_degreeCenterality( - G, method=method, configuration=configuration - ) - - if use_names: - results["userId"] = results["nodeId"].apply( - lambda nodeId: dict(gds.util.asNode(nodeId))["userId"] - ) - - if drop_projection: - _ = gds.graph.drop(G) - - return results - - -def decenterialization_score(neo4j_analytics, centrality_scores): - """ - a sample function to show how the network decentrality can be computed - - Parameters: - ------------ - neo4j_analytics : Neo4JMetrics object - our written Neo4JMetrics class instance - centrality_scores : array - array of user centrality scores - - Returns: - --------- - network_decentrality : float - the decentrality score of network - """ - network_decentrality = neo4j_analytics.compute_decentralization(centrality_scores) - - return network_decentrality - - -if __name__ == "__main__": - load_dotenv() - - protocol = os.getenv("NEO4J_PROTOCOL") - host = os.getenv("NEO4J_HOST") - port = os.getenv("NEO4J_PORT") - db_name = os.getenv("NEO4J_DB") - - url = f"{protocol}://{host}:{port}" - - user, password = (os.getenv("NEO4J_USER"), os.getenv("NEO4J_PASSWORD")) - - neo4j_ops = Neo4jOps() - neo4j_ops.set_neo4j_db_info(db_name, url, user, password) - neo4j_ops.neo4j_database_connect() - - gds = neo4j_ops.gds - - neo4j_analytics = Neo4JMetrics(gds) - - results_degreeCenterality = degree_centrality( - gds, - neo4j_analytics=neo4j_analytics, - use_names=True, - drop_projection=True, - method="stream", - node="DiscordAccount", - relationship="INTERACTED", - relationship_orientation="UNDIRECTED", - parallel_relationship=True, - ) - - # finding the output relationship counts from a node - results_degreeCentrality_OUT = degree_centrality( - gds, - neo4j_analytics=neo4j_analytics, - use_names=True, - drop_projection=True, - method="stream", - node="DiscordAccount", - relationship="INTERACTED", - relationship_orientation="NATURAL", - # parallel_relationship = True - ) - # finding the input relationship counts to a node - results_degreeCentrality_IN = degree_centrality( - gds, - neo4j_analytics=neo4j_analytics, - use_names=True, - drop_projection=True, - method="stream", - node="DiscordAccount", - relationship="INTERACTED", - relationship_orientation="REVERSE", - # parallel_relationship = True - ) - - # what guilds to find isolated nodes - guildId_arr = ["123456789101112", "993163081939165234", "1012430565959553145"] - results_isolated_discordNodes = neo4j_analytics.compute_isolated_nodes( - guildId=guildId_arr - ) - results_isolation_fraction = neo4j_analytics.compute_isolated_nodes_fraction( - guildId=guildId_arr - ) - results_network_density = neo4j_analytics.compute_network_density( - guildId=guildId_arr - ) - - # adding the scores in and scores out - # to pandas dataframe of `results_degreeCenterality` - results_degreeCenterality["score_in"] = results_degreeCentrality_IN["score"] - results_degreeCenterality["score_out"] = results_degreeCentrality_OUT["score"] - results_degreeCenterality["score_undirected"] = results_degreeCenterality["score"] - - # normalizing undirected scores - results_degreeCenterality["normalized_score_undirected"] = ( - results_degreeCenterality["score"] - / sum(results_degreeCenterality["score"].values > 0) - ) - # the normalization over positive score_out - results_degreeCenterality["normalized_score_out"] = results_degreeCenterality[ - "score_out" - ] / sum(results_degreeCenterality["score_out"].values > 0) - # the normalization over positive score_in - results_degreeCenterality["normalized_score_in"] = results_degreeCenterality[ - "score_in" - ] / sum(results_degreeCenterality["score_in"].values > 0) - - results_decentralityScore = decenterialization_score( - neo4j_analytics=neo4j_analytics, - centrality_scores=results_degreeCenterality[ - "normalized_score_undirected" - ].values, - ) - - print("------------------ Degree Centerality ------------------") - print(results_degreeCenterality, "\n") - - print("------------------ Network Decentrality Score ------------------") - print(results_decentralityScore, "\n") - - print("------------------ Isolated Nodes ------------------") - print(f"Isolated Nodes in guilds: {guildId_arr}") - print(results_isolated_discordNodes, "\n") - print("Isolation fraction: ", results_isolation_fraction, "\n") - - print("------------------ Network Density ------------------") - print(f"Network Density for guilds: {guildId_arr}") - print(results_network_density) diff --git a/discord_analyzer/analysis/neo4j_utils/compute_metrics.py b/discord_analyzer/analysis/neo4j_utils/neo4j_metrics.py similarity index 98% rename from discord_analyzer/analysis/neo4j_utils/compute_metrics.py rename to discord_analyzer/analysis/neo4j_utils/neo4j_metrics.py index 0dea17b..5d2ea89 100644 --- a/discord_analyzer/analysis/neo4j_utils/compute_metrics.py +++ b/discord_analyzer/analysis/neo4j_utils/neo4j_metrics.py @@ -2,11 +2,11 @@ from typing import Literal import numpy as np -from graphdatascience import GraphDataScience +from tc_neo4j_lib.neo4j_ops import Neo4jOps class Neo4JMetrics: - def __init__(self, gds: GraphDataScience) -> None: + def __init__(self) -> None: """ computation of Neo4J metrics @@ -15,7 +15,7 @@ def __init__(self, gds: GraphDataScience) -> None: gds : GraphDataScience the GraphDataScience instance to query the DB """ - self.gds = gds + self.gds = Neo4jOps.get_instance().gds def compute_degreeCenterality(self, graphProjection, method, configuration=None): """ diff --git a/discord_analyzer/analyzer/neo4j_analytics.py b/discord_analyzer/analyzer/neo4j_analytics.py index 3063662..c2b0540 100644 --- a/discord_analyzer/analyzer/neo4j_analytics.py +++ b/discord_analyzer/analyzer/neo4j_analytics.py @@ -61,7 +61,7 @@ def compute_local_clustering_coefficient( try: # Local Clustering Coefficient logging.info(f"{msg} Computing LocalClusteringCoefficient") - lcc = LocalClusteringCoeff(gds=self.neo4j_ops.gds) + lcc = LocalClusteringCoeff() lcc.compute(guildId=guildId, from_start=from_start) except Exception as exp: logging.error( diff --git a/discord_analyzer/analyzer/utils/analyzer_db_manager.py b/discord_analyzer/analyzer/utils/analyzer_db_manager.py index 53abee6..443a2db 100644 --- a/discord_analyzer/analyzer/utils/analyzer_db_manager.py +++ b/discord_analyzer/analyzer/utils/analyzer_db_manager.py @@ -27,29 +27,6 @@ def set_mongo_database_info( self.connection_str = f"mongodb://{self.mongo_user}:{self.mongo_pass}@{self.mongo_host}:{self.mongo_port}" - def set_neo4j_database_info(self, neo4j_creds: dict[str, Any]): - """ - set neo4J database informations - - Parameters: - ------------- - neo4j_creds : dict[str, Any] - neo4j_credentials to connect - the keys should be - - db_name: str - - protocol: str - - host: str - - port: int - - user: str - - password: str - """ - self.neo4j_db_name = neo4j_creds["db_name"] - self.neo4j_protocol = neo4j_creds["protocol"] - self.neo4j_host = neo4j_creds["host"] - self.neo4j_port = neo4j_creds["port"] - self.neo4j_user = neo4j_creds["user"] - self.neo4j_password = neo4j_creds["password"] - def database_connect(self): """ Connect to the database diff --git a/discord_analyzer/rn_analyzer.py b/discord_analyzer/rn_analyzer.py index 9a76629..82a6827 100644 --- a/discord_analyzer/rn_analyzer.py +++ b/discord_analyzer/rn_analyzer.py @@ -22,17 +22,11 @@ def __init__(self, guild_id: str, testing=False): logging.getLogger().setLevel(logging.INFO) self.testing = testing + self.neo4j_analytics = Neo4JAnalytics() self.guild_object = Guild(guild_id) self.guild_id = guild_id self.community_id = self.guild_object.get_community_id() - def setup_neo4j_metrics(self) -> None: - """ - setup the neo4j analytics wrapper - """ - - self.neo4j_analytics = Neo4JAnalytics() - def run_once(self): """Run analysis once (Wrapper)""" # check if the guild was available diff --git a/tests/integration/test_assess_engagement_mention.py b/tests/integration/test_assess_engagement_mention.py index e97c35c..7c7ca3e 100644 --- a/tests/integration/test_assess_engagement_mention.py +++ b/tests/integration/test_assess_engagement_mention.py @@ -5,7 +5,7 @@ from discord_analyzer.analyzer.analyzer_heatmaps import Heatmaps from discord_analyzer.analyzer.utils.analyzer_db_manager import AnalyzerDBManager from tc_core_analyzer_lib.utils.activity import DiscordActivity -from utils.credentials import get_mongo_credentials, get_neo4j_credentials +from utils.credentials import get_mongo_credentials from .utils.analyzer_setup import launch_db_access from .utils.remove_and_setup_guild import setup_db_guild @@ -26,8 +26,6 @@ def create_db_connections(self): mongo_db_host=mongo_creds["host"], mongo_db_port=mongo_creds["port"], ) - neo4j_creds = get_neo4j_credentials() - base_analyzer.set_neo4j_database_info(neo4j_creds) base_analyzer.database_connect() self.db_connections = base_analyzer.DB_connections diff --git a/tests/integration/test_assess_engagement_reactions.py b/tests/integration/test_assess_engagement_reactions.py index cc79e92..318ca3d 100644 --- a/tests/integration/test_assess_engagement_reactions.py +++ b/tests/integration/test_assess_engagement_reactions.py @@ -5,7 +5,7 @@ from discord_analyzer.analyzer.analyzer_heatmaps import Heatmaps from discord_analyzer.analyzer.utils.analyzer_db_manager import AnalyzerDBManager from tc_core_analyzer_lib.utils.activity import DiscordActivity -from utils.credentials import get_mongo_credentials, get_neo4j_credentials +from utils.credentials import get_mongo_credentials from .utils.analyzer_setup import launch_db_access from .utils.remove_and_setup_guild import setup_db_guild @@ -26,8 +26,6 @@ def create_db_connections(self): mongo_db_host=mongo_creds["host"], mongo_db_port=mongo_creds["port"], ) - neo4j_creds = get_neo4j_credentials() - base_analyzer.set_neo4j_database_info(neo4j_creds) base_analyzer.database_connect() self.db_connections = base_analyzer.DB_connections diff --git a/tests/integration/test_assess_engagement_replies.py b/tests/integration/test_assess_engagement_replies.py index d52c00a..c971085 100644 --- a/tests/integration/test_assess_engagement_replies.py +++ b/tests/integration/test_assess_engagement_replies.py @@ -5,7 +5,7 @@ from discord_analyzer.analyzer.analyzer_heatmaps import Heatmaps from discord_analyzer.analyzer.utils.analyzer_db_manager import AnalyzerDBManager from tc_core_analyzer_lib.utils.activity import DiscordActivity -from utils.credentials import get_mongo_credentials, get_neo4j_credentials +from utils.credentials import get_mongo_credentials from .utils.analyzer_setup import launch_db_access from .utils.remove_and_setup_guild import setup_db_guild @@ -26,8 +26,6 @@ def create_db_connections(self): mongo_db_host=mongo_creds["host"], mongo_db_port=mongo_creds["port"], ) - neo4j_creds = get_neo4j_credentials() - base_analyzer.set_neo4j_database_info(neo4j_creds) base_analyzer.database_connect() self.db_connections = base_analyzer.DB_connections diff --git a/tests/integration/test_decentralization_score.py b/tests/integration/test_decentralization_score.py index 25b4732..9a49f55 100644 --- a/tests/integration/test_decentralization_score.py +++ b/tests/integration/test_decentralization_score.py @@ -14,7 +14,7 @@ def test_decentralization_score(): guildId = "1234" neo4j_ops = Neo4jOps.get_instance() - centrality = Centerality(neo4j_ops) + centrality = Centerality() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_degree_centrality_multiple_guilds.py b/tests/integration/test_degree_centrality_multiple_guilds.py index 141b93b..e819ae0 100644 --- a/tests/integration/test_degree_centrality_multiple_guilds.py +++ b/tests/integration/test_degree_centrality_multiple_guilds.py @@ -75,7 +75,7 @@ def test_multiple_guilds(): SET r14.guildId = '{guildId2}' """ ) - centrality = Centerality(neo4j_ops) + centrality = Centerality() degree_centrality = centrality.compute_degree_centerality( guildId=guildId2, direction="undirected", diff --git a/tests/integration/test_degree_centrality_multiple_guilds_preserve_parallel.py b/tests/integration/test_degree_centrality_multiple_guilds_preserve_parallel.py index 0434914..1955e7f 100644 --- a/tests/integration/test_degree_centrality_multiple_guilds_preserve_parallel.py +++ b/tests/integration/test_degree_centrality_multiple_guilds_preserve_parallel.py @@ -17,7 +17,7 @@ def test_multiple_guilds_preserve_parallel(): guildId = "1234" neo4j_ops = Neo4jOps.get_instance() - centrality = Centerality(neo4j_ops) + centrality = Centerality() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_degree_centrality_parallel_preservation.py b/tests/integration/test_degree_centrality_parallel_preservation.py index 1c3e0e9..b43e09b 100644 --- a/tests/integration/test_degree_centrality_parallel_preservation.py +++ b/tests/integration/test_degree_centrality_parallel_preservation.py @@ -63,7 +63,7 @@ def test_partially_connected_coeffs(): SET r12.guildId = '{guildId}' """ ) - centrality = Centerality(neo4j_ops) + centrality = Centerality() degree_centrality = centrality.compute_degree_centerality( guildId=guildId, direction="undirected", diff --git a/tests/integration/test_fragmentation_score.py b/tests/integration/test_fragmentation_score.py index 1708e68..2957c91 100644 --- a/tests/integration/test_fragmentation_score.py +++ b/tests/integration/test_fragmentation_score.py @@ -10,7 +10,7 @@ def test_avg_clustering_coeff(): """ neo4j_ops = Neo4jOps.get_instance() - neo4j_analytics = Neo4JAnalytics(neo4j_ops) + neo4j_analytics = Neo4JAnalytics() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_fragmentation_score_exclude_past.py b/tests/integration/test_fragmentation_score_exclude_past.py index 40ba8f9..0f03a29 100644 --- a/tests/integration/test_fragmentation_score_exclude_past.py +++ b/tests/integration/test_fragmentation_score_exclude_past.py @@ -10,7 +10,7 @@ def test_avg_clustering_exclude_past(): """ neo4j_ops = Neo4jOps.get_instance() - neo4j_analytics = Neo4JAnalytics(neo4j_ops) + neo4j_analytics = Neo4JAnalytics() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_fragmentation_score_from_start.py b/tests/integration/test_fragmentation_score_from_start.py index 404bce0..1f11ca8 100644 --- a/tests/integration/test_fragmentation_score_from_start.py +++ b/tests/integration/test_fragmentation_score_from_start.py @@ -10,7 +10,7 @@ def test_avg_clustering_coeff_from_start(): """ neo4j_ops = Neo4jOps.get_instance() - neo4j_analytics = Neo4JAnalytics(neo4j_ops) + neo4j_analytics = Neo4JAnalytics() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_fragmentation_score_rescaling.py b/tests/integration/test_fragmentation_score_rescaling.py index 607df8f..020f6c6 100644 --- a/tests/integration/test_fragmentation_score_rescaling.py +++ b/tests/integration/test_fragmentation_score_rescaling.py @@ -10,7 +10,7 @@ def test_avg_clustering_coeff_scaling(): """ neo4j_ops = Neo4jOps.get_instance() - neo4j_analytics = Neo4JAnalytics(neo4j_ops) + neo4j_analytics = Neo4JAnalytics() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_interacted_in_deletion.py b/tests/integration/test_interacted_in_deletion.py index df09d1f..9afb481 100644 --- a/tests/integration/test_interacted_in_deletion.py +++ b/tests/integration/test_interacted_in_deletion.py @@ -9,7 +9,7 @@ def test_interacted_in_deletion(): """ neo4j_ops = Neo4jOps.get_instance() - neo4j_analytics = Neo4JAnalytics(neo4j_ops) + neo4j_analytics = Neo4JAnalytics() neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") diff --git a/tests/integration/test_lcc_all_connected.py b/tests/integration/test_lcc_all_connected.py index 0955f91..5805944 100644 --- a/tests/integration/test_lcc_all_connected.py +++ b/tests/integration/test_lcc_all_connected.py @@ -44,7 +44,7 @@ def test_all_connected_coeffs(): SET r4.guildId = '{guildId}' """ ) - lcc = LocalClusteringCoeff(gds=neo4j_ops.gds) + lcc = LocalClusteringCoeff() lcc.compute(guildId=guildId, from_start=True) # getting the results diff --git a/tests/integration/test_lcc_partially_connected.py b/tests/integration/test_lcc_partially_connected.py index 5cd81e7..11da6be 100644 --- a/tests/integration/test_lcc_partially_connected.py +++ b/tests/integration/test_lcc_partially_connected.py @@ -63,7 +63,7 @@ def test_partially_connected_coeffs(): SET r12.guildId = '{guildId}' """ ) - lcc = LocalClusteringCoeff(gds=neo4j_ops.gds) + lcc = LocalClusteringCoeff() lcc.compute(guildId=guildId) # getting the results diff --git a/tests/integration/test_louvain_algorithm_computation.py b/tests/integration/test_louvain_algorithm_computation.py index 94141d3..c8018bb 100644 --- a/tests/integration/test_louvain_algorithm_computation.py +++ b/tests/integration/test_louvain_algorithm_computation.py @@ -35,7 +35,7 @@ def test_louvain_algorithm_available_data(): SET r4.guildId = '{guild_id}' """ ) - louvain = Louvain(neo4j_ops) + louvain = Louvain() louvain.compute(guild_id=guild_id, from_start=False) @@ -104,7 +104,7 @@ def test_louvain_algorithm_more_available_data(): SET r12.guildId = '{guild_id}' """ ) - louvain = Louvain(neo4j_ops) + louvain = Louvain() louvain.compute(guild_id=guild_id, from_start=False) diff --git a/tests/integration/test_louvain_algorithm_get_computed_dates.py b/tests/integration/test_louvain_algorithm_get_computed_dates.py index c1c6cbe..8e251bf 100644 --- a/tests/integration/test_louvain_algorithm_get_computed_dates.py +++ b/tests/integration/test_louvain_algorithm_get_computed_dates.py @@ -36,10 +36,8 @@ def test_louvain_get_computed_dates_empty_data(): SET r4.guildId = '{guild_id}' """ ) - louvain = Louvain(neo4j_ops) - projection_utils = ProjectionUtils(neo4j_ops.gds, guildId=guild_id) - - projection_utils = ProjectionUtils(gds=neo4j_ops.gds, guildId=guild_id) + louvain = Louvain() + projection_utils = ProjectionUtils(guildId=guild_id) computed_dates = louvain.get_computed_dates(projection_utils, guildId=guild_id) @@ -79,10 +77,8 @@ def test_louvain_get_computed_dates_empty_data_with_have_metrics_relation(): SET r4.guildId = '{guild_id}' """ ) - louvain = Louvain(neo4j_ops) - projection_utils = ProjectionUtils(neo4j_ops.gds, guildId=guild_id) - - projection_utils = ProjectionUtils(gds=neo4j_ops.gds, guildId=guild_id) + louvain = Louvain() + projection_utils = ProjectionUtils(guildId=guild_id) computed_dates = louvain.get_computed_dates(projection_utils, guildId=guild_id) @@ -122,10 +118,8 @@ def test_louvain_get_computed_dates_one_data(): SET r4.guildId = '{guild_id}' """ ) - louvain = Louvain(neo4j_ops) - projection_utils = ProjectionUtils(neo4j_ops.gds, guildId=guild_id) - - projection_utils = ProjectionUtils(gds=neo4j_ops.gds, guildId=guild_id) + louvain = Louvain() + projection_utils = ProjectionUtils(guildId=guild_id) computed_dates = louvain.get_computed_dates(projection_utils, guildId=guild_id) diff --git a/tests/integration/test_neo4j_compute_metrics.py b/tests/integration/test_neo4j_compute_metrics.py index 62e4c8b..f22f0e3 100644 --- a/tests/integration/test_neo4j_compute_metrics.py +++ b/tests/integration/test_neo4j_compute_metrics.py @@ -62,7 +62,7 @@ def test_guild_results_available(): """ ) - analytics = Neo4JAnalytics(neo4j_ops) + analytics = Neo4JAnalytics() analytics.compute_metrics(guildId=guildId, from_start=False) diff --git a/tests/integration/test_neo4j_compute_metrics_from_start.py b/tests/integration/test_neo4j_compute_metrics_from_start.py index 1233232..5fb92b0 100644 --- a/tests/integration/test_neo4j_compute_metrics_from_start.py +++ b/tests/integration/test_neo4j_compute_metrics_from_start.py @@ -62,7 +62,7 @@ def test_neo4j_compute_metrics_from_start(): """ ) - analytics = Neo4JAnalytics(neo4j_ops) + analytics = Neo4JAnalytics() analytics.compute_metrics(guildId=guildId, from_start=True) diff --git a/tests/integration/test_neo4j_projection_utils_computed_dates.py b/tests/integration/test_neo4j_projection_utils_computed_dates.py index 9804859..bd5a754 100644 --- a/tests/integration/test_neo4j_projection_utils_computed_dates.py +++ b/tests/integration/test_neo4j_projection_utils_computed_dates.py @@ -1,6 +1,6 @@ from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils -# from tc_neo4j_lib.neo4j_ops import Neo4jOps +from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_neo4j_projection_utils_get_computed_dates(): diff --git a/tests/integration/test_node_stats.py b/tests/integration/test_node_stats.py index 5bda02b..5c82840 100644 --- a/tests/integration/test_node_stats.py +++ b/tests/integration/test_node_stats.py @@ -62,7 +62,7 @@ def test_node_stats(): """ ) - node_stats = NodeStats(neo4j_ops, threshold=2) + node_stats = NodeStats(threshold=2) node_stats.compute_stats(guildId="1234", from_start=True) # getting the results diff --git a/tests/integration/utils/analyzer_setup.py b/tests/integration/utils/analyzer_setup.py index f7d3d09..c634549 100644 --- a/tests/integration/utils/analyzer_setup.py +++ b/tests/integration/utils/analyzer_setup.py @@ -17,14 +17,6 @@ def setup_analyzer( host = os.getenv("MONGODB_HOST", "") port = os.getenv("MONGODB_PORT", "") - neo4j_creds = {} - neo4j_creds["db_name"] = os.getenv("NEO4J_DB", "") - neo4j_creds["protocol"] = os.getenv("NEO4J_PROTOCOL", "") - neo4j_creds["host"] = os.getenv("NEO4J_HOST", "") - neo4j_creds["port"] = os.getenv("NEO4J_PORT", "") - neo4j_creds["password"] = os.getenv("NEO4J_PASSWORD", "") - neo4j_creds["user"] = os.getenv("NEO4J_USER", "") - analyzer.set_mongo_database_info( mongo_db_host=host, mongo_db_password=password, @@ -32,9 +24,7 @@ def setup_analyzer( mongo_db_port=port, ) - analyzer.set_neo4j_database_info(neo4j_creds=neo4j_creds) analyzer.database_connect() - analyzer.setup_neo4j_metrics() return analyzer diff --git a/tests/integration/utils/mock_graph.py b/tests/integration/utils/mock_graph.py index 2473db0..27c531b 100644 --- a/tests/integration/utils/mock_graph.py +++ b/tests/integration/utils/mock_graph.py @@ -72,14 +72,6 @@ def store_mock_data_in_neo4j(graph_dict, guildId, community_id): host = os.getenv("MONGODB_HOST") port = os.getenv("MONGODB_PORT") - neo4j_creds = {} - neo4j_creds["db_name"] = os.getenv("NEO4J_DB") - neo4j_creds["protocol"] = os.getenv("NEO4J_PROTOCOL") - neo4j_creds["host"] = os.getenv("NEO4J_HOST") - neo4j_creds["port"] = os.getenv("NEO4J_PORT") - neo4j_creds["password"] = os.getenv("NEO4J_PASSWORD") - neo4j_creds["user"] = os.getenv("NEO4J_USER") - analyzer = RnDaoAnalyzer(guildId) analyzer.set_mongo_database_info( @@ -88,7 +80,6 @@ def store_mock_data_in_neo4j(graph_dict, guildId, community_id): mongo_db_user=user, mongo_db_port=port, ) - analyzer.set_neo4j_database_info(neo4j_creds=neo4j_creds) analyzer.database_connect() guilds_data = {} diff --git a/tests/unit/test_creds_loading.py b/tests/unit/test_creds_loading.py index 745b1c0..2d35349 100644 --- a/tests/unit/test_creds_loading.py +++ b/tests/unit/test_creds_loading.py @@ -1,6 +1,5 @@ from utils.credentials import ( get_mongo_credentials, - get_neo4j_credentials, get_rabbit_mq_credentials, get_redis_credentials, get_sentryio_service_creds, @@ -50,30 +49,6 @@ def test_rabbit_creds_values(): assert rabbit_creds["username"] is not None -def test_no4j_creds_keys(): - neo4j_creds = get_neo4j_credentials() - - credential_keys = list(neo4j_creds.keys()) - - assert "user" in credential_keys - assert "password" in credential_keys - assert "db_name" in credential_keys - assert "protocol" in credential_keys - assert "port" in credential_keys - assert "host" in credential_keys - - -def test_neo4j_creds_values(): - neo4j_creds = get_neo4j_credentials() - - assert neo4j_creds["user"] is not None - assert neo4j_creds["password"] is not None - assert neo4j_creds["protocol"] is not None - assert neo4j_creds["port"] is not None - assert neo4j_creds["db_name"] is not None - assert neo4j_creds["host"] is not None - - def test_redis_creds_keys(): redis_creds = get_redis_credentials() diff --git a/utils/credentials.py b/utils/credentials.py index 74b1d0a..3cb782c 100644 --- a/utils/credentials.py +++ b/utils/credentials.py @@ -64,34 +64,6 @@ def get_mongo_credentials(): return mongo_creds -def get_neo4j_credentials(): - """ - load neo4j credentials from .env - - Returns: - --------- - neo4j_creds : dict[str, Any] - neo4j credentials - a dictionary representive of - `user` : str - `pass` : str - `db_name` : str - `url` : str - """ - - load_dotenv() - - neo4j_creds = {} - neo4j_creds["db_name"] = os.getenv("NEO4J_DB") - neo4j_creds["protocol"] = os.getenv("NEO4J_PROTOCOL") - neo4j_creds["host"] = os.getenv("NEO4J_HOST") - neo4j_creds["port"] = os.getenv("NEO4J_PORT") - neo4j_creds["password"] = os.getenv("NEO4J_PASSWORD") - neo4j_creds["user"] = os.getenv("NEO4J_USER") - - return neo4j_creds - - def get_sentryio_service_creds(): load_dotenv() From f1ca0d68e859aa72e877e608c2a421abe08ce2ae Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 23 May 2024 10:22:20 +0330 Subject: [PATCH 4/5] fix: increase neo4j lib version! --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 26dc353..4389aba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,6 +23,6 @@ sentry-sdk rq redis tc-core-analyzer-lib==1.3.1 -tc-neo4j-lib==1.0.2 +tc-neo4j-lib==2.0.0 pybars3 backoff==2.2.1 From a83c817bc6e7fd46f254d3352908c3478b3da298 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 23 May 2024 10:45:02 +0330 Subject: [PATCH 5/5] fix: lint issues! --- .../analysis/neo4j_utils/projection_utils.py | 1 + .../analyzer/utils/analyzer_db_manager.py | 2 -- ..._generated_graph_period_1_year_run_once.py | 2 +- .../test_generated_graph_period_1year.py | 2 +- .../test_generated_graph_period_35_days.py | 3 +- ...generated_graph_period_35_days_run_once.py | 3 +- .../test_generated_graph_period_3_months.py | 3 +- ...enerated_graph_period_3_months_run_once.py | 3 +- .../test_generated_graph_period_6_months.py | 3 +- ...enerated_graph_period_6_months_run_once.py | 3 +- .../test_interacted_in_deletion.py | 1 - tests/integration/test_lcc_all_connected.py | 1 - .../test_lcc_partially_connected.py | 1 - .../test_louvain_algorithm_computation.py | 1 - ...st_louvain_algorithm_get_computed_dates.py | 1 - .../integration/test_neo4j_compute_metrics.py | 1 - .../test_neo4j_compute_metrics_from_start.py | 1 - ...t_neo4j_projection_utils_computed_dates.py | 1 - .../test_network_graph_creation.py | 2 +- tests/integration/test_node_stats.py | 1 - tests/integration/utils/neo4j_conn.py | 28 ------------------- 21 files changed, 10 insertions(+), 54 deletions(-) delete mode 100644 tests/integration/utils/neo4j_conn.py diff --git a/discord_analyzer/analysis/neo4j_utils/projection_utils.py b/discord_analyzer/analysis/neo4j_utils/projection_utils.py index 3201071..b67e7c8 100644 --- a/discord_analyzer/analysis/neo4j_utils/projection_utils.py +++ b/discord_analyzer/analysis/neo4j_utils/projection_utils.py @@ -1,4 +1,5 @@ import logging + from tc_neo4j_lib.neo4j_ops import Neo4jOps diff --git a/discord_analyzer/analyzer/utils/analyzer_db_manager.py b/discord_analyzer/analyzer/utils/analyzer_db_manager.py index 443a2db..190d3d7 100644 --- a/discord_analyzer/analyzer/utils/analyzer_db_manager.py +++ b/discord_analyzer/analyzer/utils/analyzer_db_manager.py @@ -1,5 +1,3 @@ -from typing import Any - from discord_analyzer.DB_operations.mongo_neo4j_ops import MongoNeo4jDB diff --git a/tests/integration/test_generated_graph_period_1_year_run_once.py b/tests/integration/test_generated_graph_period_1_year_run_once.py index 8ba7556..31fdc2e 100644 --- a/tests/integration/test_generated_graph_period_1_year_run_once.py +++ b/tests/integration/test_generated_graph_period_1_year_run_once.py @@ -1,11 +1,11 @@ from datetime import datetime, timedelta, timezone import numpy as np +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data -from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild diff --git a/tests/integration/test_generated_graph_period_1year.py b/tests/integration/test_generated_graph_period_1year.py index 3404075..bfe2911 100644 --- a/tests/integration/test_generated_graph_period_1year.py +++ b/tests/integration/test_generated_graph_period_1year.py @@ -1,11 +1,11 @@ from datetime import datetime, timedelta, timezone import numpy as np +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data -from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild diff --git a/tests/integration/test_generated_graph_period_35_days.py b/tests/integration/test_generated_graph_period_35_days.py index 0d20b41..1878ba3 100644 --- a/tests/integration/test_generated_graph_period_35_days.py +++ b/tests/integration/test_generated_graph_period_35_days.py @@ -1,12 +1,11 @@ from datetime import datetime, timedelta, timezone import numpy as np +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data - -from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild diff --git a/tests/integration/test_generated_graph_period_35_days_run_once.py b/tests/integration/test_generated_graph_period_35_days_run_once.py index 8e06333..ed7109e 100644 --- a/tests/integration/test_generated_graph_period_35_days_run_once.py +++ b/tests/integration/test_generated_graph_period_35_days_run_once.py @@ -1,12 +1,11 @@ from datetime import datetime, timedelta, timezone import numpy as np +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data - -from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild diff --git a/tests/integration/test_generated_graph_period_3_months.py b/tests/integration/test_generated_graph_period_3_months.py index ddae3dc..365f55f 100644 --- a/tests/integration/test_generated_graph_period_3_months.py +++ b/tests/integration/test_generated_graph_period_3_months.py @@ -1,12 +1,11 @@ from datetime import datetime, timedelta, timezone import numpy as np +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data - -from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild diff --git a/tests/integration/test_generated_graph_period_3_months_run_once.py b/tests/integration/test_generated_graph_period_3_months_run_once.py index 9d71b55..34764fd 100644 --- a/tests/integration/test_generated_graph_period_3_months_run_once.py +++ b/tests/integration/test_generated_graph_period_3_months_run_once.py @@ -1,12 +1,11 @@ from datetime import datetime, timedelta, timezone import numpy as np +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data - -from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild diff --git a/tests/integration/test_generated_graph_period_6_months.py b/tests/integration/test_generated_graph_period_6_months.py index 834f615..3434bda 100644 --- a/tests/integration/test_generated_graph_period_6_months.py +++ b/tests/integration/test_generated_graph_period_6_months.py @@ -1,12 +1,11 @@ from datetime import datetime, timedelta, timezone import numpy as np +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data - -from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild diff --git a/tests/integration/test_generated_graph_period_6_months_run_once.py b/tests/integration/test_generated_graph_period_6_months_run_once.py index a655fac..4865eeb 100644 --- a/tests/integration/test_generated_graph_period_6_months_run_once.py +++ b/tests/integration/test_generated_graph_period_6_months_run_once.py @@ -1,12 +1,11 @@ from datetime import datetime, timedelta, timezone import numpy as np +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.analyzer_setup import launch_db_access, setup_analyzer from .utils.mock_heatmaps import create_empty_heatmaps_data from .utils.mock_memberactivities import create_empty_memberactivities_data - -from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.remove_and_setup_guild import setup_db_guild diff --git a/tests/integration/test_interacted_in_deletion.py b/tests/integration/test_interacted_in_deletion.py index 9afb481..04b7d49 100644 --- a/tests/integration/test_interacted_in_deletion.py +++ b/tests/integration/test_interacted_in_deletion.py @@ -1,5 +1,4 @@ from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics - from tc_neo4j_lib.neo4j_ops import Neo4jOps diff --git a/tests/integration/test_lcc_all_connected.py b/tests/integration/test_lcc_all_connected.py index 5805944..9c24599 100644 --- a/tests/integration/test_lcc_all_connected.py +++ b/tests/integration/test_lcc_all_connected.py @@ -2,7 +2,6 @@ from discord_analyzer.analysis.neo4j_analysis.local_clustering_coefficient import ( LocalClusteringCoeff, ) - from tc_neo4j_lib.neo4j_ops import Neo4jOps diff --git a/tests/integration/test_lcc_partially_connected.py b/tests/integration/test_lcc_partially_connected.py index 11da6be..73e6b5b 100644 --- a/tests/integration/test_lcc_partially_connected.py +++ b/tests/integration/test_lcc_partially_connected.py @@ -2,7 +2,6 @@ from discord_analyzer.analysis.neo4j_analysis.local_clustering_coefficient import ( LocalClusteringCoeff, ) - from tc_neo4j_lib.neo4j_ops import Neo4jOps diff --git a/tests/integration/test_louvain_algorithm_computation.py b/tests/integration/test_louvain_algorithm_computation.py index c8018bb..2f6bef0 100644 --- a/tests/integration/test_louvain_algorithm_computation.py +++ b/tests/integration/test_louvain_algorithm_computation.py @@ -1,5 +1,4 @@ from discord_analyzer.analysis.neo4j_analysis.louvain import Louvain - from tc_neo4j_lib.neo4j_ops import Neo4jOps diff --git a/tests/integration/test_louvain_algorithm_get_computed_dates.py b/tests/integration/test_louvain_algorithm_get_computed_dates.py index 8e251bf..01114c2 100644 --- a/tests/integration/test_louvain_algorithm_get_computed_dates.py +++ b/tests/integration/test_louvain_algorithm_get_computed_dates.py @@ -1,6 +1,5 @@ from discord_analyzer.analysis.neo4j_analysis.louvain import Louvain from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils - from tc_neo4j_lib.neo4j_ops import Neo4jOps diff --git a/tests/integration/test_neo4j_compute_metrics.py b/tests/integration/test_neo4j_compute_metrics.py index f22f0e3..0cd203c 100644 --- a/tests/integration/test_neo4j_compute_metrics.py +++ b/tests/integration/test_neo4j_compute_metrics.py @@ -1,6 +1,5 @@ import numpy as np from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics - from tc_neo4j_lib.neo4j_ops import Neo4jOps diff --git a/tests/integration/test_neo4j_compute_metrics_from_start.py b/tests/integration/test_neo4j_compute_metrics_from_start.py index 5fb92b0..24b1faf 100644 --- a/tests/integration/test_neo4j_compute_metrics_from_start.py +++ b/tests/integration/test_neo4j_compute_metrics_from_start.py @@ -1,6 +1,5 @@ import numpy as np from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics - from tc_neo4j_lib.neo4j_ops import Neo4jOps diff --git a/tests/integration/test_neo4j_projection_utils_computed_dates.py b/tests/integration/test_neo4j_projection_utils_computed_dates.py index bd5a754..b359e55 100644 --- a/tests/integration/test_neo4j_projection_utils_computed_dates.py +++ b/tests/integration/test_neo4j_projection_utils_computed_dates.py @@ -1,5 +1,4 @@ from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils - from tc_neo4j_lib.neo4j_ops import Neo4jOps diff --git a/tests/integration/test_network_graph_creation.py b/tests/integration/test_network_graph_creation.py index 6464282..312404c 100644 --- a/tests/integration/test_network_graph_creation.py +++ b/tests/integration/test_network_graph_creation.py @@ -4,9 +4,9 @@ import networkx as nx import numpy as np from discord_analyzer.analysis.utils.activity import Activity +from tc_neo4j_lib.neo4j_ops import Neo4jOps from .utils.mock_graph import generate_mock_graph, store_mock_data_in_neo4j -from tc_neo4j_lib.neo4j_ops import Neo4jOps def test_network_graph_create(): diff --git a/tests/integration/test_node_stats.py b/tests/integration/test_node_stats.py index 5c82840..bcbf9cc 100644 --- a/tests/integration/test_node_stats.py +++ b/tests/integration/test_node_stats.py @@ -1,6 +1,5 @@ # test out local clustering coefficient with all nodes connected from discord_analyzer.analysis.neo4j_analysis.analyzer_node_stats import NodeStats - from tc_neo4j_lib.neo4j_ops import Neo4jOps diff --git a/tests/integration/utils/neo4j_conn.py b/tests/integration/utils/neo4j_conn.py deleted file mode 100644 index 7a7c163..0000000 --- a/tests/integration/utils/neo4j_conn.py +++ /dev/null @@ -1,28 +0,0 @@ -import os - -from dotenv import load_dotenv - - -def neo4j_setup() -> Neo4jOps: - load_dotenv() - - protocol = os.getenv("NEO4J_PROTOCOL") - host = os.getenv("NEO4J_HOST") - port = os.getenv("NEO4J_PORT") - db_name = os.getenv("NEO4J_DB") - - user = os.getenv("NEO4J_USER") - password = os.getenv("NEO4J_PASSWORD") - - neo4j_ops = Neo4jOps() - neo4j_ops.set_neo4j_db_info( - neo4j_db_name=db_name, - neo4j_protocol=protocol, - neo4j_user=user, - neo4j_password=password, - neo4j_host=host, - neo4j_port=port, - ) - neo4j_ops.neo4j_database_connect() - - return neo4j_ops