From 788b0330105cdc1974bb4088cd5a9eea2278985c Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 29 Nov 2023 10:42:52 +0330 Subject: [PATCH 01/22] update: Code update with latest db schema! also, we did some code cleaning with isort and black linters. --- .../DB_operations/mongo_neo4j_ops.py | 8 +- .../DB_operations/mongodb_interaction.py | 3 +- .../compute_interaction_matrix_discord.py | 3 +- .../analysis/compute_member_activity.py | 5 +- .../analysis/member_activity_history.py | 10 ++- .../neo4j_analysis/analyzer_node_stats.py | 3 +- .../analysis/neo4j_analysis/centrality.py | 3 +- .../local_clustering_coefficient.py | 3 +- .../analysis/neo4j_analysis/louvain.py | 3 +- discord_analyzer/analysis/neo4j_metrics.py | 3 +- .../utils/compute_interaction_mtx_utils.py | 1 + .../utils/member_activity_history_utils.py | 3 +- .../analysis/utils/member_activity_utils.py | 5 +- .../analyzer/analyzer_heatmaps.py | 5 +- discord_analyzer/analyzer/heatmaps_utils.py | 3 +- .../analyzer/memberactivity_utils.py | 4 +- discord_analyzer/analyzer/neo4j_analytics.py | 3 +- discord_analyzer/models/BaseModel.py | 4 +- discord_analyzer/models/GuildsRnDaoModel.py | 84 +++++------------- discord_analyzer/models/HeatMapModel.py | 87 ++----------------- .../models/MemberActivityModel.py | 72 +-------------- discord_analyzer/models/RawInfoModel.py | 50 ++--------- discord_analyzer/models/RnDaoModel.py | 14 --- discord_analyzer/rn_analyzer.py | 17 ++-- 24 files changed, 90 insertions(+), 306 deletions(-) delete mode 100644 discord_analyzer/models/RnDaoModel.py diff --git a/discord_analyzer/DB_operations/mongo_neo4j_ops.py b/discord_analyzer/DB_operations/mongo_neo4j_ops.py index ec0a96c..d748f0c 100644 --- a/discord_analyzer/DB_operations/mongo_neo4j_ops.py +++ b/discord_analyzer/DB_operations/mongo_neo4j_ops.py @@ -1,8 +1,9 @@ import logging +from tc_neo4j_lib.neo4j_ops import Neo4jOps + from discord_analyzer.DB_operations.mongodb_interaction import MongoDBOps from discord_analyzer.DB_operations.network_graph import make_neo4j_networkx_query_dict -from tc_neo4j_lib.neo4j_ops import Neo4jOps class MongoNeo4jDB: @@ -51,7 +52,10 @@ def set_mongo_db_ops( self.mongoOps.set_mongo_db_access() def store_analytics_data( - self, analytics_data, remove_memberactivities=False, remove_heatmaps=False + self, + analytics_data: dict, + remove_memberactivities: bool = False, + remove_heatmaps: bool = False, ): """ store the analytics data into database diff --git a/discord_analyzer/DB_operations/mongodb_interaction.py b/discord_analyzer/DB_operations/mongodb_interaction.py index 4783c2f..edeee4c 100644 --- a/discord_analyzer/DB_operations/mongodb_interaction.py +++ b/discord_analyzer/DB_operations/mongodb_interaction.py @@ -1,9 +1,10 @@ import logging -from discord_analyzer.DB_operations.mongodb_access import DB_access from pymongo.read_concern import ReadConcern from pymongo.write_concern import WriteConcern +from discord_analyzer.DB_operations.mongodb_access import DB_access + class MongoDBOps: def __init__(self, user, password, host, port): diff --git a/discord_analyzer/analysis/compute_interaction_matrix_discord.py b/discord_analyzer/analysis/compute_interaction_matrix_discord.py index e5de975..d2200dd 100644 --- a/discord_analyzer/analysis/compute_interaction_matrix_discord.py +++ b/discord_analyzer/analysis/compute_interaction_matrix_discord.py @@ -5,9 +5,10 @@ # # Author Ene SS Rawa / Tjitse van der Molen +from numpy import ndarray + from discord_analyzer.analysis.utils.activity import Activity from discord_analyzer.DB_operations.mongodb_query import MongodbQuery -from numpy import ndarray from .utils.compute_interaction_mtx_utils import ( generate_interaction_matrix, diff --git a/discord_analyzer/analysis/compute_member_activity.py b/discord_analyzer/analysis/compute_member_activity.py index 3e53443..22bdd7c 100644 --- a/discord_analyzer/analysis/compute_member_activity.py +++ b/discord_analyzer/analysis/compute_member_activity.py @@ -10,6 +10,9 @@ import networkx as nx import numpy as np from dateutil.relativedelta import relativedelta +from tc_core_analyzer_lib.assess_engagement import EngagementAssessment +from tc_core_analyzer_lib.utils.activity import DiscordActivity + from discord_analyzer.analysis.compute_interaction_matrix_discord import ( compute_interaction_matrix_discord, ) @@ -26,8 +29,6 @@ update_activities, ) from discord_analyzer.DB_operations.mongodb_access import DB_access -from tc_core_analyzer_lib.assess_engagement import EngagementAssessment -from tc_core_analyzer_lib.utils.activity import DiscordActivity def compute_member_activity( diff --git a/discord_analyzer/analysis/member_activity_history.py b/discord_analyzer/analysis/member_activity_history.py index f50ba53..3663912 100644 --- a/discord_analyzer/analysis/member_activity_history.py +++ b/discord_analyzer/analysis/member_activity_history.py @@ -4,17 +4,19 @@ import datetime from dateutil import parser + from discord_analyzer.analysis.utils.member_activity_history_utils import ( MemberActivityPastUtils, ) +from discord_analyzer.DB_operations.mongodb_access import DB_access # the main script function def check_past_history( - db_access, - date_range, - window_param, - collection_name="memberactivities", + db_access: DB_access, + date_range: list[str], + window_param: tuple[int, int], + collection_name: str = "memberactivities", verbose=False, ): """ diff --git a/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py b/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py index 5741669..b06cec3 100644 --- a/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py +++ b/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py @@ -3,9 +3,10 @@ from uuid import uuid1 import pandas as pd -from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils from tc_neo4j_lib.neo4j_ops import Neo4jOps +from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils + class NodeStats: def __init__(self, neo4j_ops: Neo4jOps, threshold: int = 2) -> None: diff --git a/discord_analyzer/analysis/neo4j_analysis/centrality.py b/discord_analyzer/analysis/neo4j_analysis/centrality.py index bf7bf17..b9623b4 100644 --- a/discord_analyzer/analysis/neo4j_analysis/centrality.py +++ b/discord_analyzer/analysis/neo4j_analysis/centrality.py @@ -2,9 +2,10 @@ from typing import Literal import pandas as pd +from tc_neo4j_lib.neo4j_ops import Neo4jOps + from discord_analyzer.analysis.neo4j_metrics import Neo4JMetrics from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils -from tc_neo4j_lib.neo4j_ops import Neo4jOps class Centerality: diff --git a/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py b/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py index 869cd8f..1d61714 100644 --- a/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py +++ b/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py @@ -1,9 +1,10 @@ import logging from uuid import uuid1 -from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils from graphdatascience import GraphDataScience +from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils + class LocalClusteringCoeff: def __init__(self, gds: GraphDataScience) -> None: diff --git a/discord_analyzer/analysis/neo4j_analysis/louvain.py b/discord_analyzer/analysis/neo4j_analysis/louvain.py index 82eb060..7ac7b40 100644 --- a/discord_analyzer/analysis/neo4j_analysis/louvain.py +++ b/discord_analyzer/analysis/neo4j_analysis/louvain.py @@ -1,9 +1,10 @@ import logging from uuid import uuid1 -from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils from tc_neo4j_lib.neo4j_ops import Neo4jOps +from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils + class Louvain: def __init__(self, neo4j_ops: Neo4jOps) -> None: diff --git a/discord_analyzer/analysis/neo4j_metrics.py b/discord_analyzer/analysis/neo4j_metrics.py index b23d6b3..9b18c6a 100644 --- a/discord_analyzer/analysis/neo4j_metrics.py +++ b/discord_analyzer/analysis/neo4j_metrics.py @@ -1,9 +1,10 @@ import os -from discord_analyzer.analysis.neo4j_utils.compute_metrics import Neo4JMetrics from dotenv import load_dotenv from tc_neo4j_lib.neo4j_ops import Neo4jOps +from discord_analyzer.analysis.neo4j_utils.compute_metrics import Neo4JMetrics + def degree_centrality( gds, diff --git a/discord_analyzer/analysis/utils/compute_interaction_mtx_utils.py b/discord_analyzer/analysis/utils/compute_interaction_mtx_utils.py index f3d8636..9f0c72e 100644 --- a/discord_analyzer/analysis/utils/compute_interaction_mtx_utils.py +++ b/discord_analyzer/analysis/utils/compute_interaction_mtx_utils.py @@ -2,6 +2,7 @@ from typing import Any import numpy as np + from discord_analyzer.analysis.analytics_interactions_script import ( per_account_interactions, ) diff --git a/discord_analyzer/analysis/utils/member_activity_history_utils.py b/discord_analyzer/analysis/utils/member_activity_history_utils.py index 2ce63fe..e602a50 100644 --- a/discord_analyzer/analysis/utils/member_activity_history_utils.py +++ b/discord_analyzer/analysis/utils/member_activity_history_utils.py @@ -3,9 +3,10 @@ from typing import Any from dateutil import parser -from discord_analyzer.DB_operations.mongodb_access import DB_access from numpy import array +from discord_analyzer.DB_operations.mongodb_access import DB_access + class MemberActivityPastUtils: def __init__(self, db_access: DB_access) -> None: diff --git a/discord_analyzer/analysis/utils/member_activity_utils.py b/discord_analyzer/analysis/utils/member_activity_utils.py index d7b1d6a..9f6d5e5 100644 --- a/discord_analyzer/analysis/utils/member_activity_utils.py +++ b/discord_analyzer/analysis/utils/member_activity_utils.py @@ -1,12 +1,13 @@ -from datetime import timedelta +from datetime import datetime, timedelta from typing import Any import numpy as np import pymongo + from discord_analyzer.DB_operations.mongodb_access import DB_access -def get_joined_accounts(db_access, date_range): +def get_joined_accounts(db_access: DB_access, date_range: tuple[datetime, datetime]): """ get the joined accounts for a time interval to a date range diff --git a/discord_analyzer/analyzer/analyzer_heatmaps.py b/discord_analyzer/analyzer/analyzer_heatmaps.py index d39cfeb..2211b59 100644 --- a/discord_analyzer/analyzer/analyzer_heatmaps.py +++ b/discord_analyzer/analyzer/analyzer_heatmaps.py @@ -9,13 +9,14 @@ getNumberOfActions, store_counts_dict, ) +from discord_analyzer.DB_operations.mongo_neo4j_ops import MongoNeo4jDB from discord_analyzer.models.GuildsRnDaoModel import GuildsRnDaoModel from discord_analyzer.models.HeatMapModel import HeatMapModel from discord_analyzer.models.RawInfoModel import RawInfoModel class Heatmaps: - def __init__(self, DB_connections, testing) -> None: + def __init__(self, DB_connections: MongoNeo4jDB, testing: bool) -> None: self.DB_connections = DB_connections self.testing = testing @@ -30,7 +31,7 @@ def is_empty(self, guildId: str): return document is None - def analysis_heatmap(self, guildId, from_start=False): + def analysis_heatmap(self, guildId: str, from_start: bool = False): """ Based on the rawdata creates and stores the heatmap data diff --git a/discord_analyzer/analyzer/heatmaps_utils.py b/discord_analyzer/analyzer/heatmaps_utils.py index 79b1b61..3068754 100644 --- a/discord_analyzer/analyzer/heatmaps_utils.py +++ b/discord_analyzer/analyzer/heatmaps_utils.py @@ -1,6 +1,7 @@ -from discord_analyzer.schemas.accounts import AccountCounts from pymongo import MongoClient +from discord_analyzer.schemas.accounts import AccountCounts + def store_counts_dict(counts_dict): # make empty result array diff --git a/discord_analyzer/analyzer/memberactivity_utils.py b/discord_analyzer/analyzer/memberactivity_utils.py index bbbe6f1..caa8602 100644 --- a/discord_analyzer/analyzer/memberactivity_utils.py +++ b/discord_analyzer/analyzer/memberactivity_utils.py @@ -2,9 +2,11 @@ from dateutil import parser +from discord_analyzer.DB_operations.mongo_neo4j_ops import MongoNeo4jDB + class MemberActivityUtils: - def __init__(self, DB_connection) -> None: + def __init__(self, DB_connection: MongoNeo4jDB) -> None: self.DB_connection = DB_connection def refine_memberactivities_data(self, all_member_activities, first_date): diff --git a/discord_analyzer/analyzer/neo4j_analytics.py b/discord_analyzer/analyzer/neo4j_analytics.py index 3672dae..77d42d8 100644 --- a/discord_analyzer/analyzer/neo4j_analytics.py +++ b/discord_analyzer/analyzer/neo4j_analytics.py @@ -1,13 +1,14 @@ # A wrapper to compute the neo4j metrics in cron-job import logging +from tc_neo4j_lib.neo4j_ops import Neo4jOps + from discord_analyzer.analysis.neo4j_analysis.analyzer_node_stats import NodeStats from discord_analyzer.analysis.neo4j_analysis.centrality import Centerality from discord_analyzer.analysis.neo4j_analysis.local_clustering_coefficient import ( LocalClusteringCoeff, ) from discord_analyzer.analysis.neo4j_analysis.louvain import Louvain -from tc_neo4j_lib.neo4j_ops import Neo4jOps class Neo4JAnalytics: diff --git a/discord_analyzer/models/BaseModel.py b/discord_analyzer/models/BaseModel.py index c15ff8f..d122926 100644 --- a/discord_analyzer/models/BaseModel.py +++ b/discord_analyzer/models/BaseModel.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import logging +from pymongo.database import Database + class BaseModel: """ @@ -8,7 +10,7 @@ class BaseModel: All integrated models inherit from this object """ - def __init__(self, collection_name, database): + def __init__(self, collection_name: str, database: Database): self.collection_name = collection_name self.database = database self.exists = False diff --git a/discord_analyzer/models/GuildsRnDaoModel.py b/discord_analyzer/models/GuildsRnDaoModel.py index 6a6b7c0..f081f7b 100644 --- a/discord_analyzer/models/GuildsRnDaoModel.py +++ b/discord_analyzer/models/GuildsRnDaoModel.py @@ -1,51 +1,48 @@ #!/usr/bin/env python3 -import logging +from bson.objectid import ObjectId +from pymongo.database import Database from discord_analyzer.models.BaseModel import BaseModel class GuildsRnDaoModel(BaseModel): - def __init__(self, database=None): - if database is None: - logging.exception("Database does not exist.") - raise Exception("Database should not be None") - super().__init__(collection_name="guilds", database=database) + def __init__(self, database: Database): + super().__init__(collection_name="platforms", database=database) # print(self.database[self.collection_name].find_one()) - def get_connected_guilds(self, guildId): + def get_connected_guilds(self, guildId: str | None = None): """ Returns the list of the connected guilds if guildId is None Otherwise the list of one connected guild with given guildId """ - findOption = {"isDisconnected": False} + findOption = {"isDisconnected": False, "name": "discord"} if guildId is not None: - findOption["guildId"] = guildId + findOption["_id"] = ObjectId(guildId) guilds = self.database[self.collection_name].find(findOption) - return [x["guildId"] for x in guilds] + return [x["metadata"]["id"] for x in guilds] def get_guild_info(self, guildId): """ Return detailed information of guild settings Return None if such guild is not exist """ - guild = self.database[self.collection_name].find({"guildId": guildId}) - if guild == []: - return None - return guild[0] + guild = self.database[self.collection_name].find_one({"metadata.id": guildId}) + return guild def get_guild_period(self, guildId: str): """ get the period field from guild saved in RnDAO collection """ data = self.database[self.collection_name].find_one( - {"guildId": guildId}, {"period": 1, "_id": 0} + {"metadata.id": guildId, "name": "discord"}, + {"metadata.period": 1, "_id": 0}, ) if data is not None: - return data["period"] + return data["metadata.period"] else: return None - def get_guild_channels(self, guildId): + def get_guild_channels(self, guildId: str) -> list[dict]: """ get the channelSelection from a guild @@ -57,60 +54,23 @@ def get_guild_channels(self, guildId): Returns: ---------- - channels : list of dictionaries + channels : list[dict] a list of dictionaries representative of channelName, channelId, and _id """ - query = {"guildId": f"{guildId}"} - feature_projection = {"selectedChannels": 1, "_id": 0} + query = {"metadata.id": guildId, "name": "discord"} + feature_projection = {"metadata.selectedChannels": 1, "_id": 0} - cursor = self.database[self.collection_name].find( + cursor = self.database[self.collection_name].find_one( query, projection=feature_projection ) - channels = list(cursor) - - # initialize - selected_channels = None + results = list(cursor) - if channels == []: + selected_channels: list[dict] + if results is None: selected_channels = [] else: - selected_channels = channels[0]["selectedChannels"] + selected_channels = results["metadata.selectedChannels"] return selected_channels - - def update_guild_channel_selection(self, guildId, selectedChannels): - """ - Update the channel selection in RnDAO for a specific guild - - Parameters: - ------------ - guildId : str - the guildId to update its channel selection - selectedChannels : dict - a dictionary for the channel selection, - each key values of dictionary must have the followings - `channelId`, `channelName`, and `_id` - example: - {'0': { - 'channelId': '1073334445554337223', - 'channelName': 'sample_channel_name', - '_id': ObjectId('156a84sd1') - }, - '1': {...} - } - - Returns: - ----------- - status : bool - if True, the channel selection is updated - else, the channel selection is not updated - """ - # query to filter the guilds of the RnDAO.guilds table - query = {"guildId": f"{guildId}"} - - update_field_query = {"$set": {"selectedChannels": selectedChannels}} - - # update the selectedChannels - self.database[self.collection_name].update_one(query, update_field_query) diff --git a/discord_analyzer/models/HeatMapModel.py b/discord_analyzer/models/HeatMapModel.py index 3b9f9b9..a0d35d5 100644 --- a/discord_analyzer/models/HeatMapModel.py +++ b/discord_analyzer/models/HeatMapModel.py @@ -1,80 +1,16 @@ #!/usr/bin/env python3 import logging -from datetime import datetime, timedelta, timezone +from datetime import datetime + +from pymongo import DESCENDING +from pymongo.database import Database -import pymongo from discord_analyzer.models.BaseModel import BaseModel class HeatMapModel(BaseModel): - def __init__(self, database=None): - if database is None: - logging.exception("Database does not exist.") - raise Exception("Database should not be None") + def __init__(self, database: Database): super().__init__(collection_name="heatmaps", database=database) - self.validator = { - "$jsonSchema": { - "bsonType": "object", - "properties": { - "date": { - "bsonType": "date", - }, - "channel": { - "bsonType": "string", - }, - "lone_messages": { - "bsonType": "array", - "items": {"bsonType": "int"}, - }, - "thr_messages": {"bsonType": "array", "items": {"bsonType": "int"}}, - "replier": {"bsonType": "array", "items": {"bsonType": "int"}}, - "replier_accounts": { - "bsonType": "array", - "items": { - "bsonType": "object", - "required": ["_id", "account", "count"], - "properties": { - "_id": {"bsonType": "string"}, - "account": {"bsonType": "string"}, - "count": {"bsonType": "int"}, - }, - }, - }, - "replied": {"bsonType": "array", "items": {"bsonType": "int"}}, - "mentioner": {"bsonType": "array", "items": {"bsonType": "int"}}, - "mentioner_accounts": { - "bsonType": "array", - "items": { - "bsonType": "object", - "required": ["_id", "account", "count"], - "properties": { - "_id": {"bsonType": "string"}, - "account": {"bsonType": "string"}, - "count": {"bsonType": "int"}, - }, - }, - }, - "mentioned": {"bsonType": "array", "items": {"bsonType": "int"}}, - "reacter": {"bsonType": "array", "items": {"bsonType": "int"}}, - "reacter_accounts": { - "bsonType": "array", - "items": { - "bsonType": "object", - "required": ["_id", "account", "count"], - "properties": { - "_id": {"bsonType": "string"}, - "account": {"bsonType": "string"}, - "count": {"bsonType": "int"}, - }, - }, - }, - "reacted": {"bsonType": "array", "items": {"bsonType": "int"}}, - "account_name": { - "bsonType": "string", - }, - }, - } - } def get_last_date(self): """ @@ -84,23 +20,12 @@ def get_last_date(self): date_str = ( self.database[self.collection_name] .find() - .sort([("date", pymongo.DESCENDING)]) + .sort([("date", DESCENDING)]) .limit(1)[0]["date"] ) date_obj = datetime.strptime(date_str, "%Y-%m-%d") return date_obj - # Parsing the time and timezone - date_str = date_str.split(" GMT") - date_str[1] = "GMT" + date_str[1] - date_str[1] = date_str[1].split(" ")[0].replace("GMT", "") - zone = [date_str[1][0:3], date_str[1][3::]] - zone_hrs = int(zone[0]) - zone_min = int(zone[1]) - date_obj = datetime.strptime(date_str[0], "%a %b %d %Y %H:%M:%S").replace( - tzinfo=timezone(timedelta(hours=zone_hrs, minutes=zone_min)) - ) - return date_obj except Exception as e: print(e) return None diff --git a/discord_analyzer/models/MemberActivityModel.py b/discord_analyzer/models/MemberActivityModel.py index f4fbf02..26e750c 100644 --- a/discord_analyzer/models/MemberActivityModel.py +++ b/discord_analyzer/models/MemberActivityModel.py @@ -3,6 +3,7 @@ from datetime import datetime import pymongo + from discord_analyzer.models.BaseModel import BaseModel @@ -12,77 +13,6 @@ def __init__(self, database=None): logging.exception("Database does not exist.") raise Exception("Database should not be None") super().__init__(collection_name="memberactivities", database=database) - self.validator = { - "$jsonSchema": { - "bsonType": "object", - "properties": { - "date": { - "bsonType": "date", - }, - "all_active": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "all_consistent": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "all_vital": {"bsonType": "array", "items": {"bsonType": "string"}}, - "all_connected": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "all_paused": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "all_new_disengaged": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "all_disengaged": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "all_unpaused": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "all_returned": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "all_new_active": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "all_still_active": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "all_dropped": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "all_joined": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "all_disengaged_were_newly_active": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "all_disengaged_were_consistenly_active": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "all_disengaged_were_vital": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - }, - } - } def get_last_date(self): """ diff --git a/discord_analyzer/models/RawInfoModel.py b/discord_analyzer/models/RawInfoModel.py index ed29447..5057e29 100644 --- a/discord_analyzer/models/RawInfoModel.py +++ b/discord_analyzer/models/RawInfoModel.py @@ -3,44 +3,16 @@ from datetime import datetime, timedelta from typing import Any -import pymongo +from pymongo import ASCENDING +from pymongo.database import Database + from discord_analyzer.models.BaseModel import BaseModel class RawInfoModel(BaseModel): - def __init__(self, database=None): - if database is None: - logging.info("Database does not exist.") - raise Exception("Database should not be None") + def __init__(self, database: Database): super().__init__(collection_name="rawinfos", database=database) self.guild_msg = f"GUILDID: {self.database.name}:" - self.validator = { - "$jsonSchema": { - "bsonType": "object", - "properties": { - "type": {"bsonType": "string"}, - "author": {"bsonType": "string"}, - "content": {"bsonType": "string"}, - "user_Mentions": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "roles_Mentions": { - "bsonType": "array", - "items": {"bsonType": "string"}, - }, - "reactions": {"bsonType": "array", "items": {"bsonType": "string"}}, - "replied_User": {"bsonType": "string"}, - "reference_Message": {"bsonType": "int"}, - "datetime": { - "bsonType": "string", - }, - "channelId": { - "bsonType": "string", - }, - }, - } - } def get_first_date(self): """ @@ -50,24 +22,14 @@ def get_first_date(self): """ if self.database[self.collection_name].count_documents({}) > 0: record = self.database[self.collection_name].find_one( - {}, sort=[("createdDate", pymongo.ASCENDING)] + {}, sort=[("createdDate", ASCENDING)] ) first_date = record["createdDate"] - - # ( - # self.database[self.collection_name] - # .find() - # .sort([("createdDate", pymongo.ASCENDING)]) - # .limit(1)[0]["createdDate"] - # ) - # date_obj = datetime.strptime(first_date, "%Y-%m-%d %H:%M:%S") - return first_date - # do something with the first document else: # handle the case where no documents are returned by the query - print(f"{self.guild_msg} No documents found in the collection") + logging.info(f"{self.guild_msg} No documents found in the collection") return None def get_day_entries(self, day: datetime, msg: str = "") -> list[dict[str, Any]]: diff --git a/discord_analyzer/models/RnDaoModel.py b/discord_analyzer/models/RnDaoModel.py deleted file mode 100644 index 898e30c..0000000 --- a/discord_analyzer/models/RnDaoModel.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 - - -import logging - -from discord_analyzer.models.BaseModel import BaseModel - - -class RnDaoModel(BaseModel): - def __init__(self, database=None): - if database is None: - logging.exception("Database does not exist.") - raise Exception("Database should not be None") - super().__init__(collection_name="RnDAO", database=database) diff --git a/discord_analyzer/rn_analyzer.py b/discord_analyzer/rn_analyzer.py index e857c11..7eb09dc 100644 --- a/discord_analyzer/rn_analyzer.py +++ b/discord_analyzer/rn_analyzer.py @@ -3,6 +3,8 @@ import os import sys +from dotenv import load_dotenv + from discord_analyzer.analyzer.analyzer_heatmaps import Heatmaps from discord_analyzer.analyzer.analyzer_memberactivities import Member_activities from discord_analyzer.analyzer.base_analyzer import Base_analyzer @@ -10,7 +12,6 @@ from discord_analyzer.models.GuildsRnDaoModel import GuildsRnDaoModel from discord_analyzer.models.HeatMapModel import HeatMapModel from discord_analyzer.models.RawInfoModel import RawInfoModel -from dotenv import load_dotenv class RnDaoAnalyzer(Base_analyzer): @@ -39,7 +40,7 @@ def run_once(self, guildId): """Run analysis once (Wrapper)""" guilds_c = GuildsRnDaoModel( - self.DB_connections.mongoOps.mongo_db_access.db_mongo_client["RnDAO"] + self.DB_connections.mongoOps.mongo_db_access.db_mongo_client["Core"] ) guilds = guilds_c.get_connected_guilds(guildId) @@ -124,7 +125,7 @@ def recompute_analytics_on_guilds(self, guildId_list): client = self.DB_connections.mongoOps.mongo_db_access.db_mongo_client # check if the guild was available in RnDAO table - guilds_c = GuildsRnDaoModel(client["RnDAO"]) + guilds_c = GuildsRnDaoModel(client["Core"]) guilds = guilds_c.get_connected_guilds(guildId_list) logging.info(f"Recomputing analytics for {guilds}") @@ -163,7 +164,7 @@ def recompute_analytics(self, guildId): client = self.DB_connections.mongoOps.mongo_db_access.db_mongo_client - guild_c = GuildsRnDaoModel(client["RnDAO"]) + guild_c = GuildsRnDaoModel(client["Core"]) selectedChannels = guild_c.get_guild_channels(guildId=guildId) if selectedChannels != []: @@ -257,15 +258,11 @@ def _update_isin_progress(self, guildId): """ client = self.DB_connections.mongoOps.mongo_db_access.db_mongo_client - client["RnDAO"]["guilds"].update_one( - {"guildId": guildId}, {"$set": {"isInProgress": False}} + client["Core"]["platforms"].update_one( + {"metadata.id": guildId}, {"$set": {"metadata.isInProgress": False}} ) -# get guildId from command, if not given return None -# python ./analyzer.py guildId - - def getParamsFromCmd(): """ get guildId and recompute analysis arguments from cmd From 8ab6bfbeb85fad7111d8024cf453e0959d8750c1 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 29 Nov 2023 13:58:50 +0330 Subject: [PATCH 02/22] feat: Added guild id fetching for platform! now the sagas would contain data["platform._id"] and we would fetch the guild id from here and other things could run the same as before. Note: tests needs to be updated! --- discord_analyzer/rn_analyzer.py | 2 +- discord_utils.py | 12 +++- tests/integration/test_analyzer_init.py | 3 +- .../test_automation_db_load_from_db.py | 3 +- .../test_automation_db_save_to_db.py | 3 +- tests/integration/test_get_guild_id.py | 55 +++++++++++++++++++ .../test_get_past_7_days_heatmaps.py | 1 + .../integration/test_neo4j_compute_metrics.py | 1 + .../test_neo4j_compute_metrics_from_start.py | 1 + .../test_network_graph_creation.py | 1 + tests/integration/test_publish_on_success.py | 3 +- tests/integration/test_service_connection.py | 1 + tests/integration/utils/analyzer_setup.py | 3 +- tests/integration/utils/mock_graph.py | 3 +- .../utils/remove_and_setup_guild.py | 1 + tests/unit/test_mongo_singleton.py | 1 + utils/get_guild_id.py | 28 ++++++++++ 17 files changed, 112 insertions(+), 10 deletions(-) create mode 100644 tests/integration/test_get_guild_id.py create mode 100644 utils/get_guild_id.py diff --git a/discord_analyzer/rn_analyzer.py b/discord_analyzer/rn_analyzer.py index 7eb09dc..4774d8a 100644 --- a/discord_analyzer/rn_analyzer.py +++ b/discord_analyzer/rn_analyzer.py @@ -259,7 +259,7 @@ def _update_isin_progress(self, guildId): client = self.DB_connections.mongoOps.mongo_db_access.db_mongo_client client["Core"]["platforms"].update_one( - {"metadata.id": guildId}, {"$set": {"metadata.isInProgress": False}} + {"metadata.id": guildId}, {"$set": {"isInProgress": False}} ) diff --git a/discord_utils.py b/discord_utils.py index dbe7b28..ab596f4 100644 --- a/discord_utils.py +++ b/discord_utils.py @@ -7,6 +7,8 @@ from utils.get_rabbitmq import prepare_rabbit_mq from utils.transactions_ordering import sort_transactions +from utils.get_guild_id import get_guild_id + def analyzer_recompute(sagaId: str, rabbit_creds: dict[str, Any]): analyzer_init = AnalyzerInit() @@ -23,7 +25,8 @@ def analyzer_recompute(sagaId: str, rabbit_creds: dict[str, Any]): f"Warn: Saga not found!, stopping the recompute for sagaId: {sagaId}" ) else: - guildId = saga.data["guildId"] + platform_id = saga.data["platform._id"] + guildId = get_guild_id(platform_id) def recompute_wrapper(**kwargs): analyzer.recompute_analytics(guildId=guildId) @@ -53,7 +56,8 @@ def analyzer_run_once(sagaId: str, rabbit_creds: dict[str, Any]): if saga is None: logging.warn(f"Saga not found!, stopping the run_once for sagaId: {sagaId}") else: - guildId = saga.data["guildId"] + platform_id = saga.data["platform._id"] + guildId = get_guild_id(platform_id) def run_once_wrapper(**kwargs): analyzer.run_once(guildId=guildId) @@ -101,7 +105,9 @@ def publish_on_success(connection, result, *args, **kwargs): (transactions_ordered, tx_not_started_count) = sort_transactions(transactions) - guildId = saga.data["guildId"] + platform_id = saga.data["platform._id"] + guildId = get_guild_id(platform_id) + msg = f"GUILDID: {guildId}: " if tx_not_started_count != 0: tx = transactions_ordered[0] diff --git a/tests/integration/test_analyzer_init.py b/tests/integration/test_analyzer_init.py index 7fd7899..056abb8 100644 --- a/tests/integration/test_analyzer_init.py +++ b/tests/integration/test_analyzer_init.py @@ -1,7 +1,8 @@ from datetime import datetime, timedelta -from analyzer_init import AnalyzerInit from pymongo import MongoClient + +from analyzer_init import AnalyzerInit from utils.daolytics_uitls import get_mongo_credentials diff --git a/tests/integration/test_automation_db_load_from_db.py b/tests/integration/test_automation_db_load_from_db.py index 213bd60..2aeaaf8 100644 --- a/tests/integration/test_automation_db_load_from_db.py +++ b/tests/integration/test_automation_db_load_from_db.py @@ -2,8 +2,9 @@ import unittest from datetime import datetime, timedelta -from automation.utils.model import AutomationDB from dotenv import load_dotenv + +from automation.utils.model import AutomationDB from utils.get_mongo_client import MongoSingleton diff --git a/tests/integration/test_automation_db_save_to_db.py b/tests/integration/test_automation_db_save_to_db.py index dd3fa64..c5d3037 100644 --- a/tests/integration/test_automation_db_save_to_db.py +++ b/tests/integration/test_automation_db_save_to_db.py @@ -2,9 +2,10 @@ import unittest from datetime import datetime, timezone +from dotenv import load_dotenv + from automation.utils.interfaces import Automation from automation.utils.model import AutomationDB -from dotenv import load_dotenv from utils.get_mongo_client import MongoSingleton diff --git a/tests/integration/test_get_guild_id.py b/tests/integration/test_get_guild_id.py new file mode 100644 index 0000000..f58cd53 --- /dev/null +++ b/tests/integration/test_get_guild_id.py @@ -0,0 +1,55 @@ +from datetime import datetime +from unittest import TestCase + +from bson.objectid import ObjectId + +from utils.get_guild_id import get_guild_id +from utils.get_mongo_client import MongoSingleton + + +class TestGetGuildId(TestCase): + def test_get_guild_id_avalable_guild(self): + client = MongoSingleton.get_instance().client + platform_id = "515151515151515151515151" + + client.drop_database("Core") + client["Core"]["Platforms"].insert_one( + { + "_id": ObjectId(platform_id), + "name": "discord", + "metadata": { + "id": "999888877766655", + "icon": "111111111111111111111111", + "name": "A guild", + "selectedChannels": [ + "11111111", + "22222222", + "33333333", + "44444444", + "55555555", + "66666666", + "77777777", + ], + "period": datetime(2023, 6, 1), + }, + "community": ObjectId("aabbccddeeff001122334455"), + "disconnectedAt": None, + "connectedAt": datetime(2023, 11, 1), + "isInProgress": True, + "createdAt": datetime(2023, 11, 1), + "updatedAt": datetime(2023, 11, 1), + "__v": 0, + } + ) + + guild_id = get_guild_id(platform_id) + self.assertEqual(guild_id, "999888877766655") + + def test_guild_id_no_document_raise_error(self): + client = MongoSingleton.get_instance().client + platform_id = "515151515151515151515151" + + client.drop_database("Core") + + with self.assertRaises(AttributeError): + get_guild_id(platform_id) diff --git a/tests/integration/test_get_past_7_days_heatmaps.py b/tests/integration/test_get_past_7_days_heatmaps.py index 4fe5d2c..4ddeecb 100644 --- a/tests/integration/test_get_past_7_days_heatmaps.py +++ b/tests/integration/test_get_past_7_days_heatmaps.py @@ -1,6 +1,7 @@ from datetime import datetime, timedelta import numpy as np + from discord_analyzer.analysis.utils.member_activity_utils import get_users_past_window from .utils.analyzer_setup import launch_db_access diff --git a/tests/integration/test_neo4j_compute_metrics.py b/tests/integration/test_neo4j_compute_metrics.py index e2d0295..2e0f75b 100644 --- a/tests/integration/test_neo4j_compute_metrics.py +++ b/tests/integration/test_neo4j_compute_metrics.py @@ -1,4 +1,5 @@ import numpy as np + from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics from .utils.neo4j_conn import neo4j_setup diff --git a/tests/integration/test_neo4j_compute_metrics_from_start.py b/tests/integration/test_neo4j_compute_metrics_from_start.py index 33d0d7d..809f4f9 100644 --- a/tests/integration/test_neo4j_compute_metrics_from_start.py +++ b/tests/integration/test_neo4j_compute_metrics_from_start.py @@ -1,4 +1,5 @@ import numpy as np + from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics from .utils.neo4j_conn import neo4j_setup diff --git a/tests/integration/test_network_graph_creation.py b/tests/integration/test_network_graph_creation.py index 591c794..b33017c 100644 --- a/tests/integration/test_network_graph_creation.py +++ b/tests/integration/test_network_graph_creation.py @@ -3,6 +3,7 @@ import networkx as nx import numpy as np + from discord_analyzer.analysis.utils.activity import Activity from .utils.mock_graph import generate_mock_graph, store_mock_data_in_neo4j diff --git a/tests/integration/test_publish_on_success.py b/tests/integration/test_publish_on_success.py index 7839ccd..8b64379 100644 --- a/tests/integration/test_publish_on_success.py +++ b/tests/integration/test_publish_on_success.py @@ -1,6 +1,8 @@ import os from datetime import datetime, timedelta +from dotenv import load_dotenv + from automation.utils.interfaces import ( Automation, AutomationAction, @@ -8,7 +10,6 @@ AutomationTrigger, ) from discord_utils import publish_on_success -from dotenv import load_dotenv from utils.daolytics_uitls import get_mongo_credentials from .utils.analyzer_setup import launch_db_access diff --git a/tests/integration/test_service_connection.py b/tests/integration/test_service_connection.py index 5e42f64..e51dd53 100644 --- a/tests/integration/test_service_connection.py +++ b/tests/integration/test_service_connection.py @@ -1,4 +1,5 @@ from tc_messageBroker.message_broker import RabbitMQ + from utils.daolytics_uitls import get_rabbit_mq_credentials diff --git a/tests/integration/utils/analyzer_setup.py b/tests/integration/utils/analyzer_setup.py index e48f473..eaae75c 100644 --- a/tests/integration/utils/analyzer_setup.py +++ b/tests/integration/utils/analyzer_setup.py @@ -1,8 +1,9 @@ import os +from dotenv import load_dotenv + from discord_analyzer.DB_operations.mongodb_access import DB_access from discord_analyzer.rn_analyzer import RnDaoAnalyzer -from dotenv import load_dotenv def setup_analyzer() -> RnDaoAnalyzer: diff --git a/tests/integration/utils/mock_graph.py b/tests/integration/utils/mock_graph.py index d244e3a..56acea8 100644 --- a/tests/integration/utils/mock_graph.py +++ b/tests/integration/utils/mock_graph.py @@ -1,10 +1,11 @@ import os -from discord_analyzer import RnDaoAnalyzer from dotenv import load_dotenv from tc_core_analyzer_lib.assess_engagement import EngagementAssessment from tc_core_analyzer_lib.utils.activity import DiscordActivity +from discord_analyzer import RnDaoAnalyzer + from .activity_params import prepare_activity_params diff --git a/tests/integration/utils/remove_and_setup_guild.py b/tests/integration/utils/remove_and_setup_guild.py index bf4b697..80b7bab 100644 --- a/tests/integration/utils/remove_and_setup_guild.py +++ b/tests/integration/utils/remove_and_setup_guild.py @@ -2,6 +2,7 @@ from typing import Optional import numpy as np + from discord_analyzer.DB_operations.mongodb_access import DB_access diff --git a/tests/unit/test_mongo_singleton.py b/tests/unit/test_mongo_singleton.py index 518c02e..fa441e2 100644 --- a/tests/unit/test_mongo_singleton.py +++ b/tests/unit/test_mongo_singleton.py @@ -1,6 +1,7 @@ import unittest from pymongo import MongoClient + from utils.get_mongo_client import MongoSingleton diff --git a/utils/get_guild_id.py b/utils/get_guild_id.py new file mode 100644 index 0000000..ddc145c --- /dev/null +++ b/utils/get_guild_id.py @@ -0,0 +1,28 @@ +from bson.objectid import ObjectId +from utils.get_mongo_client import MongoSingleton + + +def get_guild_id(platform_id: str) -> str: + """ + get the guild id from the platform id + + Parameters + ----------- + platform_id : str + the platform `_id` within the Platforms collection + + Returns + -------- + guild_id : str + the discord guild id for that specific platform + """ + mongo_client = MongoSingleton.get_instance().client + + platform = mongo_client["Core"]["Platforms"].find_one( + {"name": "discord", "_id": ObjectId(platform_id)} + ) + if platform is None: + raise AttributeError("No guild found!") + + guild_id = platform["metadata"]["id"] + return guild_id From cf67fcd144e8c3498328a60c1f1b69e96df05b1d Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 29 Nov 2023 17:00:17 +0330 Subject: [PATCH 03/22] feat: Updating tests with new cc db structure! --- .../DB_operations/mongodb_query.py | 62 +------------------ .../compute_interaction_matrix_discord.py | 11 ++-- .../analysis/compute_member_activity.py | 2 +- .../analyzer/analyzer_heatmaps.py | 2 +- .../analyzer/analyzer_memberactivities.py | 21 +++---- .../analyzer/memberactivity_utils.py | 11 +--- discord_analyzer/models/GuildsRnDaoModel.py | 17 +++-- discord_analyzer/rn_analyzer.py | 16 ++--- tests/integration/test_all_active_period.py | 26 ++++---- tests/integration/test_analyzer_init.py | 39 +++++++----- ...member_activity_from_start_no_past_data.py | 44 +++++++------ ...rom_start_with_guild_heatmaps_available.py | 4 +- ...t_with_guild_memberactivities_available.py | 4 +- ...r_activity_from_start_with_one_interval.py | 4 +- tests/integration/test_publish_on_success.py | 43 +++++++------ .../utils/remove_and_setup_guild.py | 43 +++++++------ 16 files changed, 154 insertions(+), 195 deletions(-) diff --git a/discord_analyzer/DB_operations/mongodb_query.py b/discord_analyzer/DB_operations/mongodb_query.py index fe6842e..549efcf 100644 --- a/discord_analyzer/DB_operations/mongodb_query.py +++ b/discord_analyzer/DB_operations/mongodb_query.py @@ -5,46 +5,12 @@ def __init__(self) -> None: """ pass - def _check_inputs( - self, - acc_names, - channels, - dates, - variable_aggregation_type="and", - value_aggregation_type="or", - ): - """ - just check whether the inputs are correctly entered or not - """ - # checking the length of arrays - if len(acc_names) < 1: - raise ValueError("acc_names array is empty!") - if len(channels) < 1: - raise ValueError("channels array is empty!") - if len(dates) < 1: - raise ValueError("dates array is empty!") - - # checking the variable aggregation_type variable - if variable_aggregation_type not in ["and", "or"]: - raise ValueError( - f"variable aggregation type must be either `and` or \ - `or`!\nentered value is:{variable_aggregation_type}" - ) - - # checking the value aggregation_type variable - if value_aggregation_type not in ["and", "or"]: - raise ValueError( - f"value aggregation type must be either `and` or \ - `or`!\nentered value is:{value_aggregation_type}" - ) - def create_query_filter_account_channel_dates( self, acc_names, channels, dates, variable_aggregation_type="and", - value_aggregation_type="or", date_key="date", channel_key="channelId", account_key="account_name", @@ -94,35 +60,13 @@ def create_query_filter_account_channel_dates( query : dictionary the query to get access """ - - # creating each part of query seperately - - # creating date query - date_query = [] - for date in dates: - date_query.append({date_key: {"$regex": date}}) - - # creating channels query - channel_query = [] - - for ch in channels: - channel_query.append({channel_key: ch}) - - # creating the account_name query - account_query = [] - - for account in acc_names: - account_query.append({account_key: account}) - # creating the query query = { "$" + variable_aggregation_type: [ - {"$" + value_aggregation_type: account_query}, - {"$" + value_aggregation_type: channel_query}, - # for time we should definitly use `or` because - # `and` would result in nothing! - {"$or": date_query}, + {account_key: {"$in": acc_names}}, + {channel_key: {"$in": channels}}, + {date_key: {"$in": dates}}, ] } diff --git a/discord_analyzer/analysis/compute_interaction_matrix_discord.py b/discord_analyzer/analysis/compute_interaction_matrix_discord.py index d2200dd..8b2fd20 100644 --- a/discord_analyzer/analysis/compute_interaction_matrix_discord.py +++ b/discord_analyzer/analysis/compute_interaction_matrix_discord.py @@ -14,13 +14,14 @@ generate_interaction_matrix, prepare_per_account, ) +from discord_analyzer.DB_operations.mongodb_access import DB_access def compute_interaction_matrix_discord( - acc_names, - dates, - channels, - db_access, + acc_names: list[str], + dates: list[str], + channels: list[str], + db_access: DB_access, activities: list[str] = [Activity.Mention, Activity.Reply, Activity.Reaction], ) -> dict[str, ndarray]: """ @@ -64,7 +65,7 @@ def compute_interaction_matrix_discord( query_dict = query.create_query_filter_account_channel_dates( acc_names=acc_names, channels=channels, - dates=dates, + dates=list(dates), date_key="date", channel_key="channelId", account_key="account_name", diff --git a/discord_analyzer/analysis/compute_member_activity.py b/discord_analyzer/analysis/compute_member_activity.py index 22bdd7c..be59837 100644 --- a/discord_analyzer/analysis/compute_member_activity.py +++ b/discord_analyzer/analysis/compute_member_activity.py @@ -6,6 +6,7 @@ # Author Ene SS Rawa / Tjitse van der Molen from datetime import datetime, timedelta +import logging import networkx as nx import numpy as np @@ -39,7 +40,6 @@ def compute_member_activity( date_range, window_param, act_param, - logging, load_past_data=True, ): """ diff --git a/discord_analyzer/analyzer/analyzer_heatmaps.py b/discord_analyzer/analyzer/analyzer_heatmaps.py index 2211b59..d475348 100644 --- a/discord_analyzer/analyzer/analyzer_heatmaps.py +++ b/discord_analyzer/analyzer/analyzer_heatmaps.py @@ -69,7 +69,7 @@ def analysis_heatmap(self, guildId: str, from_start: bool = False): # guild parameter is the name of the database rawinfo_c = RawInfoModel(client[guildId]) heatmap_c = HeatMapModel(client[guildId]) - guild_rndao_c = GuildsRnDaoModel(client["RnDAO"]) + guild_rndao_c = GuildsRnDaoModel(client["Core"]) # Testing if there are entries in the rawinfo collection if rawinfo_c.count() == 0: diff --git a/discord_analyzer/analyzer/analyzer_memberactivities.py b/discord_analyzer/analyzer/analyzer_memberactivities.py index 60c107f..49a858c 100644 --- a/discord_analyzer/analyzer/analyzer_memberactivities.py +++ b/discord_analyzer/analyzer/analyzer_memberactivities.py @@ -4,10 +4,11 @@ from discord_analyzer.analyzer.memberactivity_utils import MemberActivityUtils from discord_analyzer.models.MemberActivityModel import MemberActivityModel from discord_analyzer.models.RawInfoModel import RawInfoModel +from discord_analyzer.DB_operations.mongo_neo4j_ops import MongoNeo4jDB class Member_activities: - def __init__(self, DB_connections, logging) -> None: + def __init__(self, DB_connections: MongoNeo4jDB, logging) -> None: self.DB_connections = DB_connections self.logging = logging @@ -58,18 +59,15 @@ def analysis_member_activity(self, guildId, mongo_connection_str, from_start=Fal ) return (None, None) - # get current guild setting - setting = self.utils.get_one_guild(guildId) + # get current guild_info + guild_info = self.utils.get_one_guild(guildId) channels, window, action = ( - setting["selectedChannels"], - setting["window"], - setting["action"], + guild_info["metadata"]["selectedChannels"], + guild_info["metadata"]["window"], + guild_info["metadata"]["action"], ) - channels = setting["selectedChannels"] - window = setting["window"] - action = setting["action"] - period = setting["period"] + period = guild_info["metadata"]["period"] channels = list(map(lambda x: x["channelId"], channels)) @@ -93,7 +91,7 @@ def analysis_member_activity(self, guildId, mongo_connection_str, from_start=Fal first_date = period if first_date is None: - self.logging.error(f"No guild: {guildId} available in RnDAO.guilds!") + self.logging.error(f"No guild: {guildId} available in Platforms.core!") return None, None last_date = today - timedelta(days=1) @@ -126,7 +124,6 @@ def analysis_member_activity(self, guildId, mongo_connection_str, from_start=Fal date_range, window, action, - logging=self.logging, load_past_data=load_past_data, ) diff --git a/discord_analyzer/analyzer/memberactivity_utils.py b/discord_analyzer/analyzer/memberactivity_utils.py index caa8602..158cc5f 100644 --- a/discord_analyzer/analyzer/memberactivity_utils.py +++ b/discord_analyzer/analyzer/memberactivity_utils.py @@ -34,14 +34,9 @@ def refine_memberactivities_data(self, all_member_activities, first_date): def get_one_guild(self, guild): """Get one guild setting from guilds collection by guild""" - # guild_c = GuildsRnDaoModel( - # self.DB_connection.mongoOps.mongo_db_access.db_mongo_client["RnDAO"] - # ) - - # result = guild_c.get_guild_info(guild) - result = self.DB_connection.mongoOps.mongo_db_access.db_mongo_client["RnDAO"][ - "guilds" - ].find_one({"guildId": guild}) + result = self.DB_connection.mongoOps.mongo_db_access.db_mongo_client["Core"][ + "Platforms" + ].find_one({"metadata.id": guild}) return result # get all user accounts during date_range in guild from rawinfo data diff --git a/discord_analyzer/models/GuildsRnDaoModel.py b/discord_analyzer/models/GuildsRnDaoModel.py index f081f7b..fa06bcd 100644 --- a/discord_analyzer/models/GuildsRnDaoModel.py +++ b/discord_analyzer/models/GuildsRnDaoModel.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -from bson.objectid import ObjectId from pymongo.database import Database from discord_analyzer.models.BaseModel import BaseModel @@ -7,7 +6,7 @@ class GuildsRnDaoModel(BaseModel): def __init__(self, database: Database): - super().__init__(collection_name="platforms", database=database) + super().__init__(collection_name="Platforms", database=database) # print(self.database[self.collection_name].find_one()) def get_connected_guilds(self, guildId: str | None = None): @@ -15,9 +14,9 @@ def get_connected_guilds(self, guildId: str | None = None): Returns the list of the connected guilds if guildId is None Otherwise the list of one connected guild with given guildId """ - findOption = {"isDisconnected": False, "name": "discord"} + findOption = {"disconnectedAt": None, "name": "discord"} if guildId is not None: - findOption["_id"] = ObjectId(guildId) + findOption["metadata.id"] = guildId guilds = self.database[self.collection_name].find(findOption) return [x["metadata"]["id"] for x in guilds] @@ -31,14 +30,14 @@ def get_guild_info(self, guildId): def get_guild_period(self, guildId: str): """ - get the period field from guild saved in RnDAO collection + get the period field from guild saved in Platforms collection """ data = self.database[self.collection_name].find_one( {"metadata.id": guildId, "name": "discord"}, {"metadata.period": 1, "_id": 0}, ) if data is not None: - return data["metadata.period"] + return data["metadata"]["period"] else: return None @@ -61,16 +60,14 @@ def get_guild_channels(self, guildId: str) -> list[dict]: query = {"metadata.id": guildId, "name": "discord"} feature_projection = {"metadata.selectedChannels": 1, "_id": 0} - cursor = self.database[self.collection_name].find_one( + results = self.database[self.collection_name].find_one( query, projection=feature_projection ) - results = list(cursor) - selected_channels: list[dict] if results is None: selected_channels = [] else: - selected_channels = results["metadata.selectedChannels"] + selected_channels = results["metadata"]["selectedChannels"] return selected_channels diff --git a/discord_analyzer/rn_analyzer.py b/discord_analyzer/rn_analyzer.py index 4774d8a..66034f6 100644 --- a/discord_analyzer/rn_analyzer.py +++ b/discord_analyzer/rn_analyzer.py @@ -96,9 +96,9 @@ def run_once(self, guildId): remove_memberactivities=False, ) - self.neo4j_analytics.compute_metrics(guildId=guildId, from_start=False) + self.neo4j_analytics.compute_metrics(guildId=guild, from_start=False) - self._update_isin_progress(guildId=guild) + self._update_isin_progress(guildId=guild) def get_guilds(self): """Returns the list of all guilds""" @@ -109,8 +109,8 @@ def get_guilds(self): def recompute_analytics_on_guilds(self, guildId_list): """ - recompute the analytics for the guilds available in RnDAO table - if the guildId_list wasn't available in RnDAO then don't recompute the analytics + recompute the analytics for the guilds available in Core table + if the guildId_list wasn't available in Core then don't recompute the analytics Parameters: -------------- @@ -124,7 +124,7 @@ def recompute_analytics_on_guilds(self, guildId_list): """ client = self.DB_connections.mongoOps.mongo_db_access.db_mongo_client - # check if the guild was available in RnDAO table + # check if the guild was available in Core table guilds_c = GuildsRnDaoModel(client["Core"]) guilds = guilds_c.get_connected_guilds(guildId_list) @@ -143,7 +143,7 @@ def recompute_analytics(self, guildId): for a new selection of channels - - first it would update the channel selection in RnDAO + - first it would update the channel selection in Core.Platform - Second the memebracitivites collection of the input guildId would become empty @@ -249,7 +249,7 @@ def recompute_analytics(self, guildId): def _update_isin_progress(self, guildId): """ - update isInProgress field of RnDAO collection + update isInProgress field of Platforms collection Parameters: ------------ @@ -258,7 +258,7 @@ def _update_isin_progress(self, guildId): """ client = self.DB_connections.mongoOps.mongo_db_access.db_mongo_client - client["Core"]["platforms"].update_one( + client["Core"]["Platforms"].update_one( {"metadata.id": guildId}, {"$set": {"isInProgress": False}} ) diff --git a/tests/integration/test_all_active_period.py b/tests/integration/test_all_active_period.py index 9b016f0..3dbb302 100644 --- a/tests/integration/test_all_active_period.py +++ b/tests/integration/test_all_active_period.py @@ -122,52 +122,52 @@ def test_two_weeks_period_active_members(): ) memberactivities = list(memberactivities_cursor) - print(f"memberactivities: {memberactivities}") + # print(f"memberactivities: {memberactivities}") date_now = datetime.now() for activities in memberactivities: date = datetime.fromisoformat(activities["date"]).date() - print("date: ", date) + # print("date: ", date) # 14 days minues 7 if date == (date_now - timedelta(days=14)).date(): - print("time delta days: 14") + # print("time delta days: 14") assert set(activities["all_active"]) == set(["user0", "user1"]) elif date == (date_now - timedelta(days=13)).date(): - print("time delta days: 13") + # print("time delta days: 13") assert set(activities["all_active"]) == set(["user0", "user1"]) elif date == (date_now - timedelta(days=12)).date(): - print("time delta days: 12") + # print("time delta days: 12") assert set(activities["all_active"]) == set(["user0", "user1"]) elif date == (date_now - timedelta(days=11)).date(): - print("time delta days: 11") + # print("time delta days: 11") assert set(activities["all_active"]) == set( ["user0", "user1", "user2", "user3"] ) elif date == (date_now - timedelta(days=10)).date(): - print("time delta days: 10") + # print("time delta days: 10") assert set(activities["all_active"]) == set( ["user0", "user1", "user2", "user3"] ) elif date == (date_now - timedelta(days=9)).date(): - print("time delta days: 9") + # print("time delta days: 9") assert set(activities["all_active"]) == set( ["user0", "user1", "user2", "user3"] ) elif date == (date_now - timedelta(days=8)).date(): - print("time delta days: 8") + # print("time delta days: 8") assert set(activities["all_active"]) == set( ["user0", "user1", "user2", "user3"] ) elif date == (date_now - timedelta(days=7)).date(): - print("time delta days: 7") + # print("time delta days: 7") assert set(activities["all_active"]) == set(["user2", "user3"]) elif date == (date_now - timedelta(days=6)).date(): - print("time delta days: 6") + # print("time delta days: 6") assert set(activities["all_active"]) == set(["user2", "user3"]) elif date == (date_now - timedelta(days=5)).date(): - print("time delta days: 5") + # print("time delta days: 5") assert set(activities["all_active"]) == set(["user2", "user3"]) else: - print("time delta days: else") + # print("time delta days: else") assert set(activities["all_active"]) == set() diff --git a/tests/integration/test_analyzer_init.py b/tests/integration/test_analyzer_init.py index 056abb8..58eb327 100644 --- a/tests/integration/test_analyzer_init.py +++ b/tests/integration/test_analyzer_init.py @@ -1,5 +1,6 @@ from datetime import datetime, timedelta +from bson.objectid import ObjectId from pymongo import MongoClient from analyzer_init import AnalyzerInit @@ -10,6 +11,7 @@ def test_analyzer_init(): analyzer = AnalyzerInit() guildId = "1234" + platform_id = "515151515151515151515151" days_ago_period = 30 mongo_creds = get_mongo_credentials() user = mongo_creds["user"] @@ -21,27 +23,30 @@ def test_analyzer_init(): mongo_client: MongoClient = MongoClient(url) - mongo_client["RnDAO"]["guilds"].delete_one({"guildId": guildId}) + mongo_client["Core"]["Platforms"].delete_one({"metadata.id": guildId}) mongo_client.drop_database(guildId) - mongo_client["RnDAO"]["guilds"].insert_one( + mongo_client["Core"]["Platforms"].insert_one( { - "guildId": guildId, - "user": "876487027099582524", - "name": "Sample Guild", - "connectedAt": (datetime.now() - timedelta(days=10)), + "_id": ObjectId(platform_id), + "name": "discord", + "metadata": { + "id": guildId, + "icon": "111111111111111111111111", + "name": "A guild", + "selectedChannels": [ + {"channelId": "1020707129214111827", "channelName": "general"} + ], + "window": [7, 1], + "action": [1, 1, 1, 4, 3, 5, 5, 4, 3, 3, 2, 2, 1], + "period": datetime.now() - timedelta(days=days_ago_period), + }, + "community": ObjectId("aabbccddeeff001122334455"), + "disconnectedAt": None, + "connectedAt": (datetime.now() - timedelta(days=days_ago_period + 10)), "isInProgress": True, - "isDisconnected": False, - "icon": "afd0d06fd12b2905c53708ca742e6c66", - "window": [7, 1], - "action": [1, 1, 1, 4, 3, 5, 5, 4, 3, 3, 2, 2, 1], - "selectedChannels": [ - { - "channelId": "1020707129214111827", - "channelName": "general", - }, - ], - "period": (datetime.now() - timedelta(days=days_ago_period)), + "createdAt": datetime(2023, 11, 1), + "updatedAt": datetime(2023, 11, 1), } ) diff --git a/tests/integration/test_member_activity_from_start_no_past_data.py b/tests/integration/test_member_activity_from_start_no_past_data.py index 23ee60c..1c4637d 100644 --- a/tests/integration/test_member_activity_from_start_no_past_data.py +++ b/tests/integration/test_member_activity_from_start_no_past_data.py @@ -1,9 +1,11 @@ # test analyzing memberactivities from datetime import datetime, timedelta +from bson.objectid import ObjectId from .utils.analyzer_setup import launch_db_access, setup_analyzer + def test_analyzer_member_activities_from_start_empty_memberactivities(): """ run the analyzer for a specific guild with from_start option equal to True @@ -11,29 +13,33 @@ def test_analyzer_member_activities_from_start_empty_memberactivities(): """ # first create the collections guildId = "1234" + platform_id = "515151515151515151515151" db_access = launch_db_access(guildId) - db_access.db_mongo_client["RnDAO"]["guilds"].delete_one({"guildId": guildId}) + db_access.db_mongo_client["Core"]["Platforms"].delete_one({"metadata.id": guildId}) db_access.db_mongo_client.drop_database(guildId) - db_access.db_mongo_client["RnDAO"]["guilds"].insert_one( + db_access.db_mongo_client["Core"]["Platforms"].insert_one( { - "guildId": guildId, - "user": "1223455", - "name": "Loud place", - "connectedAt": (datetime.now() - timedelta(days=10)), + "_id": ObjectId(platform_id), + "name": "discord", + "metadata": { + "id": guildId, + "icon": "111111111111111111111111", + "name": "A guild", + "selectedChannels": [ + {"channelId": "1020707129214111827", "channelName": "general"} + ], + "window": [7, 1], + "action": [1, 1, 1, 4, 3, 5, 5, 4, 3, 3, 2, 2, 1], + "period": datetime.now() - timedelta(days=30), + }, + "community": ObjectId("aabbccddeeff001122334455"), + "disconnectedAt": None, + "connectedAt": (datetime.now() - timedelta(days=40)), "isInProgress": True, - "isDisconnected": False, - "icon": "afd0d06fd12b2905c53708ca742e6c66", - "window": [7, 1], - "action": [1, 1, 1, 4, 3, 5, 5, 4, 3, 3, 2, 2, 1], - "selectedChannels": [ - { - "channelId": "41414262", - "channelName": "general", - }, - ], - "period": (datetime.now() - timedelta(days=30)), + "createdAt": datetime(2023, 11, 1), + "updatedAt": datetime(2023, 11, 1), } ) db_access.db_mongo_client[guildId].create_collection("heatmaps") @@ -81,8 +87,8 @@ def test_analyzer_member_activities_from_start_empty_memberactivities(): "memberactivities" ].find_one({}) heatmaps_data = db_access.db_mongo_client[guildId]["heatmaps"].find_one({}) - guild_document = db_access.db_mongo_client["RnDAO"]["guilds"].find_one( - {"guildId": guildId} + guild_document = db_access.db_mongo_client["Core"]["Platforms"].find_one( + {"metadata.id": guildId} ) # testing whether any data is available diff --git a/tests/integration/test_member_activity_from_start_with_guild_heatmaps_available.py b/tests/integration/test_member_activity_from_start_with_guild_heatmaps_available.py index e195209..bd55f4f 100644 --- a/tests/integration/test_member_activity_from_start_with_guild_heatmaps_available.py +++ b/tests/integration/test_member_activity_from_start_with_guild_heatmaps_available.py @@ -63,8 +63,8 @@ def test_analyzer_member_activities_from_start_available_heatmaps(): "memberactivities" ].find_one({}) heatmaps_data = db_access.db_mongo_client[guildId]["heatmaps"].find_one({}) - guild_document = db_access.db_mongo_client["RnDAO"]["guilds"].find_one( - {"guildId": guildId} + guild_document = db_access.db_mongo_client["Core"]["Platforms"].find_one( + {"metadata.id": guildId} ) # testing whether any data is available diff --git a/tests/integration/test_member_activity_from_start_with_guild_memberactivities_available.py b/tests/integration/test_member_activity_from_start_with_guild_memberactivities_available.py index d857d0e..c6eba05 100644 --- a/tests/integration/test_member_activity_from_start_with_guild_memberactivities_available.py +++ b/tests/integration/test_member_activity_from_start_with_guild_memberactivities_available.py @@ -57,8 +57,8 @@ def test_analyzer_member_activities_from_start_available_member_activity(): "memberactivities" ].find_one({}) heatmaps_data = db_access.db_mongo_client[guildId]["heatmaps"].find_one({}) - guild_document = db_access.db_mongo_client["RnDAO"]["guilds"].find_one( - {"guildId": guildId} + guild_document = db_access.db_mongo_client["Core"]["Platforms"].find_one( + {"metadata.id": guildId} ) # testing whether any data is available diff --git a/tests/integration/test_member_activity_from_start_with_one_interval.py b/tests/integration/test_member_activity_from_start_with_one_interval.py index 7dfea94..3451155 100644 --- a/tests/integration/test_member_activity_from_start_with_one_interval.py +++ b/tests/integration/test_member_activity_from_start_with_one_interval.py @@ -48,8 +48,8 @@ def test_analyzer_from_start_one_interval(): "memberactivities" ].find_one({}) heatmaps_data = db_access.db_mongo_client[guildId]["heatmaps"].find_one({}) - guild_document = db_access.db_mongo_client["RnDAO"]["guilds"].find_one( - {"guildId": guildId} + guild_document = db_access.db_mongo_client["Core"]["Platforms"].find_one( + {"metadata.id": guildId} ) # testing whether any data is available diff --git a/tests/integration/test_publish_on_success.py b/tests/integration/test_publish_on_success.py index 8b64379..b80e163 100644 --- a/tests/integration/test_publish_on_success.py +++ b/tests/integration/test_publish_on_success.py @@ -1,6 +1,7 @@ import os from datetime import datetime, timedelta +from bson.objectid import ObjectId from dotenv import load_dotenv from automation.utils.interfaces import ( @@ -21,7 +22,7 @@ def test_publish_on_success_check_notification_choreographies(): we want to check the database if the notify choreographies are created """ load_dotenv() - + platform_id = "515151515151515151515151" guild_id = "1234" saga_id = "000000011111113333377777ie0w" expected_owner_id = "334461287892" @@ -31,30 +32,36 @@ def test_publish_on_success_check_notification_choreographies(): at_db = os.getenv("AUTOMATION_DB_NAME") at_collection = os.getenv("AUTOMATION_DB_COLLECTION") - db_access.db_mongo_client["RnDAO"].drop_collection("guilds") + db_access.db_mongo_client["Core"]["Platforms"].delete_one( + {"_id": ObjectId(platform_id)} + ) db_access.db_mongo_client[guild_id].drop_collection("memberactivities") db_access.db_mongo_client[saga_db].drop_collection(saga_collection) db_access.db_mongo_client[guild_id].drop_collection("guildmembers") db_access.db_mongo_client[at_db].drop_collection(at_collection) - db_access.db_mongo_client["RnDAO"]["guilds"].insert_one( + db_access.db_mongo_client["Core"]["Platforms"].insert_one( { - "guildId": guild_id, - "user": expected_owner_id, - "name": "Sample Guild", - "connectedAt": datetime.now(), - "isInProgress": False, - "isDisconnected": False, - "icon": "4256asdiqwjo032", - "window": [7, 1], - "action": [1, 1, 1, 4, 3, 5, 5, 4, 3, 2, 2, 2, 1], - "selectedChannels": [ - { - "channelId": "11111111111111", - "channelName": "general", - }, - ], + "_id": ObjectId(platform_id), + "name": "discord", + "metadata": { + "id": guild_id, + "icon": "111111111111111111111111", + "name": "A guild", + "selectedChannels": [ + {"channelId": "4455178", "channelName": "general"} + ], + "window": [7, 1], + "action": [1, 1, 1, 4, 3, 5, 5, 4, 3, 3, 2, 2, 1], + "period": datetime.now() - timedelta(days=10), + }, + "community": ObjectId("aabbccddeeff001122334455"), + "disconnectedAt": None, + "connectedAt": (datetime.now() - timedelta(days=10)), + "isInProgress": True, + "createdAt": datetime(2023, 11, 1), + "updatedAt": datetime(2023, 11, 1), } ) diff --git a/tests/integration/utils/remove_and_setup_guild.py b/tests/integration/utils/remove_and_setup_guild.py index 80b7bab..9ea027e 100644 --- a/tests/integration/utils/remove_and_setup_guild.py +++ b/tests/integration/utils/remove_and_setup_guild.py @@ -2,6 +2,7 @@ from typing import Optional import numpy as np +from bson.objectid import ObjectId from discord_analyzer.DB_operations.mongodb_access import DB_access @@ -15,34 +16,40 @@ def setup_db_guild( days_ago_period: int = 30, ): """ - Remove the guild from RnDAO databse and then insert it there + Remove the guild from Core databse and then insert it there also drop the guildId database and re-create it then create the guildmembers collection in it `discordId_isbot` is representative if each user is bot or not """ + platform_id = "515151515151515151515151" - db_access.db_mongo_client["RnDAO"]["guilds"].delete_one({"guildId": guildId}) + db_access.db_mongo_client["Core"]["Platforms"].delete_one( + {"_id": ObjectId(platform_id)} + ) db_access.db_mongo_client.drop_database(guildId) - db_access.db_mongo_client["RnDAO"]["guilds"].insert_one( + db_access.db_mongo_client["Core"]["Platforms"].insert_one( { - "guildId": guildId, - "user": "876487027099582524", - "name": "Sample Guild", - "connectedAt": (datetime.now() - timedelta(days=10)), + "_id": ObjectId(platform_id), + "name": "discord", + "metadata": { + "id": guildId, + "icon": "111111111111111111111111", + "name": "A guild", + "selectedChannels": [ + {"channelId": "1020707129214111827", "channelName": "general"} + ], + "window": [7, 1], + "action": [1, 1, 1, 4, 3, 5, 5, 4, 3, 3, 2, 2, 1], + "period": datetime.now() - timedelta(days=days_ago_period), + }, + "community": ObjectId("aabbccddeeff001122334455"), + "disconnectedAt": None, + "connectedAt": (datetime.now() - timedelta(days=days_ago_period + 10)), "isInProgress": True, - "isDisconnected": False, - "icon": "afd0d06fd12b2905c53708ca742e6c66", - "window": [7, 1], - "action": [1, 1, 1, 4, 3, 5, 5, 4, 3, 3, 2, 2, 1], - "selectedChannels": [ - { - "channelId": "1020707129214111827", - "channelName": "general", - }, - ], - "period": (datetime.now() - timedelta(days=days_ago_period)), + "createdAt": datetime(2023, 11, 1), + "updatedAt": datetime(2023, 11, 1), } ) From a9a1dc577b4aca93535a1b2c793fcd8c0b53e224 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 29 Nov 2023 18:04:09 +0330 Subject: [PATCH 04/22] update: Fixed failing tests! --- ...fire_messages_mongo_saga_message_no_handlebar.py | 13 ++++++++++--- .../test_member_activity_from_start_no_past_data.py | 1 - tests/integration/test_publish_on_success.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_automation_fire_messages_mongo_saga_message_no_handlebar.py b/tests/integration/test_automation_fire_messages_mongo_saga_message_no_handlebar.py index 9de908b..75fdef7 100644 --- a/tests/integration/test_automation_fire_messages_mongo_saga_message_no_handlebar.py +++ b/tests/integration/test_automation_fire_messages_mongo_saga_message_no_handlebar.py @@ -227,6 +227,13 @@ def test_automation_fire_message_check_mongodb_document_messages_ngu_strategy(): user_cm_doc = db_access.db_mongo_client["Saga"]["sagas"].find_one( {"data.discordId": "999"} ) - expected_msg = "hey body! This users were messaged:\n" - expected_msg += "- 1111\n- 1112\n- 1113\n" - assert user_cm_doc["data"]["message"] == expected_msg + + # splitting because sometimes the orders change + user_cm_doc_splitted = user_cm_doc["data"]["message"].split("\n") + assert len(user_cm_doc_splitted) == 5 + + assert user_cm_doc_splitted[0] == "hey body! This users were messaged:\n" + assert user_cm_doc_splitted[1] in ["- 1111", "- 1112", "- 1113", ""] + assert user_cm_doc_splitted[2] in ["- 1111", "- 1112", "- 1113", ""] + assert user_cm_doc_splitted[3] in ["- 1111", "- 1112", "- 1113", ""] + assert user_cm_doc_splitted[4] in ["- 1111", "- 1112", "- 1113", ""] diff --git a/tests/integration/test_member_activity_from_start_no_past_data.py b/tests/integration/test_member_activity_from_start_no_past_data.py index 1c4637d..4f45f30 100644 --- a/tests/integration/test_member_activity_from_start_no_past_data.py +++ b/tests/integration/test_member_activity_from_start_no_past_data.py @@ -5,7 +5,6 @@ from .utils.analyzer_setup import launch_db_access, setup_analyzer - def test_analyzer_member_activities_from_start_empty_memberactivities(): """ run the analyzer for a specific guild with from_start option equal to True diff --git a/tests/integration/test_publish_on_success.py b/tests/integration/test_publish_on_success.py index b80e163..eb21366 100644 --- a/tests/integration/test_publish_on_success.py +++ b/tests/integration/test_publish_on_success.py @@ -105,7 +105,7 @@ def test_publish_on_success_check_notification_choreographies(): }, "status": "IN_PROGRESS", "data": { - "guildId": guild_id, + "platform._id": platform_id, "created": False, "discordId": expected_owner_id, "message": "data is ready", From 1d80d044ec372c1ebc2abc64c79c7fd360805a3d Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 29 Nov 2023 18:29:48 +0330 Subject: [PATCH 05/22] fix: linter issue based on superlinter rules! --- discord_analyzer/DB_operations/mongo_neo4j_ops.py | 3 +-- discord_analyzer/DB_operations/mongodb_interaction.py | 3 +-- .../analysis/compute_interaction_matrix_discord.py | 5 ++--- discord_analyzer/analysis/compute_member_activity.py | 6 +++--- discord_analyzer/analysis/member_activity_history.py | 1 - .../analysis/neo4j_analysis/analyzer_node_stats.py | 3 +-- discord_analyzer/analysis/neo4j_analysis/centrality.py | 2 +- .../analysis/neo4j_analysis/local_clustering_coefficient.py | 3 +-- discord_analyzer/analysis/neo4j_analysis/louvain.py | 3 +-- discord_analyzer/analysis/neo4j_metrics.py | 3 +-- .../analysis/utils/compute_interaction_mtx_utils.py | 1 - .../analysis/utils/member_activity_history_utils.py | 3 +-- discord_analyzer/analysis/utils/member_activity_utils.py | 1 - discord_analyzer/analyzer/analyzer_memberactivities.py | 2 +- discord_analyzer/analyzer/heatmaps_utils.py | 3 +-- discord_analyzer/analyzer/memberactivity_utils.py | 1 - discord_analyzer/analyzer/neo4j_analytics.py | 3 +-- discord_analyzer/models/GuildsRnDaoModel.py | 3 +-- discord_analyzer/models/HeatMapModel.py | 4 +--- discord_analyzer/models/MemberActivityModel.py | 1 - discord_analyzer/models/RawInfoModel.py | 3 +-- discord_analyzer/rn_analyzer.py | 3 +-- discord_utils.py | 3 +-- tests/integration/test_analyzer_init.py | 2 +- tests/integration/test_automation_db_load_from_db.py | 2 +- tests/integration/test_automation_db_save_to_db.py | 3 +-- tests/integration/test_get_guild_id.py | 1 - tests/integration/test_get_past_7_days_heatmaps.py | 1 - .../test_member_activity_from_start_no_past_data.py | 1 - tests/integration/test_neo4j_compute_metrics.py | 1 - tests/integration/test_neo4j_compute_metrics_from_start.py | 1 - tests/integration/test_network_graph_creation.py | 1 - tests/unit/test_example.py | 2 +- 33 files changed, 25 insertions(+), 53 deletions(-) diff --git a/discord_analyzer/DB_operations/mongo_neo4j_ops.py b/discord_analyzer/DB_operations/mongo_neo4j_ops.py index d748f0c..1429068 100644 --- a/discord_analyzer/DB_operations/mongo_neo4j_ops.py +++ b/discord_analyzer/DB_operations/mongo_neo4j_ops.py @@ -1,9 +1,8 @@ import logging -from tc_neo4j_lib.neo4j_ops import Neo4jOps - from discord_analyzer.DB_operations.mongodb_interaction import MongoDBOps from discord_analyzer.DB_operations.network_graph import make_neo4j_networkx_query_dict +from tc_neo4j_lib.neo4j_ops import Neo4jOps class MongoNeo4jDB: diff --git a/discord_analyzer/DB_operations/mongodb_interaction.py b/discord_analyzer/DB_operations/mongodb_interaction.py index edeee4c..4783c2f 100644 --- a/discord_analyzer/DB_operations/mongodb_interaction.py +++ b/discord_analyzer/DB_operations/mongodb_interaction.py @@ -1,10 +1,9 @@ import logging +from discord_analyzer.DB_operations.mongodb_access import DB_access from pymongo.read_concern import ReadConcern from pymongo.write_concern import WriteConcern -from discord_analyzer.DB_operations.mongodb_access import DB_access - class MongoDBOps: def __init__(self, user, password, host, port): diff --git a/discord_analyzer/analysis/compute_interaction_matrix_discord.py b/discord_analyzer/analysis/compute_interaction_matrix_discord.py index 8b2fd20..b2337b5 100644 --- a/discord_analyzer/analysis/compute_interaction_matrix_discord.py +++ b/discord_analyzer/analysis/compute_interaction_matrix_discord.py @@ -5,16 +5,15 @@ # # Author Ene SS Rawa / Tjitse van der Molen -from numpy import ndarray - from discord_analyzer.analysis.utils.activity import Activity +from discord_analyzer.DB_operations.mongodb_access import DB_access from discord_analyzer.DB_operations.mongodb_query import MongodbQuery +from numpy import ndarray from .utils.compute_interaction_mtx_utils import ( generate_interaction_matrix, prepare_per_account, ) -from discord_analyzer.DB_operations.mongodb_access import DB_access def compute_interaction_matrix_discord( diff --git a/discord_analyzer/analysis/compute_member_activity.py b/discord_analyzer/analysis/compute_member_activity.py index be59837..523837e 100644 --- a/discord_analyzer/analysis/compute_member_activity.py +++ b/discord_analyzer/analysis/compute_member_activity.py @@ -5,14 +5,12 @@ # # Author Ene SS Rawa / Tjitse van der Molen -from datetime import datetime, timedelta import logging +from datetime import datetime, timedelta import networkx as nx import numpy as np from dateutil.relativedelta import relativedelta -from tc_core_analyzer_lib.assess_engagement import EngagementAssessment -from tc_core_analyzer_lib.utils.activity import DiscordActivity from discord_analyzer.analysis.compute_interaction_matrix_discord import ( compute_interaction_matrix_discord, @@ -30,6 +28,8 @@ update_activities, ) from discord_analyzer.DB_operations.mongodb_access import DB_access +from tc_core_analyzer_lib.assess_engagement import EngagementAssessment +from tc_core_analyzer_lib.utils.activity import DiscordActivity def compute_member_activity( diff --git a/discord_analyzer/analysis/member_activity_history.py b/discord_analyzer/analysis/member_activity_history.py index 3663912..1797f23 100644 --- a/discord_analyzer/analysis/member_activity_history.py +++ b/discord_analyzer/analysis/member_activity_history.py @@ -4,7 +4,6 @@ import datetime from dateutil import parser - from discord_analyzer.analysis.utils.member_activity_history_utils import ( MemberActivityPastUtils, ) diff --git a/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py b/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py index b06cec3..5741669 100644 --- a/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py +++ b/discord_analyzer/analysis/neo4j_analysis/analyzer_node_stats.py @@ -3,9 +3,8 @@ from uuid import uuid1 import pandas as pd -from tc_neo4j_lib.neo4j_ops import Neo4jOps - from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils +from tc_neo4j_lib.neo4j_ops import Neo4jOps class NodeStats: diff --git a/discord_analyzer/analysis/neo4j_analysis/centrality.py b/discord_analyzer/analysis/neo4j_analysis/centrality.py index b9623b4..f20cd94 100644 --- a/discord_analyzer/analysis/neo4j_analysis/centrality.py +++ b/discord_analyzer/analysis/neo4j_analysis/centrality.py @@ -2,10 +2,10 @@ from typing import Literal import pandas as pd -from tc_neo4j_lib.neo4j_ops import Neo4jOps from discord_analyzer.analysis.neo4j_metrics import Neo4JMetrics from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils +from tc_neo4j_lib.neo4j_ops import Neo4jOps class Centerality: diff --git a/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py b/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py index 1d61714..869cd8f 100644 --- a/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py +++ b/discord_analyzer/analysis/neo4j_analysis/local_clustering_coefficient.py @@ -1,9 +1,8 @@ import logging from uuid import uuid1 -from graphdatascience import GraphDataScience - from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils +from graphdatascience import GraphDataScience class LocalClusteringCoeff: diff --git a/discord_analyzer/analysis/neo4j_analysis/louvain.py b/discord_analyzer/analysis/neo4j_analysis/louvain.py index 7ac7b40..82eb060 100644 --- a/discord_analyzer/analysis/neo4j_analysis/louvain.py +++ b/discord_analyzer/analysis/neo4j_analysis/louvain.py @@ -1,9 +1,8 @@ import logging from uuid import uuid1 -from tc_neo4j_lib.neo4j_ops import Neo4jOps - from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils +from tc_neo4j_lib.neo4j_ops import Neo4jOps class Louvain: diff --git a/discord_analyzer/analysis/neo4j_metrics.py b/discord_analyzer/analysis/neo4j_metrics.py index 9b18c6a..b23d6b3 100644 --- a/discord_analyzer/analysis/neo4j_metrics.py +++ b/discord_analyzer/analysis/neo4j_metrics.py @@ -1,10 +1,9 @@ import os +from discord_analyzer.analysis.neo4j_utils.compute_metrics import Neo4JMetrics from dotenv import load_dotenv from tc_neo4j_lib.neo4j_ops import Neo4jOps -from discord_analyzer.analysis.neo4j_utils.compute_metrics import Neo4JMetrics - def degree_centrality( gds, diff --git a/discord_analyzer/analysis/utils/compute_interaction_mtx_utils.py b/discord_analyzer/analysis/utils/compute_interaction_mtx_utils.py index 9f0c72e..f3d8636 100644 --- a/discord_analyzer/analysis/utils/compute_interaction_mtx_utils.py +++ b/discord_analyzer/analysis/utils/compute_interaction_mtx_utils.py @@ -2,7 +2,6 @@ from typing import Any import numpy as np - from discord_analyzer.analysis.analytics_interactions_script import ( per_account_interactions, ) diff --git a/discord_analyzer/analysis/utils/member_activity_history_utils.py b/discord_analyzer/analysis/utils/member_activity_history_utils.py index e602a50..2ce63fe 100644 --- a/discord_analyzer/analysis/utils/member_activity_history_utils.py +++ b/discord_analyzer/analysis/utils/member_activity_history_utils.py @@ -3,9 +3,8 @@ from typing import Any from dateutil import parser -from numpy import array - from discord_analyzer.DB_operations.mongodb_access import DB_access +from numpy import array class MemberActivityPastUtils: diff --git a/discord_analyzer/analysis/utils/member_activity_utils.py b/discord_analyzer/analysis/utils/member_activity_utils.py index 9f6d5e5..a388e3c 100644 --- a/discord_analyzer/analysis/utils/member_activity_utils.py +++ b/discord_analyzer/analysis/utils/member_activity_utils.py @@ -3,7 +3,6 @@ import numpy as np import pymongo - from discord_analyzer.DB_operations.mongodb_access import DB_access diff --git a/discord_analyzer/analyzer/analyzer_memberactivities.py b/discord_analyzer/analyzer/analyzer_memberactivities.py index 49a858c..6609829 100644 --- a/discord_analyzer/analyzer/analyzer_memberactivities.py +++ b/discord_analyzer/analyzer/analyzer_memberactivities.py @@ -2,9 +2,9 @@ from discord_analyzer.analysis.compute_member_activity import compute_member_activity from discord_analyzer.analyzer.memberactivity_utils import MemberActivityUtils +from discord_analyzer.DB_operations.mongo_neo4j_ops import MongoNeo4jDB from discord_analyzer.models.MemberActivityModel import MemberActivityModel from discord_analyzer.models.RawInfoModel import RawInfoModel -from discord_analyzer.DB_operations.mongo_neo4j_ops import MongoNeo4jDB class Member_activities: diff --git a/discord_analyzer/analyzer/heatmaps_utils.py b/discord_analyzer/analyzer/heatmaps_utils.py index 3068754..79b1b61 100644 --- a/discord_analyzer/analyzer/heatmaps_utils.py +++ b/discord_analyzer/analyzer/heatmaps_utils.py @@ -1,6 +1,5 @@ -from pymongo import MongoClient - from discord_analyzer.schemas.accounts import AccountCounts +from pymongo import MongoClient def store_counts_dict(counts_dict): diff --git a/discord_analyzer/analyzer/memberactivity_utils.py b/discord_analyzer/analyzer/memberactivity_utils.py index 158cc5f..a2c6d76 100644 --- a/discord_analyzer/analyzer/memberactivity_utils.py +++ b/discord_analyzer/analyzer/memberactivity_utils.py @@ -1,7 +1,6 @@ import logging from dateutil import parser - from discord_analyzer.DB_operations.mongo_neo4j_ops import MongoNeo4jDB diff --git a/discord_analyzer/analyzer/neo4j_analytics.py b/discord_analyzer/analyzer/neo4j_analytics.py index 77d42d8..3672dae 100644 --- a/discord_analyzer/analyzer/neo4j_analytics.py +++ b/discord_analyzer/analyzer/neo4j_analytics.py @@ -1,14 +1,13 @@ # A wrapper to compute the neo4j metrics in cron-job import logging -from tc_neo4j_lib.neo4j_ops import Neo4jOps - from discord_analyzer.analysis.neo4j_analysis.analyzer_node_stats import NodeStats from discord_analyzer.analysis.neo4j_analysis.centrality import Centerality from discord_analyzer.analysis.neo4j_analysis.local_clustering_coefficient import ( LocalClusteringCoeff, ) from discord_analyzer.analysis.neo4j_analysis.louvain import Louvain +from tc_neo4j_lib.neo4j_ops import Neo4jOps class Neo4JAnalytics: diff --git a/discord_analyzer/models/GuildsRnDaoModel.py b/discord_analyzer/models/GuildsRnDaoModel.py index fa06bcd..333f7f7 100644 --- a/discord_analyzer/models/GuildsRnDaoModel.py +++ b/discord_analyzer/models/GuildsRnDaoModel.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 -from pymongo.database import Database - from discord_analyzer.models.BaseModel import BaseModel +from pymongo.database import Database class GuildsRnDaoModel(BaseModel): diff --git a/discord_analyzer/models/HeatMapModel.py b/discord_analyzer/models/HeatMapModel.py index a0d35d5..afcba0e 100644 --- a/discord_analyzer/models/HeatMapModel.py +++ b/discord_analyzer/models/HeatMapModel.py @@ -1,12 +1,10 @@ #!/usr/bin/env python3 -import logging from datetime import datetime +from discord_analyzer.models.BaseModel import BaseModel from pymongo import DESCENDING from pymongo.database import Database -from discord_analyzer.models.BaseModel import BaseModel - class HeatMapModel(BaseModel): def __init__(self, database: Database): diff --git a/discord_analyzer/models/MemberActivityModel.py b/discord_analyzer/models/MemberActivityModel.py index 26e750c..6b3c256 100644 --- a/discord_analyzer/models/MemberActivityModel.py +++ b/discord_analyzer/models/MemberActivityModel.py @@ -3,7 +3,6 @@ from datetime import datetime import pymongo - from discord_analyzer.models.BaseModel import BaseModel diff --git a/discord_analyzer/models/RawInfoModel.py b/discord_analyzer/models/RawInfoModel.py index 5057e29..76f78fd 100644 --- a/discord_analyzer/models/RawInfoModel.py +++ b/discord_analyzer/models/RawInfoModel.py @@ -3,11 +3,10 @@ from datetime import datetime, timedelta from typing import Any +from discord_analyzer.models.BaseModel import BaseModel from pymongo import ASCENDING from pymongo.database import Database -from discord_analyzer.models.BaseModel import BaseModel - class RawInfoModel(BaseModel): def __init__(self, database: Database): diff --git a/discord_analyzer/rn_analyzer.py b/discord_analyzer/rn_analyzer.py index 66034f6..5099869 100644 --- a/discord_analyzer/rn_analyzer.py +++ b/discord_analyzer/rn_analyzer.py @@ -3,8 +3,6 @@ import os import sys -from dotenv import load_dotenv - from discord_analyzer.analyzer.analyzer_heatmaps import Heatmaps from discord_analyzer.analyzer.analyzer_memberactivities import Member_activities from discord_analyzer.analyzer.base_analyzer import Base_analyzer @@ -12,6 +10,7 @@ from discord_analyzer.models.GuildsRnDaoModel import GuildsRnDaoModel from discord_analyzer.models.HeatMapModel import HeatMapModel from discord_analyzer.models.RawInfoModel import RawInfoModel +from dotenv import load_dotenv class RnDaoAnalyzer(Base_analyzer): diff --git a/discord_utils.py b/discord_utils.py index ab596f4..d4595ac 100644 --- a/discord_utils.py +++ b/discord_utils.py @@ -5,9 +5,8 @@ from automation.automation_workflow import AutomationWorkflow from tc_messageBroker.rabbit_mq.saga.saga_base import get_saga from utils.get_rabbitmq import prepare_rabbit_mq -from utils.transactions_ordering import sort_transactions - from utils.get_guild_id import get_guild_id +from utils.transactions_ordering import sort_transactions def analyzer_recompute(sagaId: str, rabbit_creds: dict[str, Any]): diff --git a/tests/integration/test_analyzer_init.py b/tests/integration/test_analyzer_init.py index 58eb327..517d67c 100644 --- a/tests/integration/test_analyzer_init.py +++ b/tests/integration/test_analyzer_init.py @@ -1,9 +1,9 @@ from datetime import datetime, timedelta +from analyzer_init import AnalyzerInit from bson.objectid import ObjectId from pymongo import MongoClient -from analyzer_init import AnalyzerInit from utils.daolytics_uitls import get_mongo_credentials diff --git a/tests/integration/test_automation_db_load_from_db.py b/tests/integration/test_automation_db_load_from_db.py index 2aeaaf8..5557790 100644 --- a/tests/integration/test_automation_db_load_from_db.py +++ b/tests/integration/test_automation_db_load_from_db.py @@ -2,9 +2,9 @@ import unittest from datetime import datetime, timedelta +from automation.utils.model import AutomationDB from dotenv import load_dotenv -from automation.utils.model import AutomationDB from utils.get_mongo_client import MongoSingleton diff --git a/tests/integration/test_automation_db_save_to_db.py b/tests/integration/test_automation_db_save_to_db.py index c5d3037..dd3fa64 100644 --- a/tests/integration/test_automation_db_save_to_db.py +++ b/tests/integration/test_automation_db_save_to_db.py @@ -2,10 +2,9 @@ import unittest from datetime import datetime, timezone -from dotenv import load_dotenv - from automation.utils.interfaces import Automation from automation.utils.model import AutomationDB +from dotenv import load_dotenv from utils.get_mongo_client import MongoSingleton diff --git a/tests/integration/test_get_guild_id.py b/tests/integration/test_get_guild_id.py index f58cd53..da625d0 100644 --- a/tests/integration/test_get_guild_id.py +++ b/tests/integration/test_get_guild_id.py @@ -2,7 +2,6 @@ from unittest import TestCase from bson.objectid import ObjectId - from utils.get_guild_id import get_guild_id from utils.get_mongo_client import MongoSingleton diff --git a/tests/integration/test_get_past_7_days_heatmaps.py b/tests/integration/test_get_past_7_days_heatmaps.py index 4ddeecb..4fe5d2c 100644 --- a/tests/integration/test_get_past_7_days_heatmaps.py +++ b/tests/integration/test_get_past_7_days_heatmaps.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta import numpy as np - from discord_analyzer.analysis.utils.member_activity_utils import get_users_past_window from .utils.analyzer_setup import launch_db_access diff --git a/tests/integration/test_member_activity_from_start_no_past_data.py b/tests/integration/test_member_activity_from_start_no_past_data.py index 4f45f30..0d80c48 100644 --- a/tests/integration/test_member_activity_from_start_no_past_data.py +++ b/tests/integration/test_member_activity_from_start_no_past_data.py @@ -1,7 +1,6 @@ # test analyzing memberactivities from datetime import datetime, timedelta from bson.objectid import ObjectId - from .utils.analyzer_setup import launch_db_access, setup_analyzer diff --git a/tests/integration/test_neo4j_compute_metrics.py b/tests/integration/test_neo4j_compute_metrics.py index 2e0f75b..e2d0295 100644 --- a/tests/integration/test_neo4j_compute_metrics.py +++ b/tests/integration/test_neo4j_compute_metrics.py @@ -1,5 +1,4 @@ import numpy as np - from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics from .utils.neo4j_conn import neo4j_setup diff --git a/tests/integration/test_neo4j_compute_metrics_from_start.py b/tests/integration/test_neo4j_compute_metrics_from_start.py index 809f4f9..33d0d7d 100644 --- a/tests/integration/test_neo4j_compute_metrics_from_start.py +++ b/tests/integration/test_neo4j_compute_metrics_from_start.py @@ -1,5 +1,4 @@ import numpy as np - from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics from .utils.neo4j_conn import neo4j_setup diff --git a/tests/integration/test_network_graph_creation.py b/tests/integration/test_network_graph_creation.py index b33017c..591c794 100644 --- a/tests/integration/test_network_graph_creation.py +++ b/tests/integration/test_network_graph_creation.py @@ -3,7 +3,6 @@ import networkx as nx import numpy as np - from discord_analyzer.analysis.utils.activity import Activity from .utils.mock_graph import generate_mock_graph, store_mock_data_in_neo4j diff --git a/tests/unit/test_example.py b/tests/unit/test_example.py index 2137401..50ffa60 100644 --- a/tests/unit/test_example.py +++ b/tests/unit/test_example.py @@ -6,7 +6,7 @@ def test_mongo_db_info_set(): port = 1234 host = "http://www.google.com" - password = "sample_passxyz" + password = "sample_pass" user = "sample_user" analyzer.set_mongo_database_info( From 1508967ed7d9a96535cc85caaf1e26595c15567a Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 29 Nov 2023 18:53:55 +0330 Subject: [PATCH 06/22] fix: linter issues based on superlinter! --- discord_analyzer/analysis/compute_member_activity.py | 1 - discord_analyzer/analysis/neo4j_analysis/centrality.py | 1 - discord_utils.py | 2 +- tests/integration/test_analyzer_init.py | 1 - tests/integration/test_automation_db_load_from_db.py | 1 - .../test_member_activity_from_start_no_past_data.py | 2 ++ tests/integration/test_publish_on_success.py | 3 +-- tests/integration/test_service_connection.py | 1 - tests/integration/utils/analyzer_setup.py | 3 +-- tests/integration/utils/mock_graph.py | 3 +-- tests/integration/utils/remove_and_setup_guild.py | 1 - tests/unit/test_mongo_singleton.py | 1 - 12 files changed, 6 insertions(+), 14 deletions(-) diff --git a/discord_analyzer/analysis/compute_member_activity.py b/discord_analyzer/analysis/compute_member_activity.py index 523837e..dc78655 100644 --- a/discord_analyzer/analysis/compute_member_activity.py +++ b/discord_analyzer/analysis/compute_member_activity.py @@ -11,7 +11,6 @@ import networkx as nx import numpy as np from dateutil.relativedelta import relativedelta - from discord_analyzer.analysis.compute_interaction_matrix_discord import ( compute_interaction_matrix_discord, ) diff --git a/discord_analyzer/analysis/neo4j_analysis/centrality.py b/discord_analyzer/analysis/neo4j_analysis/centrality.py index f20cd94..bf7bf17 100644 --- a/discord_analyzer/analysis/neo4j_analysis/centrality.py +++ b/discord_analyzer/analysis/neo4j_analysis/centrality.py @@ -2,7 +2,6 @@ from typing import Literal import pandas as pd - from discord_analyzer.analysis.neo4j_metrics import Neo4JMetrics from discord_analyzer.analysis.neo4j_utils.projection_utils import ProjectionUtils from tc_neo4j_lib.neo4j_ops import Neo4jOps diff --git a/discord_utils.py b/discord_utils.py index d4595ac..dd1f87e 100644 --- a/discord_utils.py +++ b/discord_utils.py @@ -4,8 +4,8 @@ from analyzer_init import AnalyzerInit from automation.automation_workflow import AutomationWorkflow from tc_messageBroker.rabbit_mq.saga.saga_base import get_saga -from utils.get_rabbitmq import prepare_rabbit_mq from utils.get_guild_id import get_guild_id +from utils.get_rabbitmq import prepare_rabbit_mq from utils.transactions_ordering import sort_transactions diff --git a/tests/integration/test_analyzer_init.py b/tests/integration/test_analyzer_init.py index 517d67c..33298da 100644 --- a/tests/integration/test_analyzer_init.py +++ b/tests/integration/test_analyzer_init.py @@ -3,7 +3,6 @@ from analyzer_init import AnalyzerInit from bson.objectid import ObjectId from pymongo import MongoClient - from utils.daolytics_uitls import get_mongo_credentials diff --git a/tests/integration/test_automation_db_load_from_db.py b/tests/integration/test_automation_db_load_from_db.py index 5557790..213bd60 100644 --- a/tests/integration/test_automation_db_load_from_db.py +++ b/tests/integration/test_automation_db_load_from_db.py @@ -4,7 +4,6 @@ from automation.utils.model import AutomationDB from dotenv import load_dotenv - from utils.get_mongo_client import MongoSingleton diff --git a/tests/integration/test_member_activity_from_start_no_past_data.py b/tests/integration/test_member_activity_from_start_no_past_data.py index 0d80c48..d495112 100644 --- a/tests/integration/test_member_activity_from_start_no_past_data.py +++ b/tests/integration/test_member_activity_from_start_no_past_data.py @@ -1,6 +1,8 @@ # test analyzing memberactivities from datetime import datetime, timedelta + from bson.objectid import ObjectId + from .utils.analyzer_setup import launch_db_access, setup_analyzer diff --git a/tests/integration/test_publish_on_success.py b/tests/integration/test_publish_on_success.py index eb21366..b9f6763 100644 --- a/tests/integration/test_publish_on_success.py +++ b/tests/integration/test_publish_on_success.py @@ -1,8 +1,6 @@ import os from datetime import datetime, timedelta - from bson.objectid import ObjectId -from dotenv import load_dotenv from automation.utils.interfaces import ( Automation, @@ -11,6 +9,7 @@ AutomationTrigger, ) from discord_utils import publish_on_success +from dotenv import load_dotenv from utils.daolytics_uitls import get_mongo_credentials from .utils.analyzer_setup import launch_db_access diff --git a/tests/integration/test_service_connection.py b/tests/integration/test_service_connection.py index e51dd53..5e42f64 100644 --- a/tests/integration/test_service_connection.py +++ b/tests/integration/test_service_connection.py @@ -1,5 +1,4 @@ from tc_messageBroker.message_broker import RabbitMQ - from utils.daolytics_uitls import get_rabbit_mq_credentials diff --git a/tests/integration/utils/analyzer_setup.py b/tests/integration/utils/analyzer_setup.py index eaae75c..e48f473 100644 --- a/tests/integration/utils/analyzer_setup.py +++ b/tests/integration/utils/analyzer_setup.py @@ -1,9 +1,8 @@ import os -from dotenv import load_dotenv - from discord_analyzer.DB_operations.mongodb_access import DB_access from discord_analyzer.rn_analyzer import RnDaoAnalyzer +from dotenv import load_dotenv def setup_analyzer() -> RnDaoAnalyzer: diff --git a/tests/integration/utils/mock_graph.py b/tests/integration/utils/mock_graph.py index 56acea8..d244e3a 100644 --- a/tests/integration/utils/mock_graph.py +++ b/tests/integration/utils/mock_graph.py @@ -1,11 +1,10 @@ import os +from discord_analyzer import RnDaoAnalyzer from dotenv import load_dotenv from tc_core_analyzer_lib.assess_engagement import EngagementAssessment from tc_core_analyzer_lib.utils.activity import DiscordActivity -from discord_analyzer import RnDaoAnalyzer - from .activity_params import prepare_activity_params diff --git a/tests/integration/utils/remove_and_setup_guild.py b/tests/integration/utils/remove_and_setup_guild.py index 9ea027e..9b741bc 100644 --- a/tests/integration/utils/remove_and_setup_guild.py +++ b/tests/integration/utils/remove_and_setup_guild.py @@ -3,7 +3,6 @@ import numpy as np from bson.objectid import ObjectId - from discord_analyzer.DB_operations.mongodb_access import DB_access diff --git a/tests/unit/test_mongo_singleton.py b/tests/unit/test_mongo_singleton.py index fa441e2..518c02e 100644 --- a/tests/unit/test_mongo_singleton.py +++ b/tests/unit/test_mongo_singleton.py @@ -1,7 +1,6 @@ import unittest from pymongo import MongoClient - from utils.get_mongo_client import MongoSingleton From 8b4347c1da0d52b8948fd5dccf70d21a3913dc29 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 08:38:00 +0330 Subject: [PATCH 07/22] fix: superlinter errors! --- tests/integration/test_publish_on_success.py | 2 +- tests/unit/test_example.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_publish_on_success.py b/tests/integration/test_publish_on_success.py index b9f6763..4f4564e 100644 --- a/tests/integration/test_publish_on_success.py +++ b/tests/integration/test_publish_on_success.py @@ -1,6 +1,5 @@ import os from datetime import datetime, timedelta -from bson.objectid import ObjectId from automation.utils.interfaces import ( Automation, @@ -8,6 +7,7 @@ AutomationReport, AutomationTrigger, ) +from bson.objectid import ObjectId from discord_utils import publish_on_success from dotenv import load_dotenv from utils.daolytics_uitls import get_mongo_credentials diff --git a/tests/unit/test_example.py b/tests/unit/test_example.py index 50ffa60..85f135c 100644 --- a/tests/unit/test_example.py +++ b/tests/unit/test_example.py @@ -1,12 +1,16 @@ +import os from discord_analyzer import RnDaoAnalyzer +from dotenv import load_dotenv def test_mongo_db_info_set(): analyzer = RnDaoAnalyzer() + load_dotenv() port = 1234 host = "http://www.google.com" - password = "sample_pass" + # to ignore gitleaks + password = os.getenv("MONGODB_PASS") user = "sample_user" analyzer.set_mongo_database_info( @@ -22,12 +26,14 @@ def test_mongo_db_info_set(): def test_neo4j_db_info_set(): + load_dotenv() port = 1234 db_name = "db" protocol = "bolt" user = "user" host = "localhost" - password = "password" + # to ignore gitleaks + password = os.getenv("NEO4J_PASSWORD") neo4j_creds = { "db_name": db_name, "password": password, From 6d0224261b3732bf43181269ece9d642789266f7 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 09:14:20 +0330 Subject: [PATCH 08/22] fix: linter issues! --- tests/unit/test_example.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_example.py b/tests/unit/test_example.py index 85f135c..9fe6e30 100644 --- a/tests/unit/test_example.py +++ b/tests/unit/test_example.py @@ -1,4 +1,5 @@ import os + from discord_analyzer import RnDaoAnalyzer from dotenv import load_dotenv From 2aa5a1a4ccfd26580c9ff40719ed8f7054ed0761 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 09:55:39 +0330 Subject: [PATCH 09/22] feat: Update action param type! --- .../analysis/compute_member_activity.py | 116 +++++++++--------- .../analyzer/analyzer_memberactivities.py | 28 +++-- discord_analyzer/rn_analyzer.py | 10 +- requirements.txt | 2 +- .../utils/remove_and_setup_guild.py | 22 +++- 5 files changed, 99 insertions(+), 79 deletions(-) diff --git a/discord_analyzer/analysis/compute_member_activity.py b/discord_analyzer/analysis/compute_member_activity.py index dc78655..c295990 100644 --- a/discord_analyzer/analysis/compute_member_activity.py +++ b/discord_analyzer/analysis/compute_member_activity.py @@ -32,72 +32,66 @@ def compute_member_activity( - db_name, - connection_string, - channels, - acc_names, - date_range, - window_param, - act_param, + db_name: str, + connection_string: str, + channels: list[str], + acc_names: list[str], + date_range: tuple[str, str], + window_param: dict[str, int], + act_param: dict[str, int], load_past_data=True, ): """ Computes member activity and member interaction network - Input + Parameters + ------------ db_name: (str) - guild id connection_string: (str) - connection to db string channels: [str] - list of all channel ids that should be analysed acc_names: [str] - list of all account names that should be analysed date_range: [str] - list of first and last date to be analysed (one output per date) - window_param: [int] - - entry 1: window size in days. default = 7 - entry 2: step size of sliding window in days. default = 1 + window_param: dict[str, int] - + "period_size": window size in days. default = 7 + "step_size": step size of sliding window in days. default = 1 (Currently these values will be default values, in the future, the user might be able to set these in the extraction settings page) - act_param: [int] - - entry 1: INT_THR - int : - minimum number of interactions to be active. - Default = 1 - entry 2: UW_DEG_THR - int : - minimum number of connections to be active. - Default = 1 - entry 3: PAUSED_T_THR - int : - time period to remain paused. - Default = 1 - entry 4: CON_T_THR - int : - time period to assess consistently active. - Default = 4 - entry 5: CON_O_THR - int : - times to be active within CON_T_THR to be consistently active. - Default = 3 - entry 6: EDGE_STR_THR - int : - minimum number of interactions for connected. - Default = 5 - entry 7: UW_THR_DEG_THR - int : - minimum number of accounts for connected. - Default = 5 - entry 8: VITAL_T_THR - int : - time period to assess for vital. - Default = 4 - entry 9: VITAL_O_THR - int : - times to be connected within VITAL_T_THR to be vital. - Default = 3 - entry 10: STILL_T_THR - int : - time period to assess for still active. - Default = 3 - entry 11: STILL_O_THR - int : - times to be active within STILL_T_THR to be still active. - Default = 2 - entry 12: DROP_H_THR - int: - Default = 2 - entry 13: DROP_I_THR - int: - Default = 1 + act_param : dict[str, int] + parameters for activity types: + keys are listed below + - INT_THR : int + minimum number of interactions to be active + - UW_DEG_THR : int + minimum number of connections to be active + - EDGE_STR_THR : int + minimum number of interactions for connected + - UW_THR_DEG_THR : int + minimum number of accounts for connected + - CON_T_THR : int + time period to assess consistently active + - CON_O_THR : int + times to be active within CON_T_THR to be + consistently active + - VITAL_T_THR : int + time period to assess for vital + - VITAL_O_THR : int + times to be connected within VITAL_T_THR to be vital + - PAUSED_T_THR : int + time period to remain paused + - STILL_T_THR : int + time period to assess for still active + - STILL_O_THR : int + times to be active within STILL_T_THR to be still active + - DROP_H_THR : int + time periods in the past to have been newly active + - DROP_I_THR : int + time periods to have been inactive (Currently these values will be default values, in the future, the user might be able to adjust these) - Output + Returns + ---------- network_dict: {datetime:networkx obj} - dictionary with python datetime objects as keys and networkx graph objects as values. @@ -190,8 +184,8 @@ def compute_member_activity( # no analytics for the days would be computed # so make it as a window_d lenght to have the computations new_date_range_interval = (new_date_range[1] - new_date_range[0]).days - if new_date_range_interval < window_param[0] - 1: - interval_before = (new_date_range_interval) + (window_param[0] - 1) + if new_date_range_interval < window_param["period_size"] - 1: + interval_before = (new_date_range_interval) + (window_param["period_size"] - 1) new_date_range[0] = new_date_range[1] - timedelta(days=interval_before) member_activity_utils = MemberActivityPastUtils(db_access=db_access) @@ -203,7 +197,7 @@ def compute_member_activity( end_dt=new_date_range[1], all_joined_day=activity_dict["all_joined_day"], starting_key=starting_key, - window_d=window_param[0], + window_d=window_param["period_size"], ) # # # DEFINE SLIDING WINDOW RANGE # # # @@ -215,7 +209,7 @@ def compute_member_activity( time_diff = end_dt - start_dt # determine maximum start time (include last day in date_range) - last_start = time_diff - relativedelta(days=window_param[0] - 1) + last_start = time_diff - relativedelta(days=window_param["period_size"] - 1) # # # ACTUAL ANALYSIS # # # @@ -230,7 +224,7 @@ def compute_member_activity( ) # for every window index - max_range = int(np.floor(last_start.days / window_param[1]) + 1) + max_range = int(np.floor(last_start.days / window_param["step_size"]) + 1) # if max range was chosen negative, # then we have to make it zero # (won't affect the loop but will affect codes after it) @@ -245,13 +239,13 @@ def compute_member_activity( last_date = ( new_date_range[0] - + relativedelta(days=window_param[1] * w_i) - + relativedelta(days=window_param[0] - 1) + + relativedelta(days=window_param["step_size"] * w_i) + + relativedelta(days=window_param["period_size"] - 1) ) # make list of all dates in window date_list_w = [] - for x in range(window_param[0]): + for x in range(window_param["period_size"]): date_list_w.append(last_date - relativedelta(days=x)) # make empty array for date string values @@ -261,7 +255,7 @@ def compute_member_activity( for i in range(len(date_list_w_str)): date_list_w_str[i] = date_list_w[i].strftime("%Y-%m-%d") - window_start = last_date - relativedelta(days=window_param[0]) + window_start = last_date - relativedelta(days=window_param["period_size"]) # updating account names for past 7 days acc_names = get_users_past_window( @@ -300,7 +294,7 @@ def compute_member_activity( w_i=new_window_i, acc_names=np.asarray(acc_names), act_param=act_param, - WINDOW_D=window_param[0], + WINDOW_D=window_param["period_size"], **activity_dict, ) @@ -335,7 +329,7 @@ def compute_member_activity( activity_dict_per_date = store_based_date( start_date=start_dt, all_activities=activity_dict, - analytics_day_range=window_param[0] - 1, + analytics_day_range=window_param["period_size"] - 1, joined_acc_dict=joined_acc_dict, load_past=load_past_data, empty_channel_acc=(len(channels) != 0 and len(acc_names) != 0), diff --git a/discord_analyzer/analyzer/analyzer_memberactivities.py b/discord_analyzer/analyzer/analyzer_memberactivities.py index 6609829..3a6e46e 100644 --- a/discord_analyzer/analyzer/analyzer_memberactivities.py +++ b/discord_analyzer/analyzer/analyzer_memberactivities.py @@ -1,3 +1,4 @@ +import logging from datetime import datetime, timedelta from discord_analyzer.analysis.compute_member_activity import compute_member_activity @@ -7,10 +8,9 @@ from discord_analyzer.models.RawInfoModel import RawInfoModel -class Member_activities: - def __init__(self, DB_connections: MongoNeo4jDB, logging) -> None: +class MemberActivities: + def __init__(self, DB_connections: MongoNeo4jDB) -> None: self.DB_connections = DB_connections - self.logging = logging self.utils = MemberActivityUtils(DB_connections) @@ -44,9 +44,9 @@ def analysis_member_activity(self, guildId, mongo_connection_str, from_start=Fal # check current guild is exist if guildId not in client.list_database_names(): - self.logging.error(f"{guild_msg} Database {guildId} doesn't exist") - self.logging.error(f"{guild_msg} No such databse!") - self.logging.info(f"{guild_msg} Continuing") + logging.error(f"{guild_msg} Database {guildId} doesn't exist") + logging.error(f"{guild_msg} No such databse!") + logging.info(f"{guild_msg} Continuing") return (None, None) member_activity_c = MemberActivityModel(client[guildId]) @@ -54,7 +54,7 @@ def analysis_member_activity(self, guildId, mongo_connection_str, from_start=Fal # Testing if there are entries in the rawinfo collection if rawinfo_c.count() == 0: - self.logging.warning( + logging.warning( f"No entries in the collection 'rawinfos' in {guildId} databse" ) return (None, None) @@ -74,7 +74,7 @@ def analysis_member_activity(self, guildId, mongo_connection_str, from_start=Fal # get date range to be analyzed today = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) - self.logging.info(f"{guild_msg} memberactivities Analysis started!") + logging.info(f"{guild_msg} memberactivities Analysis started!") # initialize load_past_data = False @@ -91,7 +91,7 @@ def analysis_member_activity(self, guildId, mongo_connection_str, from_start=Fal first_date = period if first_date is None: - self.logging.error(f"No guild: {guildId} available in Platforms.core!") + logging.error(f"No guild: {guildId} available in Platforms.core!") return None, None last_date = today - timedelta(days=1) @@ -103,7 +103,15 @@ def analysis_member_activity(self, guildId, mongo_connection_str, from_start=Fal # max([CON_T_THR, VITAL_T_THR, STILL_T_THR, PAUSED_T_THR])+1 # ) * WINDOW_D num_days_to_load = ( - max([action[3], action[7], action[9], action[2]]) + 1 + max( + [ + action["CON_T_THR"], + action["VITAL_T_THR"], + action["STILL_T_THR"], + action["PAUSED_T_THR"], + ] + ) + + 1 ) * window[0] date_range[0] = date_range[1] - timedelta(days=num_days_to_load) diff --git a/discord_analyzer/rn_analyzer.py b/discord_analyzer/rn_analyzer.py index 5099869..c2e995b 100644 --- a/discord_analyzer/rn_analyzer.py +++ b/discord_analyzer/rn_analyzer.py @@ -4,7 +4,7 @@ import sys from discord_analyzer.analyzer.analyzer_heatmaps import Heatmaps -from discord_analyzer.analyzer.analyzer_memberactivities import Member_activities +from discord_analyzer.analyzer.analyzer_memberactivities import MemberActivities from discord_analyzer.analyzer.base_analyzer import Base_analyzer from discord_analyzer.analyzer.neo4j_analytics import Neo4JAnalytics from discord_analyzer.models.GuildsRnDaoModel import GuildsRnDaoModel @@ -70,8 +70,8 @@ def run_once(self, guildId): remove_heatmaps=False, ) - memberactivities_analysis = Member_activities( - self.DB_connections, logging=logging + memberactivities_analysis = MemberActivities( + self.DB_connections ) ( member_activities_data, @@ -213,8 +213,8 @@ def recompute_analytics(self, guildId): # run the member_activity analyze logging.info(f"Analyzing the MemberActivities data for guild: {guildId}!") - memberactivity_analysis = Member_activities( - self.DB_connections, logging=logging + memberactivity_analysis = MemberActivities( + self.DB_connections ) ( member_activities_data, diff --git a/requirements.txt b/requirements.txt index 09a2a38..aa76080 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,6 @@ tc-messageBroker==1.4.0 sentry-sdk rq redis -tc-core-analyzer-lib==1.0.2 +tc-core-analyzer-lib==1.1.0 tc-neo4j-lib==1.0.0 pybars3 diff --git a/tests/integration/utils/remove_and_setup_guild.py b/tests/integration/utils/remove_and_setup_guild.py index 9b741bc..c1e5abc 100644 --- a/tests/integration/utils/remove_and_setup_guild.py +++ b/tests/integration/utils/remove_and_setup_guild.py @@ -28,6 +28,21 @@ def setup_db_guild( ) db_access.db_mongo_client.drop_database(guildId) + action = { + "INT_THR": 1, + "UW_DEG_THR": 1, + "PAUSED_T_THR": 1, + "CON_T_THR": 4, + "CON_O_THR": 3, + "EDGE_STR_THR": 5, + "UW_THR_DEG_THR": 5, + "VITAL_T_THR": 4, + "VITAL_O_THR": 3, + "STILL_T_THR": 2, + "STILL_O_THR": 2, + "DROP_H_THR": 2, + "DROP_I_THR": 1, + } db_access.db_mongo_client["Core"]["Platforms"].insert_one( { "_id": ObjectId(platform_id), @@ -39,8 +54,11 @@ def setup_db_guild( "selectedChannels": [ {"channelId": "1020707129214111827", "channelName": "general"} ], - "window": [7, 1], - "action": [1, 1, 1, 4, 3, 5, 5, 4, 3, 3, 2, 2, 1], + "window": { + "period_size": 7, + "step_size": 1 + }, + "action": action, "period": datetime.now() - timedelta(days=days_ago_period), }, "community": ObjectId("aabbccddeeff001122334455"), From 5a5acc61f790b4b95335f6800d362258767801f1 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 10:12:17 +0330 Subject: [PATCH 10/22] fix: linter issues based on superlinter! --- discord_analyzer/analysis/compute_member_activity.py | 10 +++++++--- discord_analyzer/analysis/member_activity_history.py | 11 ++++++----- .../analysis/utils/member_activity_history_utils.py | 4 ++-- .../analysis/utils/member_activity_utils.py | 4 ++-- discord_analyzer/rn_analyzer.py | 8 ++------ tests/integration/utils/remove_and_setup_guild.py | 5 +---- 6 files changed, 20 insertions(+), 22 deletions(-) diff --git a/discord_analyzer/analysis/compute_member_activity.py b/discord_analyzer/analysis/compute_member_activity.py index c295990..b0c6d84 100644 --- a/discord_analyzer/analysis/compute_member_activity.py +++ b/discord_analyzer/analysis/compute_member_activity.py @@ -124,7 +124,7 @@ def compute_member_activity( network_dict = {} # initiate result dictionaries for engagement types - activity_dict = { + activity_dict: dict[str, dict] = { "all_joined": {}, "all_joined_day": {}, "all_consistent": {}, @@ -185,7 +185,9 @@ def compute_member_activity( # so make it as a window_d lenght to have the computations new_date_range_interval = (new_date_range[1] - new_date_range[0]).days if new_date_range_interval < window_param["period_size"] - 1: - interval_before = (new_date_range_interval) + (window_param["period_size"] - 1) + interval_before = (new_date_range_interval) + ( + window_param["period_size"] - 1 + ) new_date_range[0] = new_date_range[1] - timedelta(days=interval_before) member_activity_utils = MemberActivityPastUtils(db_access=db_access) @@ -255,7 +257,9 @@ def compute_member_activity( for i in range(len(date_list_w_str)): date_list_w_str[i] = date_list_w[i].strftime("%Y-%m-%d") - window_start = last_date - relativedelta(days=window_param["period_size"]) + window_start = last_date - relativedelta( + days=window_param["period_size"] + ) # updating account names for past 7 days acc_names = get_users_past_window( diff --git a/discord_analyzer/analysis/member_activity_history.py b/discord_analyzer/analysis/member_activity_history.py index 1797f23..72abc77 100644 --- a/discord_analyzer/analysis/member_activity_history.py +++ b/discord_analyzer/analysis/member_activity_history.py @@ -13,8 +13,8 @@ # the main script function def check_past_history( db_access: DB_access, - date_range: list[str], - window_param: tuple[int, int], + date_range: tuple[str, str], + window_param: dict[str, int], collection_name: str = "memberactivities", verbose=False, ): @@ -30,9 +30,10 @@ def check_past_history( a list of length 2, the first index has the start of the interval and the second index is end of the interval *Note*: Each value of the array should be in the format of `str(%y/%m/%d)` - window_param : tuple of int - a tuple with length 2, first parameter is window length - and the second one is the step + window_param : dict[str, int] + a dictionary 2 values, the keys and values + - "period_size": window size in days. default = 7 + - "step_size": step size of sliding window in days. default = 1 collection_name: string the collection of db to use default is `memberactivities` diff --git a/discord_analyzer/analysis/utils/member_activity_history_utils.py b/discord_analyzer/analysis/utils/member_activity_history_utils.py index 2ce63fe..51dd41f 100644 --- a/discord_analyzer/analysis/utils/member_activity_history_utils.py +++ b/discord_analyzer/analysis/utils/member_activity_history_utils.py @@ -215,7 +215,7 @@ def convert_back_to_old_schema(self, retrieved_data, date_start, window_param): for idx in range(len(retrieved_data)): db_record = retrieved_data[idx] - parser.parse(db_record["date"]) - timedelta(days=window_param[0]) + parser.parse(db_record["date"]) - timedelta(days=window_param["period_size"]) for activity in activity_dict.keys(): try: @@ -235,7 +235,7 @@ def convert_back_to_old_schema(self, retrieved_data, date_start, window_param): logging.error(str(exp)) activity_dict["first_end_date"] = ( - date_start - timedelta(days=window_param[0]) + date_start - timedelta(days=window_param["period_size"]) ).isoformat() return activity_dict diff --git a/discord_analyzer/analysis/utils/member_activity_utils.py b/discord_analyzer/analysis/utils/member_activity_utils.py index a388e3c..57f0014 100644 --- a/discord_analyzer/analysis/utils/member_activity_utils.py +++ b/discord_analyzer/analysis/utils/member_activity_utils.py @@ -151,7 +151,7 @@ def update_activities(past_activities, activities_list): return activity_dictionaries -def convert_to_dict(data: list[Any], dict_keys: list[str]) -> dict: +def convert_to_dict(data: list[Any], dict_keys: list[str]) -> dict[str, dict]: """ convert data into dictionary Note: the length of data and dict_keys always must be the same @@ -165,7 +165,7 @@ def convert_to_dict(data: list[Any], dict_keys: list[str]) -> dict: Returns: --------- - converted_data : dict + converted_data : dict[str, dict] the data that is converted to dictionary with their corresponding keys """ diff --git a/discord_analyzer/rn_analyzer.py b/discord_analyzer/rn_analyzer.py index c2e995b..2a25965 100644 --- a/discord_analyzer/rn_analyzer.py +++ b/discord_analyzer/rn_analyzer.py @@ -70,9 +70,7 @@ def run_once(self, guildId): remove_heatmaps=False, ) - memberactivities_analysis = MemberActivities( - self.DB_connections - ) + memberactivities_analysis = MemberActivities(self.DB_connections) ( member_activities_data, member_acitivities_networkx_data, @@ -213,9 +211,7 @@ def recompute_analytics(self, guildId): # run the member_activity analyze logging.info(f"Analyzing the MemberActivities data for guild: {guildId}!") - memberactivity_analysis = MemberActivities( - self.DB_connections - ) + memberactivity_analysis = MemberActivities(self.DB_connections) ( member_activities_data, member_acitivities_networkx_data, diff --git a/tests/integration/utils/remove_and_setup_guild.py b/tests/integration/utils/remove_and_setup_guild.py index c1e5abc..b5ed1fa 100644 --- a/tests/integration/utils/remove_and_setup_guild.py +++ b/tests/integration/utils/remove_and_setup_guild.py @@ -54,10 +54,7 @@ def setup_db_guild( "selectedChannels": [ {"channelId": "1020707129214111827", "channelName": "general"} ], - "window": { - "period_size": 7, - "step_size": 1 - }, + "window": {"period_size": 7, "step_size": 1}, "action": action, "period": datetime.now() - timedelta(days=days_ago_period), }, From 6a4496d8e512311c81f92096360c6743d7462f48 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 10:29:38 +0330 Subject: [PATCH 11/22] update: the saga data! --- discord_utils.py | 6 +++--- tests/integration/test_publish_on_success.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/discord_utils.py b/discord_utils.py index dd1f87e..cec5dcb 100644 --- a/discord_utils.py +++ b/discord_utils.py @@ -24,7 +24,7 @@ def analyzer_recompute(sagaId: str, rabbit_creds: dict[str, Any]): f"Warn: Saga not found!, stopping the recompute for sagaId: {sagaId}" ) else: - platform_id = saga.data["platform._id"] + platform_id = saga.data["PlatformId"] guildId = get_guild_id(platform_id) def recompute_wrapper(**kwargs): @@ -55,7 +55,7 @@ def analyzer_run_once(sagaId: str, rabbit_creds: dict[str, Any]): if saga is None: logging.warn(f"Saga not found!, stopping the run_once for sagaId: {sagaId}") else: - platform_id = saga.data["platform._id"] + platform_id = saga.data["PlatformId"] guildId = get_guild_id(platform_id) def run_once_wrapper(**kwargs): @@ -104,7 +104,7 @@ def publish_on_success(connection, result, *args, **kwargs): (transactions_ordered, tx_not_started_count) = sort_transactions(transactions) - platform_id = saga.data["platform._id"] + platform_id = saga.data["PlatformId"] guildId = get_guild_id(platform_id) msg = f"GUILDID: {guildId}: " diff --git a/tests/integration/test_publish_on_success.py b/tests/integration/test_publish_on_success.py index 4f4564e..1140d33 100644 --- a/tests/integration/test_publish_on_success.py +++ b/tests/integration/test_publish_on_success.py @@ -104,7 +104,7 @@ def test_publish_on_success_check_notification_choreographies(): }, "status": "IN_PROGRESS", "data": { - "platform._id": platform_id, + "PlatformId": platform_id, "created": False, "discordId": expected_owner_id, "message": "data is ready", From 54bde23d390dae3b71ade4cd2e45726242d0038d Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 10:29:50 +0330 Subject: [PATCH 12/22] fix: superlinter issues! --- discord_analyzer/analysis/compute_member_activity.py | 2 +- .../analysis/utils/member_activity_history_utils.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/discord_analyzer/analysis/compute_member_activity.py b/discord_analyzer/analysis/compute_member_activity.py index b0c6d84..70d55d3 100644 --- a/discord_analyzer/analysis/compute_member_activity.py +++ b/discord_analyzer/analysis/compute_member_activity.py @@ -327,7 +327,7 @@ def compute_member_activity( # get the accounts with their joining date joined_acc_dict = get_joined_accounts( - db_access=db_access, date_range=[start_dt, end_dt + timedelta(days=1)] + db_access=db_access, date_range=(start_dt, end_dt + timedelta(days=1)) ) activity_dict_per_date = store_based_date( diff --git a/discord_analyzer/analysis/utils/member_activity_history_utils.py b/discord_analyzer/analysis/utils/member_activity_history_utils.py index 51dd41f..20e8c00 100644 --- a/discord_analyzer/analysis/utils/member_activity_history_utils.py +++ b/discord_analyzer/analysis/utils/member_activity_history_utils.py @@ -215,7 +215,9 @@ def convert_back_to_old_schema(self, retrieved_data, date_start, window_param): for idx in range(len(retrieved_data)): db_record = retrieved_data[idx] - parser.parse(db_record["date"]) - timedelta(days=window_param["period_size"]) + parser.parse(db_record["date"]) - timedelta( + days=window_param["period_size"] + ) for activity in activity_dict.keys(): try: From 3e32ee53402a85a0dec949d3c43b949265684d79 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 10:40:59 +0330 Subject: [PATCH 13/22] feat: update platformId in sagas to bson.ObjectId! --- tests/integration/test_get_guild_id.py | 4 ++-- tests/integration/test_publish_on_success.py | 2 +- utils/get_guild_id.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_get_guild_id.py b/tests/integration/test_get_guild_id.py index da625d0..1060509 100644 --- a/tests/integration/test_get_guild_id.py +++ b/tests/integration/test_get_guild_id.py @@ -9,7 +9,7 @@ class TestGetGuildId(TestCase): def test_get_guild_id_avalable_guild(self): client = MongoSingleton.get_instance().client - platform_id = "515151515151515151515151" + platform_id = ObjectId("515151515151515151515151") client.drop_database("Core") client["Core"]["Platforms"].insert_one( @@ -46,7 +46,7 @@ def test_get_guild_id_avalable_guild(self): def test_guild_id_no_document_raise_error(self): client = MongoSingleton.get_instance().client - platform_id = "515151515151515151515151" + platform_id = ObjectId("515151515151515151515151") client.drop_database("Core") diff --git a/tests/integration/test_publish_on_success.py b/tests/integration/test_publish_on_success.py index 1140d33..e95b0ca 100644 --- a/tests/integration/test_publish_on_success.py +++ b/tests/integration/test_publish_on_success.py @@ -104,7 +104,7 @@ def test_publish_on_success_check_notification_choreographies(): }, "status": "IN_PROGRESS", "data": { - "PlatformId": platform_id, + "PlatformId": ObjectId(platform_id), "created": False, "discordId": expected_owner_id, "message": "data is ready", diff --git a/utils/get_guild_id.py b/utils/get_guild_id.py index ddc145c..4107066 100644 --- a/utils/get_guild_id.py +++ b/utils/get_guild_id.py @@ -2,7 +2,7 @@ from utils.get_mongo_client import MongoSingleton -def get_guild_id(platform_id: str) -> str: +def get_guild_id(platform_id: ObjectId) -> str: """ get the guild id from the platform id @@ -19,7 +19,7 @@ def get_guild_id(platform_id: str) -> str: mongo_client = MongoSingleton.get_instance().client platform = mongo_client["Core"]["Platforms"].find_one( - {"name": "discord", "_id": ObjectId(platform_id)} + {"name": "discord", "_id": platform_id} ) if platform is None: raise AttributeError("No guild found!") From 3e9e944049011001946c68647eab15fa5e87571a Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 10:58:58 +0330 Subject: [PATCH 14/22] fix: update action param of tests! --- tests/integration/utils/activity_params.py | 44 ++++++++-------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/tests/integration/utils/activity_params.py b/tests/integration/utils/activity_params.py index 76c662c..a822dda 100644 --- a/tests/integration/utils/activity_params.py +++ b/tests/integration/utils/activity_params.py @@ -1,32 +1,18 @@ def prepare_activity_params(): - INT_THR = 1 # minimum number of interactions to be active - UW_DEG_THR = 1 # minimum number of accounts interacted with to be active - PAUSED_T_THR = 1 # time period to remain paused - CON_T_THR = 4 # time period to be consistent active - CON_O_THR = 3 # time period to be consistent active - EDGE_STR_THR = 5 # minimum number of interactions for connected - UW_THR_DEG_THR = 5 # minimum number of accounts for connected - VITAL_T_THR = 4 # time period to assess for vital - VITAL_O_THR = 3 # times to be connected within VITAL_T_THR to be vital - STILL_T_THR = 2 # time period to assess for still active - STILL_O_THR = 2 # times to be active within STILL_T_THR to be still active - DROP_H_THR = 2 - DROP_I_THR = 1 - - act_param = [ - INT_THR, - UW_DEG_THR, - PAUSED_T_THR, - CON_T_THR, - CON_O_THR, - EDGE_STR_THR, - UW_THR_DEG_THR, - VITAL_T_THR, - VITAL_O_THR, - STILL_T_THR, - STILL_O_THR, - DROP_H_THR, - DROP_I_THR, - ] + act_param = { + "INT_THR": 1, + "UW_DEG_THR": 1, + "PAUSED_T_THR": 1, + "CON_T_THR": 4, + "CON_O_THR": 3, + "EDGE_STR_THR": 5, + "UW_THR_DEG_THR": 5, + "VITAL_T_THR": 4, + "VITAL_O_THR": 3, + "STILL_T_THR": 2, + "STILL_O_THR": 2, + "DROP_H_THR": 2, + "DROP_I_THR": 1, + } return act_param From c1b00f77112ed0e75d8b0ac8dd1f7e870952682c Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 11:38:26 +0330 Subject: [PATCH 15/22] fix: tests based on new action type! --- tests/integration/test_analyzer_init.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_analyzer_init.py b/tests/integration/test_analyzer_init.py index 33298da..be634f5 100644 --- a/tests/integration/test_analyzer_init.py +++ b/tests/integration/test_analyzer_init.py @@ -25,6 +25,26 @@ def test_analyzer_init(): mongo_client["Core"]["Platforms"].delete_one({"metadata.id": guildId}) mongo_client.drop_database(guildId) + act_param = { + "INT_THR": 1, + "UW_DEG_THR": 1, + "PAUSED_T_THR": 1, + "CON_T_THR": 4, + "CON_O_THR": 3, + "EDGE_STR_THR": 5, + "UW_THR_DEG_THR": 5, + "VITAL_T_THR": 4, + "VITAL_O_THR": 3, + "STILL_T_THR": 2, + "STILL_O_THR": 2, + "DROP_H_THR": 2, + "DROP_I_THR": 1, + } + window = { + "period_size": 7, + "step_size": 1, + } + mongo_client["Core"]["Platforms"].insert_one( { "_id": ObjectId(platform_id), @@ -36,8 +56,8 @@ def test_analyzer_init(): "selectedChannels": [ {"channelId": "1020707129214111827", "channelName": "general"} ], - "window": [7, 1], - "action": [1, 1, 1, 4, 3, 5, 5, 4, 3, 3, 2, 2, 1], + "window": window, + "action": act_param, "period": datetime.now() - timedelta(days=days_ago_period), }, "community": ObjectId("aabbccddeeff001122334455"), From 4408e2c998e2bb87c8f70c3d766a3b3f4eb34fd8 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 11:40:50 +0330 Subject: [PATCH 16/22] update: based on new window param type! --- discord_analyzer/analyzer/analyzer_memberactivities.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/discord_analyzer/analyzer/analyzer_memberactivities.py b/discord_analyzer/analyzer/analyzer_memberactivities.py index 3a6e46e..a19eeb5 100644 --- a/discord_analyzer/analyzer/analyzer_memberactivities.py +++ b/discord_analyzer/analyzer/analyzer_memberactivities.py @@ -112,12 +112,12 @@ def analysis_member_activity(self, guildId, mongo_connection_str, from_start=Fal ] ) + 1 - ) * window[0] + ) * window["period_size"] date_range[0] = date_range[1] - timedelta(days=num_days_to_load) # if the date range goes back more than the "7 days `period` forward" - if date_range[0] < period + timedelta(days=window[0]): - date_range[0] = period + timedelta(days=window[0]) + if date_range[0] < period + timedelta(days=window["period_size"]): + date_range[0] = period + timedelta(days=window["period_size"]) # get all users during date_range all_users = self.utils.get_all_users(guildId) From 8be9d99dc81033026e10a3abfc1ee9fe9b2c5ecc Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 11:57:01 +0330 Subject: [PATCH 17/22] fix: test errors! --- ...essages_mongo_saga_message_no_handlebar.py | 2 +- ...member_activity_from_start_no_past_data.py | 20 +++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_automation_fire_messages_mongo_saga_message_no_handlebar.py b/tests/integration/test_automation_fire_messages_mongo_saga_message_no_handlebar.py index 75fdef7..cee59c6 100644 --- a/tests/integration/test_automation_fire_messages_mongo_saga_message_no_handlebar.py +++ b/tests/integration/test_automation_fire_messages_mongo_saga_message_no_handlebar.py @@ -232,7 +232,7 @@ def test_automation_fire_message_check_mongodb_document_messages_ngu_strategy(): user_cm_doc_splitted = user_cm_doc["data"]["message"].split("\n") assert len(user_cm_doc_splitted) == 5 - assert user_cm_doc_splitted[0] == "hey body! This users were messaged:\n" + assert user_cm_doc_splitted[0] == "hey body! This users were messaged:" assert user_cm_doc_splitted[1] in ["- 1111", "- 1112", "- 1113", ""] assert user_cm_doc_splitted[2] in ["- 1111", "- 1112", "- 1113", ""] assert user_cm_doc_splitted[3] in ["- 1111", "- 1112", "- 1113", ""] diff --git a/tests/integration/test_member_activity_from_start_no_past_data.py b/tests/integration/test_member_activity_from_start_no_past_data.py index d495112..dd9d415 100644 --- a/tests/integration/test_member_activity_from_start_no_past_data.py +++ b/tests/integration/test_member_activity_from_start_no_past_data.py @@ -19,6 +19,22 @@ def test_analyzer_member_activities_from_start_empty_memberactivities(): db_access.db_mongo_client["Core"]["Platforms"].delete_one({"metadata.id": guildId}) db_access.db_mongo_client.drop_database(guildId) + action = { + "INT_THR": 1, + "UW_DEG_THR": 1, + "PAUSED_T_THR": 1, + "CON_T_THR": 4, + "CON_O_THR": 3, + "EDGE_STR_THR": 5, + "UW_THR_DEG_THR": 5, + "VITAL_T_THR": 4, + "VITAL_O_THR": 3, + "STILL_T_THR": 2, + "STILL_O_THR": 2, + "DROP_H_THR": 2, + "DROP_I_THR": 1, + } + db_access.db_mongo_client["Core"]["Platforms"].insert_one( { "_id": ObjectId(platform_id), @@ -30,8 +46,8 @@ def test_analyzer_member_activities_from_start_empty_memberactivities(): "selectedChannels": [ {"channelId": "1020707129214111827", "channelName": "general"} ], - "window": [7, 1], - "action": [1, 1, 1, 4, 3, 5, 5, 4, 3, 3, 2, 2, 1], + "window": {"period_size": 7, "step_size": 1}, + "action": action, "period": datetime.now() - timedelta(days=30), }, "community": ObjectId("aabbccddeeff001122334455"), From fdc90049b243f3d6e8b4cfcfc56af3bead7d8298 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 14:53:50 +0330 Subject: [PATCH 18/22] feat: Added community node creation! --- analyzer_init.py | 6 +- automation/automation_workflow.py | 3 +- .../DB_operations/mongo_neo4j_ops.py | 7 +- .../DB_operations/network_graph.py | 91 ++++++++++++++++--- discord_analyzer/rn_analyzer.py | 51 ++--------- discord_utils.py | 21 +++-- tests/integration/test_analyzer_init.py | 5 +- ..._id.py => test_get_guild_community_ids.py} | 11 ++- .../test_network_graph_creation.py | 6 +- tests/integration/utils/analyzer_setup.py | 2 +- tests/integration/utils/mock_graph.py | 5 +- tests/unit/test_example.py | 6 +- tests/unit/test_get_timestamp.py | 67 ++++++++++++++ ...guild_id.py => get_guild_community_ids.py} | 9 +- 14 files changed, 201 insertions(+), 89 deletions(-) rename tests/integration/{test_get_guild_id.py => test_get_guild_community_ids.py} (83%) create mode 100644 tests/unit/test_get_timestamp.py rename utils/{get_guild_id.py => get_guild_community_ids.py} (67%) diff --git a/analyzer_init.py b/analyzer_init.py index f0f6b0d..a241acd 100644 --- a/analyzer_init.py +++ b/analyzer_init.py @@ -13,8 +13,8 @@ class AnalyzerInit: initialize the analyzer with its configs """ - def __init__(self) -> None: - pass + def __init__(self, community_id: str) -> None: + self.community_id = community_id def get_analyzer(self) -> tuple[RnDaoAnalyzer, dict[str, Any]]: """ @@ -23,7 +23,7 @@ def get_analyzer(self) -> tuple[RnDaoAnalyzer, dict[str, Any]]: analyzer : RnDaoAnalyzer mongo_creds : dict[str, Any] """ - analyzer = RnDaoAnalyzer() + analyzer = RnDaoAnalyzer(self.community_id) # credentials mongo_creds = get_mongo_credentials() diff --git a/automation/automation_workflow.py b/automation/automation_workflow.py index 98ae719..7ed1d6c 100644 --- a/automation/automation_workflow.py +++ b/automation/automation_workflow.py @@ -49,7 +49,8 @@ def start(self, guild_id: str): users1, users2 = self._get_users_from_memberactivities( guild_id, category ) - users = self._subtract_users(users1, users2) + users = users1 + # users = self._subtract_users(users1, users2) for action in at.actions: if action.enabled: diff --git a/discord_analyzer/DB_operations/mongo_neo4j_ops.py b/discord_analyzer/DB_operations/mongo_neo4j_ops.py index 1429068..af2086e 100644 --- a/discord_analyzer/DB_operations/mongo_neo4j_ops.py +++ b/discord_analyzer/DB_operations/mongo_neo4j_ops.py @@ -53,6 +53,7 @@ def set_mongo_db_ops( def store_analytics_data( self, analytics_data: dict, + community_id: str, remove_memberactivities: bool = False, remove_heatmaps: bool = False, ): @@ -68,6 +69,8 @@ def store_analytics_data( heatmaps is also a list of dictinoaries and memberactivities is a tuple of memberactivities dictionary list and memebractivities networkx object dictionary list + community_id : str + the community id to save the data for remove_memberactivities : bool remove the whole memberactivity data and insert default is `False` which means don't delete the existing data @@ -101,7 +104,9 @@ def store_analytics_data( and memberactivities_networkx_data != [] ): queries_list = make_neo4j_networkx_query_dict( - networkx_graphs=memberactivities_networkx_data, guildId=guildId + networkx_graphs=memberactivities_networkx_data, + guildId=guildId, + community_id=community_id, ) self.run_operations_transaction( guildId=guildId, diff --git a/discord_analyzer/DB_operations/network_graph.py b/discord_analyzer/DB_operations/network_graph.py index 504c5db..2ec6e93 100644 --- a/discord_analyzer/DB_operations/network_graph.py +++ b/discord_analyzer/DB_operations/network_graph.py @@ -5,7 +5,11 @@ import networkx -def make_neo4j_networkx_query_dict(networkx_graphs, guildId): +def make_neo4j_networkx_query_dict( + networkx_graphs: dict[networkx.classes.graph.Graph], + guildId: str, + community_id: str, +): """ make a list of queries to store networkx graphs into the neo4j @@ -17,6 +21,8 @@ def make_neo4j_networkx_query_dict(networkx_graphs, guildId): are the actual networkx graphs guildId : str the guild that the members belong to + community_id : str + the community id to save the data for Returns: ----------- @@ -34,6 +40,7 @@ def make_neo4j_networkx_query_dict(networkx_graphs, guildId): networkx_graphs=graph_list, networkx_dates=graph_dates, guildId=guildId, + community_id=community_id, toGuildRelation="IS_MEMBER", ) @@ -44,6 +51,7 @@ def make_graph_list_query( networkx_graphs: networkx.classes.graph.Graph, networkx_dates: list[datetime.datetime], guildId: str, + community_id: str, toGuildRelation: str = "IS_MEMBER", ): """ @@ -59,6 +67,8 @@ def make_graph_list_query( guildId : str the guild that the members belong to default is `None` meaning that it wouldn't be belonged to any guild + community_id : str + the community id to save the data for toGuildRelation : str the relationship label that connect the users to guilds default value is `IS_MEMBER` @@ -81,13 +91,46 @@ def make_graph_list_query( guildId=guildId, toGuildRelation=toGuildRelation, ) + community_query = create_community_node_query(community_id, guildId) final_queries.extend(node_queries) final_queries.extend(query_relations) + final_queries.append(community_query) return final_queries +def create_community_node_query( + community_id: str, + guild_id: str, + community_node: str = "Community", +) -> str: + """ + create the community node + + Parameters + ------------ + community_id : str + the community id to create its node + guild_id : str + the guild node to attach to community + """ + date_now_timestamp = get_timestamp() + + query = f""" + MERGE (g:Guild {{guildId: '{guild_id}'}}) + ON CREATE SET g.createdAt = {int(date_now_timestamp)} + WITH g + MERGE (c:{community_node} {{id: '{community_id}'}}) + ON CREATE SET c.createdAt = {int(date_now_timestamp)} + WITH g, c + MERGE (g) -[r:IS_WITHIN]-> (c) + ON CREATE SET r.createdAt = {int(date_now_timestamp)} + """ + + return query + + def create_network_query( nodes_dict: networkx.classes.reportviews.NodeDataView, edge_dict: networkx.classes.reportviews.EdgeDataView, @@ -127,19 +170,8 @@ def create_network_query( the list of MERGE queries for creating all relationships """ # getting the timestamp `date` - graph_date_timestamp = ( - graph_date.replace( - hour=0, minute=0, second=0, microsecond=0, tzinfo=datetime.timezone.utc - ).timestamp() - * 1000 - ) - date_now_timestamp = ( - datetime.datetime.now() - .replace( - hour=0, minute=0, second=0, microsecond=0, tzinfo=datetime.timezone.utc - ) - .timestamp() - ) * 1000 + graph_date_timestamp = get_timestamp(graph_date) + date_now_timestamp = get_timestamp() # initializiation of queries rel_queries = [] @@ -208,3 +240,34 @@ def create_network_query( rel_queries.append(rel_str_query + ";") return node_queries, rel_queries + + +def get_timestamp(time: datetime.datetime | None = None) -> float: + """ + get the timestamp of the given time or just now + + Parameters + ------------ + time : datetime.datetime + the time to get its timestamp + default is `None` meaning to send the time of now + + Returns + -------- + timestamp : float + the timestamp of the time multiplied to 1000 + """ + using_time: datetime.datetime + if time is not None: + using_time = time + else: + using_time = datetime.datetime.now() + + timestamp = ( + using_time.replace( + hour=0, minute=0, second=0, microsecond=0, tzinfo=datetime.timezone.utc + ).timestamp() + * 1000 + ) + + return timestamp diff --git a/discord_analyzer/rn_analyzer.py b/discord_analyzer/rn_analyzer.py index 2a25965..d70d5a3 100644 --- a/discord_analyzer/rn_analyzer.py +++ b/discord_analyzer/rn_analyzer.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import logging -import os import sys from discord_analyzer.analyzer.analyzer_heatmaps import Heatmaps @@ -10,7 +9,6 @@ from discord_analyzer.models.GuildsRnDaoModel import GuildsRnDaoModel from discord_analyzer.models.HeatMapModel import HeatMapModel from discord_analyzer.models.RawInfoModel import RawInfoModel -from dotenv import load_dotenv class RnDaoAnalyzer(Base_analyzer): @@ -19,12 +17,13 @@ class RnDaoAnalyzer(Base_analyzer): class that handles database connection and data analysis """ - def __init__(self, testing=False): + def __init__(self, community_id: str, testing=False): """ Class initiation function """ """ Testing, prevents from data upload""" self.testing = testing + self.community_id = community_id logging.basicConfig() logging.getLogger().setLevel(logging.INFO) @@ -66,6 +65,7 @@ def run_once(self, guildId): } self.DB_connections.store_analytics_data( analytics_data=analytics_data, + community_id=self.community_id, remove_memberactivities=False, remove_heatmaps=False, ) @@ -89,6 +89,7 @@ def run_once(self, guildId): self.DB_connections.store_analytics_data( analytics_data=guilds_data, + community_id=self.community_id, remove_heatmaps=False, remove_memberactivities=False, ) @@ -205,6 +206,7 @@ def recompute_analytics(self, guildId): } self.DB_connections.store_analytics_data( analytics_data=analytics_data, + community_id=self.community_id, remove_memberactivities=False, remove_heatmaps=not is_available, ) @@ -231,6 +233,7 @@ def recompute_analytics(self, guildId): self.DB_connections.store_analytics_data( analytics_data=analytics_data, + community_id=self.community_id, remove_memberactivities=True, remove_heatmaps=False, ) @@ -282,45 +285,3 @@ def getParamsFromCmd(): guildId = args[1] recompute_analysis = True return guildId, recompute_analysis - - -if __name__ == "__main__": - load_dotenv() - - # logging.basicConfig() - # logging.getLogger().setLevel(logging.INFO) - analyzer = RnDaoAnalyzer() - - user = os.getenv("MONGODB_USER", "") - password = os.getenv("MONGODB_PASS", "") - host = os.getenv("MONGODB_HOST", "") - port = os.getenv("MONGODB_PORT", "") - - neo4j_creds = {} - neo4j_creds["db_name"] = os.getenv("NEO4J_DB", "") - neo4j_creds["protocol"] = os.getenv("NEO4J_PROTOCOL", "") - neo4j_creds["host"] = os.getenv("NEO4J_HOST", "") - neo4j_creds["port"] = os.getenv("NEO4J_PORT", "") - neo4j_creds["password"] = os.getenv("NEO4J_PASSWORD", "") - neo4j_creds["user"] = os.getenv("NEO4J_USER", "") - - neo4j_user = os.getenv("NEO4J_USER", "") - neo4j_password = os.getenv("NEO4J_PASSWORD", "") - - analyzer.set_mongo_database_info( - mongo_db_host=host, - mongo_db_password=password, - mongo_db_user=user, - mongo_db_port=port, - ) - - analyzer.set_neo4j_database_info(neo4j_creds=neo4j_creds) - - guildId, recompute_analysis = getParamsFromCmd() - analyzer.database_connect() - analyzer.setup_neo4j_metrics() - - if not recompute_analysis: - analyzer.run_once(guildId) - else: - analyzer.recompute_analytics_on_guilds(guildId) diff --git a/discord_utils.py b/discord_utils.py index cec5dcb..abd95aa 100644 --- a/discord_utils.py +++ b/discord_utils.py @@ -4,14 +4,14 @@ from analyzer_init import AnalyzerInit from automation.automation_workflow import AutomationWorkflow from tc_messageBroker.rabbit_mq.saga.saga_base import get_saga -from utils.get_guild_id import get_guild_id +from utils.get_guild_community_ids import get_guild_community_ids from utils.get_rabbitmq import prepare_rabbit_mq from utils.transactions_ordering import sort_transactions +from utils.daolytics_uitls import get_mongo_credentials def analyzer_recompute(sagaId: str, rabbit_creds: dict[str, Any]): - analyzer_init = AnalyzerInit() - analyzer, mongo_creds = analyzer_init.get_analyzer() + mongo_creds = get_mongo_credentials() saga = get_saga_instance( sagaId=sagaId, @@ -25,7 +25,10 @@ def analyzer_recompute(sagaId: str, rabbit_creds: dict[str, Any]): ) else: platform_id = saga.data["PlatformId"] - guildId = get_guild_id(platform_id) + guildId, commnity_id = get_guild_community_ids(platform_id) + + analyzer_init = AnalyzerInit(commnity_id) + analyzer, mongo_creds = analyzer_init.get_analyzer() def recompute_wrapper(**kwargs): analyzer.recompute_analytics(guildId=guildId) @@ -43,8 +46,7 @@ def publish_wrapper(**kwargs): def analyzer_run_once(sagaId: str, rabbit_creds: dict[str, Any]): - analyzer_init = AnalyzerInit() - analyzer, mongo_creds = analyzer_init.get_analyzer() + mongo_creds = get_mongo_credentials() saga = get_saga_instance( sagaId=sagaId, @@ -56,7 +58,10 @@ def analyzer_run_once(sagaId: str, rabbit_creds: dict[str, Any]): logging.warn(f"Saga not found!, stopping the run_once for sagaId: {sagaId}") else: platform_id = saga.data["PlatformId"] - guildId = get_guild_id(platform_id) + guildId, commnity_id = get_guild_community_ids(platform_id) + + analyzer_init = AnalyzerInit(commnity_id) + analyzer, mongo_creds = analyzer_init.get_analyzer() def run_once_wrapper(**kwargs): analyzer.run_once(guildId=guildId) @@ -105,7 +110,7 @@ def publish_on_success(connection, result, *args, **kwargs): (transactions_ordered, tx_not_started_count) = sort_transactions(transactions) platform_id = saga.data["PlatformId"] - guildId = get_guild_id(platform_id) + guildId, _ = get_guild_community_ids(platform_id) msg = f"GUILDID: {guildId}: " if tx_not_started_count != 0: diff --git a/tests/integration/test_analyzer_init.py b/tests/integration/test_analyzer_init.py index be634f5..b9fff9e 100644 --- a/tests/integration/test_analyzer_init.py +++ b/tests/integration/test_analyzer_init.py @@ -7,7 +7,8 @@ def test_analyzer_init(): - analyzer = AnalyzerInit() + community_id = "aabbccddeeff001122334455" + analyzer = AnalyzerInit(community_id) guildId = "1234" platform_id = "515151515151515151515151" @@ -60,7 +61,7 @@ def test_analyzer_init(): "action": act_param, "period": datetime.now() - timedelta(days=days_ago_period), }, - "community": ObjectId("aabbccddeeff001122334455"), + "community": ObjectId(community_id), "disconnectedAt": None, "connectedAt": (datetime.now() - timedelta(days=days_ago_period + 10)), "isInProgress": True, diff --git a/tests/integration/test_get_guild_id.py b/tests/integration/test_get_guild_community_ids.py similarity index 83% rename from tests/integration/test_get_guild_id.py rename to tests/integration/test_get_guild_community_ids.py index 1060509..992f560 100644 --- a/tests/integration/test_get_guild_id.py +++ b/tests/integration/test_get_guild_community_ids.py @@ -2,12 +2,12 @@ from unittest import TestCase from bson.objectid import ObjectId -from utils.get_guild_id import get_guild_id +from utils.get_guild_community_ids import get_guild_community_ids from utils.get_mongo_client import MongoSingleton class TestGetGuildId(TestCase): - def test_get_guild_id_avalable_guild(self): + def test_get_avalable_guild(self): client = MongoSingleton.get_instance().client platform_id = ObjectId("515151515151515151515151") @@ -41,14 +41,15 @@ def test_get_guild_id_avalable_guild(self): } ) - guild_id = get_guild_id(platform_id) + guild_id, community_id = get_guild_community_ids(platform_id) self.assertEqual(guild_id, "999888877766655") + self.assertEqual(community_id, "aabbccddeeff001122334455") - def test_guild_id_no_document_raise_error(self): + def test_no_document_raise_error(self): client = MongoSingleton.get_instance().client platform_id = ObjectId("515151515151515151515151") client.drop_database("Core") with self.assertRaises(AttributeError): - get_guild_id(platform_id) + get_guild_community_ids(platform_id) diff --git a/tests/integration/test_network_graph_creation.py b/tests/integration/test_network_graph_creation.py index 591c794..e2cc15e 100644 --- a/tests/integration/test_network_graph_creation.py +++ b/tests/integration/test_network_graph_creation.py @@ -10,7 +10,7 @@ def test_network_graph_create(): - """ """ + community_id = "4321" neo4j_ops = neo4j_setup() # deleting all data neo4j_ops.gds.run_cypher("MATCH (n) DETACH DELETE (n)") @@ -158,7 +158,9 @@ def test_network_graph_create(): # DATABASE SAVING - store_mock_data_in_neo4j(graph_dict=graph_dict, guildId=guildId) + store_mock_data_in_neo4j( + graph_dict=graph_dict, guildId=guildId, community_id=community_id + ) results = neo4j_ops.gds.run_cypher( f""" diff --git a/tests/integration/utils/analyzer_setup.py b/tests/integration/utils/analyzer_setup.py index e48f473..252aed1 100644 --- a/tests/integration/utils/analyzer_setup.py +++ b/tests/integration/utils/analyzer_setup.py @@ -7,7 +7,7 @@ def setup_analyzer() -> RnDaoAnalyzer: load_dotenv() - analyzer = RnDaoAnalyzer() + analyzer = RnDaoAnalyzer(community_id="1234555") user = os.getenv("MONGODB_USER", "") password = os.getenv("MONGODB_PASS", "") diff --git a/tests/integration/utils/mock_graph.py b/tests/integration/utils/mock_graph.py index d244e3a..d693d78 100644 --- a/tests/integration/utils/mock_graph.py +++ b/tests/integration/utils/mock_graph.py @@ -64,7 +64,7 @@ def generate_mock_graph(int_matrix, acc_names): return graph -def store_mock_data_in_neo4j(graph_dict, guildId): +def store_mock_data_in_neo4j(graph_dict, guildId, community_id): # CREDS load_dotenv() user = os.getenv("MONGODB_USER") @@ -80,7 +80,7 @@ def store_mock_data_in_neo4j(graph_dict, guildId): neo4j_creds["password"] = os.getenv("NEO4J_PASSWORD") neo4j_creds["user"] = os.getenv("NEO4J_USER") - analyzer = RnDaoAnalyzer() + analyzer = RnDaoAnalyzer(community_id) analyzer.set_mongo_database_info( mongo_db_host=host, @@ -103,6 +103,7 @@ def store_mock_data_in_neo4j(graph_dict, guildId): analyzer.DB_connections.store_analytics_data( analytics_data=guilds_data, + community_id=community_id, remove_heatmaps=False, remove_memberactivities=False, ) diff --git a/tests/unit/test_example.py b/tests/unit/test_example.py index 9fe6e30..1af8c0d 100644 --- a/tests/unit/test_example.py +++ b/tests/unit/test_example.py @@ -5,7 +5,8 @@ def test_mongo_db_info_set(): - analyzer = RnDaoAnalyzer() + community_id = "4321" + analyzer = RnDaoAnalyzer(community_id) load_dotenv() port = 1234 @@ -44,7 +45,8 @@ def test_neo4j_db_info_set(): "user": user, } - analyzer = RnDaoAnalyzer() + community_id = "4321" + analyzer = RnDaoAnalyzer(community_id) analyzer.set_neo4j_database_info(neo4j_creds=neo4j_creds) assert analyzer.neo4j_port == port diff --git a/tests/unit/test_get_timestamp.py b/tests/unit/test_get_timestamp.py new file mode 100644 index 0000000..2c46474 --- /dev/null +++ b/tests/unit/test_get_timestamp.py @@ -0,0 +1,67 @@ +import unittest +from datetime import datetime, timezone, timedelta +from discord_analyzer.DB_operations.network_graph import get_timestamp + + +class TestGetTimestamp(unittest.TestCase): + def test_current_time(self): + """ + Test when no time is provided, it should return the current timestamp. + """ + result = get_timestamp() + current_time = ( + datetime.now(timezone.utc) + .replace(hour=0, minute=0, second=0, microsecond=0) + .timestamp() + * 1000 + ) + self.assertAlmostEqual(result, current_time, delta=1000) + + def test_specific_time(self): + """ + Test when a specific time is provided, it should return the correct timestamp. + """ + specific_time = datetime(2023, 1, 1, 12, 30, 0, tzinfo=timezone.utc) + result = get_timestamp(specific_time) + expected_timestamp = ( + specific_time.replace(hour=0, minute=0, second=0, microsecond=0).timestamp() + * 1000 + ) + self.assertAlmostEqual(result, expected_timestamp, delta=1000) + + def test_none_input(self): + """ + Test when `None` is provided as input, it should behave the same as not providing any time. + """ + result_with_none = get_timestamp(None) + result_without_none = get_timestamp() + + self.assertAlmostEqual(result_with_none, result_without_none, delta=1000) + + def test_past_time(self): + """ + Test when a past time is provided, it should return the correct timestamp. + """ + past_time = datetime(2022, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + result = get_timestamp(past_time) + expected_timestamp = ( + past_time.replace(hour=0, minute=0, second=0, microsecond=0).timestamp() + * 1000 + ) + self.assertAlmostEqual(result, expected_timestamp, delta=1000) + + def test_microseconds(self): + """ + Test when the provided time has microseconds, it should handle them correctly. + """ + time_with_microseconds = datetime( + 2023, 1, 1, 12, 30, 0, 500000, tzinfo=timezone.utc + ) + result = get_timestamp(time_with_microseconds) + expected_timestamp = ( + time_with_microseconds.replace( + hour=0, minute=0, second=0, microsecond=0 + ).timestamp() + * 1000 + ) + self.assertAlmostEqual(result, expected_timestamp, delta=1000) diff --git a/utils/get_guild_id.py b/utils/get_guild_community_ids.py similarity index 67% rename from utils/get_guild_id.py rename to utils/get_guild_community_ids.py index 4107066..576cb1c 100644 --- a/utils/get_guild_id.py +++ b/utils/get_guild_community_ids.py @@ -2,9 +2,9 @@ from utils.get_mongo_client import MongoSingleton -def get_guild_id(platform_id: ObjectId) -> str: +def get_guild_community_ids(platform_id: ObjectId) -> tuple[str, str]: """ - get the guild id from the platform id + get both the guild id and community from the platform id Parameters ----------- @@ -15,6 +15,8 @@ def get_guild_id(platform_id: ObjectId) -> str: -------- guild_id : str the discord guild id for that specific platform + community_id : str + the community id that the guild is related """ mongo_client = MongoSingleton.get_instance().client @@ -25,4 +27,5 @@ def get_guild_id(platform_id: ObjectId) -> str: raise AttributeError("No guild found!") guild_id = platform["metadata"]["id"] - return guild_id + community_id = str(platform["community"]) + return guild_id, community_id From c85d05f3b2737fdec48c2f24049ae1c70b45273e Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 15:00:10 +0330 Subject: [PATCH 19/22] update: More assertion to the tests for community --- .../test_generated_graph_period_1_year_run_once.py | 11 +++++++++++ .../integration/test_generated_graph_period_1year.py | 11 +++++++++++ .../test_generated_graph_period_35_days.py | 11 +++++++++++ .../test_generated_graph_period_35_days_run_once.py | 11 +++++++++++ .../test_generated_graph_period_3_months.py | 11 +++++++++++ .../test_generated_graph_period_3_months_run_once.py | 11 +++++++++++ .../test_generated_graph_period_6_months.py | 11 +++++++++++ .../test_generated_graph_period_6_months_run_once.py | 11 +++++++++++ 8 files changed, 88 insertions(+) diff --git a/tests/integration/test_generated_graph_period_1_year_run_once.py b/tests/integration/test_generated_graph_period_1_year_run_once.py index 4f2e63e..f99b697 100644 --- a/tests/integration/test_generated_graph_period_1_year_run_once.py +++ b/tests/integration/test_generated_graph_period_1_year_run_once.py @@ -106,3 +106,14 @@ def test_networkgraph_one_year_period_run_once_available_analytics(): assert dates[-1] == start_analytics_date.timestamp() * 1000 assert dates[0] == end_analytics_date.timestamp() * 1000 + + results = neo4j_ops.gds.run_cypher( + f""" + MATCH + (g:Guild {{guildId: '{guildId}'}}) + -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) + RETURN c.id as cid + """ + ) + assert len(results.values) == 1 + assert results["cid"].values == ["1234555"] diff --git a/tests/integration/test_generated_graph_period_1year.py b/tests/integration/test_generated_graph_period_1year.py index 84c4af3..4f3524f 100644 --- a/tests/integration/test_generated_graph_period_1year.py +++ b/tests/integration/test_generated_graph_period_1year.py @@ -104,3 +104,14 @@ def test_networkgraph_one_year_period_recompute_available_analytics(): assert dates[-1] == start_analytics_date.timestamp() * 1000 assert dates[0] == end_analytics_date.timestamp() * 1000 + + results = neo4j_ops.gds.run_cypher( + f""" + MATCH + (g:Guild {{guildId: '{guildId}'}}) + -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) + RETURN c.id as cid + """ + ) + assert len(results.values) == 1 + assert results["cid"].values == ["1234555"] diff --git a/tests/integration/test_generated_graph_period_35_days.py b/tests/integration/test_generated_graph_period_35_days.py index 4701d7e..d975228 100644 --- a/tests/integration/test_generated_graph_period_35_days.py +++ b/tests/integration/test_generated_graph_period_35_days.py @@ -104,3 +104,14 @@ def test_networkgraph_35_days_period_recompute_available_analytics(): assert dates[-1] == start_analytics_date.timestamp() * 1000 assert dates[0] == end_analytics_date.timestamp() * 1000 + + results = neo4j_ops.gds.run_cypher( + f""" + MATCH + (g:Guild {{guildId: '{guildId}'}}) + -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) + RETURN c.id as cid + """ + ) + assert len(results.values) == 1 + assert results["cid"].values == ["1234555"] diff --git a/tests/integration/test_generated_graph_period_35_days_run_once.py b/tests/integration/test_generated_graph_period_35_days_run_once.py index aea2f77..f9b15d3 100644 --- a/tests/integration/test_generated_graph_period_35_days_run_once.py +++ b/tests/integration/test_generated_graph_period_35_days_run_once.py @@ -105,3 +105,14 @@ def test_networkgraph_35_days_period_run_once_available_analytics(): assert dates[-1] == start_analytics_date.timestamp() * 1000 assert dates[0] == end_analytics_date.timestamp() * 1000 + + results = neo4j_ops.gds.run_cypher( + f""" + MATCH + (g:Guild {{guildId: '{guildId}'}}) + -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) + RETURN c.id as cid + """ + ) + assert len(results.values) == 1 + assert results["cid"].values == ["1234555"] diff --git a/tests/integration/test_generated_graph_period_3_months.py b/tests/integration/test_generated_graph_period_3_months.py index fec5821..d893b17 100644 --- a/tests/integration/test_generated_graph_period_3_months.py +++ b/tests/integration/test_generated_graph_period_3_months.py @@ -104,3 +104,14 @@ def test_networkgraph_three_months_period_recompute_available_analytics(): assert dates[-1] == start_analytics_date.timestamp() * 1000 assert dates[0] == end_analytics_date.timestamp() * 1000 + + results = neo4j_ops.gds.run_cypher( + f""" + MATCH + (g:Guild {{guildId: '{guildId}'}}) + -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) + RETURN c.id as cid + """ + ) + assert len(results.values) == 1 + assert results["cid"].values == ["1234555"] diff --git a/tests/integration/test_generated_graph_period_3_months_run_once.py b/tests/integration/test_generated_graph_period_3_months_run_once.py index c079294..351f212 100644 --- a/tests/integration/test_generated_graph_period_3_months_run_once.py +++ b/tests/integration/test_generated_graph_period_3_months_run_once.py @@ -105,3 +105,14 @@ def test_networkgraph_three_months_period_run_once_available_analytics(): assert dates[-1] == start_analytics_date.timestamp() * 1000 assert dates[0] == end_analytics_date.timestamp() * 1000 + + results = neo4j_ops.gds.run_cypher( + f""" + MATCH + (g:Guild {{guildId: '{guildId}'}}) + -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) + RETURN c.id as cid + """ + ) + assert len(results.values) == 1 + assert results["cid"].values == ["1234555"] diff --git a/tests/integration/test_generated_graph_period_6_months.py b/tests/integration/test_generated_graph_period_6_months.py index b2f969e..1f804fa 100644 --- a/tests/integration/test_generated_graph_period_6_months.py +++ b/tests/integration/test_generated_graph_period_6_months.py @@ -104,3 +104,14 @@ def test_networkgraph_six_months_period_recompute_available_analytics(): assert dates[-1] == start_analytics_date.timestamp() * 1000 assert dates[0] == end_analytics_date.timestamp() * 1000 + + results = neo4j_ops.gds.run_cypher( + f""" + MATCH + (g:Guild {{guildId: '{guildId}'}}) + -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) + RETURN c.id as cid + """ + ) + assert len(results.values) == 1 + assert results["cid"].values == ["1234555"] diff --git a/tests/integration/test_generated_graph_period_6_months_run_once.py b/tests/integration/test_generated_graph_period_6_months_run_once.py index 0146b54..f546552 100644 --- a/tests/integration/test_generated_graph_period_6_months_run_once.py +++ b/tests/integration/test_generated_graph_period_6_months_run_once.py @@ -106,3 +106,14 @@ def test_networkgraph_six_months_period_run_once_available_analytics(): assert dates[-1] == start_analytics_date.timestamp() * 1000 assert dates[0] == end_analytics_date.timestamp() * 1000 + + results = neo4j_ops.gds.run_cypher( + f""" + MATCH + (g:Guild {{guildId: '{guildId}'}}) + -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) + RETURN c.id as cid + """ + ) + assert len(results.values) == 1 + assert results["cid"].values == ["1234555"] From e40ce74f7405b9626cd973bdbfa4daf52f8a7ef7 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 15:03:25 +0330 Subject: [PATCH 20/22] fix: Forgot to change back to subtracting! --- automation/automation_workflow.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/automation/automation_workflow.py b/automation/automation_workflow.py index 7ed1d6c..98ae719 100644 --- a/automation/automation_workflow.py +++ b/automation/automation_workflow.py @@ -49,8 +49,7 @@ def start(self, guild_id: str): users1, users2 = self._get_users_from_memberactivities( guild_id, category ) - users = users1 - # users = self._subtract_users(users1, users2) + users = self._subtract_users(users1, users2) for action in at.actions: if action.enabled: From f30553dfd19f005a6bddbd8fc7a99e37a74a77ba Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 15:07:59 +0330 Subject: [PATCH 21/22] fix: linter issues based on superlinter rules! --- discord_analyzer/DB_operations/network_graph.py | 2 +- discord_utils.py | 2 +- tests/unit/test_get_timestamp.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/discord_analyzer/DB_operations/network_graph.py b/discord_analyzer/DB_operations/network_graph.py index 2ec6e93..b2dfec6 100644 --- a/discord_analyzer/DB_operations/network_graph.py +++ b/discord_analyzer/DB_operations/network_graph.py @@ -6,7 +6,7 @@ def make_neo4j_networkx_query_dict( - networkx_graphs: dict[networkx.classes.graph.Graph], + networkx_graphs: dict[datetime.datetime, networkx.classes.graph.Graph], guildId: str, community_id: str, ): diff --git a/discord_utils.py b/discord_utils.py index abd95aa..9bd166a 100644 --- a/discord_utils.py +++ b/discord_utils.py @@ -4,10 +4,10 @@ from analyzer_init import AnalyzerInit from automation.automation_workflow import AutomationWorkflow from tc_messageBroker.rabbit_mq.saga.saga_base import get_saga +from utils.daolytics_uitls import get_mongo_credentials from utils.get_guild_community_ids import get_guild_community_ids from utils.get_rabbitmq import prepare_rabbit_mq from utils.transactions_ordering import sort_transactions -from utils.daolytics_uitls import get_mongo_credentials def analyzer_recompute(sagaId: str, rabbit_creds: dict[str, Any]): diff --git a/tests/unit/test_get_timestamp.py b/tests/unit/test_get_timestamp.py index 2c46474..37c36c7 100644 --- a/tests/unit/test_get_timestamp.py +++ b/tests/unit/test_get_timestamp.py @@ -1,5 +1,6 @@ import unittest -from datetime import datetime, timezone, timedelta +from datetime import datetime, timezone + from discord_analyzer.DB_operations.network_graph import get_timestamp From 02a664a6823b95577b745b2cedaa5ab3d8d5de23 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Thu, 30 Nov 2023 15:19:12 +0330 Subject: [PATCH 22/22] fix: linter flake8 whitespace error! --- .../integration/test_generated_graph_period_1_year_run_once.py | 2 +- tests/integration/test_generated_graph_period_1year.py | 2 +- tests/integration/test_generated_graph_period_35_days.py | 2 +- .../integration/test_generated_graph_period_35_days_run_once.py | 2 +- tests/integration/test_generated_graph_period_3_months.py | 2 +- .../test_generated_graph_period_3_months_run_once.py | 2 +- tests/integration/test_generated_graph_period_6_months.py | 2 +- .../test_generated_graph_period_6_months_run_once.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_generated_graph_period_1_year_run_once.py b/tests/integration/test_generated_graph_period_1_year_run_once.py index f99b697..c9ec541 100644 --- a/tests/integration/test_generated_graph_period_1_year_run_once.py +++ b/tests/integration/test_generated_graph_period_1_year_run_once.py @@ -109,7 +109,7 @@ def test_networkgraph_one_year_period_run_once_available_analytics(): results = neo4j_ops.gds.run_cypher( f""" - MATCH + MATCH (g:Guild {{guildId: '{guildId}'}}) -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) RETURN c.id as cid diff --git a/tests/integration/test_generated_graph_period_1year.py b/tests/integration/test_generated_graph_period_1year.py index 4f3524f..b332395 100644 --- a/tests/integration/test_generated_graph_period_1year.py +++ b/tests/integration/test_generated_graph_period_1year.py @@ -107,7 +107,7 @@ def test_networkgraph_one_year_period_recompute_available_analytics(): results = neo4j_ops.gds.run_cypher( f""" - MATCH + MATCH (g:Guild {{guildId: '{guildId}'}}) -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) RETURN c.id as cid diff --git a/tests/integration/test_generated_graph_period_35_days.py b/tests/integration/test_generated_graph_period_35_days.py index d975228..9ee6c65 100644 --- a/tests/integration/test_generated_graph_period_35_days.py +++ b/tests/integration/test_generated_graph_period_35_days.py @@ -107,7 +107,7 @@ def test_networkgraph_35_days_period_recompute_available_analytics(): results = neo4j_ops.gds.run_cypher( f""" - MATCH + MATCH (g:Guild {{guildId: '{guildId}'}}) -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) RETURN c.id as cid diff --git a/tests/integration/test_generated_graph_period_35_days_run_once.py b/tests/integration/test_generated_graph_period_35_days_run_once.py index f9b15d3..8404904 100644 --- a/tests/integration/test_generated_graph_period_35_days_run_once.py +++ b/tests/integration/test_generated_graph_period_35_days_run_once.py @@ -108,7 +108,7 @@ def test_networkgraph_35_days_period_run_once_available_analytics(): results = neo4j_ops.gds.run_cypher( f""" - MATCH + MATCH (g:Guild {{guildId: '{guildId}'}}) -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) RETURN c.id as cid diff --git a/tests/integration/test_generated_graph_period_3_months.py b/tests/integration/test_generated_graph_period_3_months.py index d893b17..a92d2dd 100644 --- a/tests/integration/test_generated_graph_period_3_months.py +++ b/tests/integration/test_generated_graph_period_3_months.py @@ -107,7 +107,7 @@ def test_networkgraph_three_months_period_recompute_available_analytics(): results = neo4j_ops.gds.run_cypher( f""" - MATCH + MATCH (g:Guild {{guildId: '{guildId}'}}) -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) RETURN c.id as cid diff --git a/tests/integration/test_generated_graph_period_3_months_run_once.py b/tests/integration/test_generated_graph_period_3_months_run_once.py index 351f212..ccb3c13 100644 --- a/tests/integration/test_generated_graph_period_3_months_run_once.py +++ b/tests/integration/test_generated_graph_period_3_months_run_once.py @@ -108,7 +108,7 @@ def test_networkgraph_three_months_period_run_once_available_analytics(): results = neo4j_ops.gds.run_cypher( f""" - MATCH + MATCH (g:Guild {{guildId: '{guildId}'}}) -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) RETURN c.id as cid diff --git a/tests/integration/test_generated_graph_period_6_months.py b/tests/integration/test_generated_graph_period_6_months.py index 1f804fa..a5beb77 100644 --- a/tests/integration/test_generated_graph_period_6_months.py +++ b/tests/integration/test_generated_graph_period_6_months.py @@ -107,7 +107,7 @@ def test_networkgraph_six_months_period_recompute_available_analytics(): results = neo4j_ops.gds.run_cypher( f""" - MATCH + MATCH (g:Guild {{guildId: '{guildId}'}}) -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) RETURN c.id as cid diff --git a/tests/integration/test_generated_graph_period_6_months_run_once.py b/tests/integration/test_generated_graph_period_6_months_run_once.py index f546552..870c8c7 100644 --- a/tests/integration/test_generated_graph_period_6_months_run_once.py +++ b/tests/integration/test_generated_graph_period_6_months_run_once.py @@ -109,7 +109,7 @@ def test_networkgraph_six_months_period_run_once_available_analytics(): results = neo4j_ops.gds.run_cypher( f""" - MATCH + MATCH (g:Guild {{guildId: '{guildId}'}}) -[r:IS_WITHIN]-> (c:Community {{id: '1234555'}}) RETURN c.id as cid