From 44cb6ee4759e1e7e36bc625626e05e732467af20 Mon Sep 17 00:00:00 2001 From: Robin Cohen-Selmon <87440088+RobsOnWaves@users.noreply.github.com> Date: Sun, 28 Jan 2024 21:58:41 +0100 Subject: [PATCH 1/3] adding function to get data grouped by month --- Code/libs/mongo_db_handler.py | 60 +++++++++++++++++++++++++++++++++++ Code/public_api.py | 11 +++++-- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/Code/libs/mongo_db_handler.py b/Code/libs/mongo_db_handler.py index 8f941b2..9fbdb79 100644 --- a/Code/libs/mongo_db_handler.py +++ b/Code/libs/mongo_db_handler.py @@ -347,3 +347,63 @@ def get_df(self, db_name: str, collection_name: str, query: dict): print("Exception in getting meps documents in Mongo" + str(e)) return {"ged_insert_status": "Exception in getting meps documents in Mongo" + str(e)} + def get_df_grouped_by_month(self, + db_name: str, + collection_name: str, + query: dict, + date_start: datetime.datetime, + date_end: datetime.datetime): + + def get_first_day_of_next_month(date): + """ Retourne le premier jour du mois suivant pour une date donnée. """ + if date.month == 12: + return datetime.datetime(date.year + 1, 1, 1) + else: + return datetime.datetime(date.year, date.month + 1, 1) + + def get_last_day_of_month(date): + """ Retourne le dernier jour d'un mois pour une date donnée. """ + next_month_start = get_first_day_of_next_month(date) + return next_month_start - datetime.timedelta(days=1) + + if date_start is None: + date_start = datetime.datetime(2010, 1, 1) + if date_end is None: + date_end = datetime.datetime.now() + + + client = self.__mongo_client__ + db = client[db_name] + collection = db[collection_name] + cumulated_data = {}; + try: + + current_date = get_first_day_of_next_month(date_start) + dates = [] + + while current_date < date_end: + start_of_month = current_date + end_of_month = get_last_day_of_month(current_date) + dates.append((start_of_month, end_of_month)) + current_date = get_first_day_of_next_month(current_date) + + for date in dates: + query['Date'] = {'$gte': date[0], '$lte': date[1]} + data = list(collection.find(query, {'_id': False})) + df = pd.DataFrame(data) + + if 'Date' in df.columns: + df['Date'] = df['Date'].dt.strftime('%d/%m/%Y') + else: + print('No Date column') + + if not data: + print('No data for this date' + str(date)) + else: + cumulated_data[date[0].strftime('%Y-%m-%d')] = df + + return cumulated_data + + except Exception as e: + print("Exception in getting meps documents in Mongo" + str(e)) + return {"ged_insert_status": "Exception in getting meps documents in Mongo" + str(e)} diff --git a/Code/public_api.py b/Code/public_api.py index fc72a2e..465ad0c 100644 --- a/Code/public_api.py +++ b/Code/public_api.py @@ -615,8 +615,15 @@ def wild_card(word_to_search: str) : query['Date'] = {"$lte": end_date} try: - df = mongo_handler.get_df(db_name=db_name, collection_name=collection_name, query=query) - + df = mongo_handler.get_df(db_name=db_name, + collection_name=collection_name, + query=query) + + dfs_grouped_by_month = mongo_handler.get_df_grouped_by_month(db_name=db_name, + collection_name=collection_name, + query=query, + date_start=start_date, + date_end=end_date) return meps_handler.get_stats(df) except Exception as e: From eee697accf0577df4cdcafcc40e5863152185fce Mon Sep 17 00:00:00 2001 From: Robin Cohen-Selmon <87440088+RobsOnWaves@users.noreply.github.com> Date: Sun, 28 Jan 2024 22:57:24 +0100 Subject: [PATCH 2/3] adding export monthly data to stats --- Code/libs/mongo_db_handler.py | 6 +----- Code/public_api.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Code/libs/mongo_db_handler.py b/Code/libs/mongo_db_handler.py index 9fbdb79..28c13fa 100644 --- a/Code/libs/mongo_db_handler.py +++ b/Code/libs/mongo_db_handler.py @@ -394,12 +394,8 @@ def get_last_day_of_month(date): if 'Date' in df.columns: df['Date'] = df['Date'].dt.strftime('%d/%m/%Y') - else: - print('No Date column') - if not data: - print('No data for this date' + str(date)) - else: + if data: cumulated_data[date[0].strftime('%Y-%m-%d')] = df return cumulated_data diff --git a/Code/public_api.py b/Code/public_api.py index 465ad0c..6e2036f 100644 --- a/Code/public_api.py +++ b/Code/public_api.py @@ -57,7 +57,7 @@ class Roles(str, Enum): print(e) ALGORITHM = "HS256" -ACCESS_TOKEN_EXPIRE_MINUTES = 30 +ACCESS_TOKEN_EXPIRE_MINUTES = 1440 JSON_EXTENSION = ".json" @@ -624,7 +624,13 @@ def wild_card(word_to_search: str) : query=query, date_start=start_date, date_end=end_date) - return meps_handler.get_stats(df) + dfs_grouped_by_month_stats = {} + for date in dfs_grouped_by_month.keys(): + dfs_grouped_by_month_stats[date] = meps_handler.get_stats(dfs_grouped_by_month[date]) + + global_stats = meps_handler.get_stats(df) + global_stats['meps_stats_grouped_by_month'] = dfs_grouped_by_month_stats + return global_stats except Exception as e: print("get_meps_stats : " + str(e), flush=True) From 6459fbed797879f9545592c33bffa884f5b47081 Mon Sep 17 00:00:00 2001 From: Robin Cohen-Selmon <87440088+RobsOnWaves@users.noreply.github.com> Date: Thu, 1 Feb 2024 00:52:10 +0100 Subject: [PATCH 3/3] fixing date comparison --- Code/libs/mongo_db_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/libs/mongo_db_handler.py b/Code/libs/mongo_db_handler.py index 28c13fa..d668f65 100644 --- a/Code/libs/mongo_db_handler.py +++ b/Code/libs/mongo_db_handler.py @@ -381,7 +381,7 @@ def get_last_day_of_month(date): current_date = get_first_day_of_next_month(date_start) dates = [] - while current_date < date_end: + while current_date < date_end.replace(tzinfo=None): start_of_month = current_date end_of_month = get_last_day_of_month(current_date) dates.append((start_of_month, end_of_month))