Merge pull request #59 from RobsOnWaves/develop

adding filters for export file

Code/libs/meps_handler.py

@@ -20,6 +20,10 @@ def __init__(self):
         self.__messages__ = Messages()
         self.__max_length__ = 1000
         self.__timeout_duration__ = 60
+        self.__mep_field_list__ = ["MEP Name", "MEP nationalPoliticalGroup", "MEP politicalGroup", "Title", "Place",
+                                   "Meeting With", "Meeting Related to Procedure"]
+        self.__mep_db_name__ = "MEPS"
+        self.__collection_name__ = "meps_meetings"
 
     class TimeoutException(Exception):
             pass
@@ -52,3 +56,11 @@ async def load_csv_file(self, upload_file: UploadFile, answer: dict = None):
             # Arrête le timer
             timer.cancel()
 
+    def get_mep_field_list(self):
+        return self.__mep_field_list__
+
+    def get_mep_db_name(self):
+        return self.__mep_db_name__
+
+    def get_mep_collection_name(self):
+        return self.__collection_name__

Code/libs/mongo_db_handler.py

@@ -278,20 +278,55 @@ def from_df_to_mongo_meps(self, collection_name: str, df: pd.DataFrame):
             print("Exception in pushing meps documents in Mongo" + str(e))
             return {"ged_insert_status": "Exception in pushing meps documents in Mongo" + str(e)}
 
-    def from_mongo_to_xlsx_meps(self):
-        db = self.__mongo_client__.MEPS
-        # Récupération des données
-        collection = db.meps_meetings  # Nom de la collection
+    def from_mongo_to_xlsx(self, db_name: str, collection_name: str, query: dict):
+
+        client = self.__mongo_client__
+        db = client[db_name]
+        collection = db[collection_name]
 
         try:
-            data = list(collection.find({}, {'_id': False}))
+            data = list(collection.find(query, {'_id': False}))
             df = pd.DataFrame(data)
-            df['Date'] = df['Date'].dt.strftime('%d/%m/%Y')
+            if 'Date' in df.columns:
+                df['Date'] = df['Date'].dt.strftime('%d/%m/%Y')
+            else:
+                print('No Date column')
+
             # Création d'un fichier Excel
-            excel_file_path = 'meps_fichier.xlsx'  # Spécifiez le chemin et le nom de fichier souhaités
+            excel_file_path = 'export_file.xlsx'  # Spécifiez le chemin et le nom de fichier souhaités
             df.to_excel(excel_file_path, index=False)
             return True
 
         except Exception as e:
             print("Exception in getting meps documents in Mongo" + str(e))
             return {"ged_insert_status": "Exception in getting meps documents in Mongo" + str(e)}
+
+    def get_unique_values(self, db_name: str, collection_name: str, fields: list):
+
+        # Connexion à la base de données MongoDB
+        client = self.__mongo_client__
+        db = client[db_name]
+        collection = db[collection_name]
+
+        # Initialiser un dictionnaire pour stocker les valeurs uniques
+        # Dictionnaire pour stocker les valeurs dédupliquées pour chaque champ
+        valeurs_dedupliquees = {}
+
+        # Récupérer les valeurs dédupliquées pour chaque champ
+        for field in fields:
+            valeurs = collection.distinct(field)
+            valeurs_conformes = []
+            for val in valeurs:
+                if not isinstance(val, str):
+                    val = str(val)
+                # Tronquer la chaîne si elle dépasse 50 caractères et ajouter "..."
+                if len(val) > 50:
+                    val = val[:50] + "..."
+                valeurs_conformes.append(val)
+
+            valeurs_dedupliquees[field] = valeurs_conformes
+
+
+        return valeurs_dedupliquees
+
+

Code/public_api.py

@@ -21,6 +21,8 @@
 import logging
 from pythonjsonlogger import jsonlogger
 import time
+from typing import Optional
+
 
 class Roles(str, Enum):
     admin = "admin"
@@ -54,7 +56,6 @@ class Roles(str, Enum):
     print("Error getting SECRET_KEY")
     print(e)
 
-
 ALGORITHM = "HS256"
 ACCESS_TOKEN_EXPIRE_MINUTES = 30
 JSON_EXTENSION = ".json"
@@ -101,13 +102,15 @@ class UserInDB(User):
 
 oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
 
+
 def get_user_id_from_token(token: str):
     try:
         payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
         return payload.get("sub")  # 'sub' est généralement l'identifiant de l'utilisateur
     except jwt.JWTError:
         return None
 
+
 def log_to_json_file(log_data):
     log_file_path = 'logz.json'
     try:
@@ -260,7 +263,7 @@ async def get_current_active_user(current_user: User = Depends(get_current_user)
 async def get_request_body(request: Request):
     body = await request.body()
 
-    async def app(scope, receive, send):
+    async def app(scope, send):
         async def override_receive():
             return {"type": "http.request", "body": body}
 
@@ -284,6 +287,7 @@ def mask_sensitive_data_and_exclude_files(body: str) -> str:
 
     return body
 
+
 @app.middleware("http")
 async def log_requests(request: Request, call_next):
     start_time = time.time()
@@ -304,22 +308,22 @@ async def log_requests(request: Request, call_next):
         token = await oauth2_scheme(request)
         user_id = get_user_id_from_token(token) if token else "anonymous"
     except Exception as e:
-        user_id = "error_in_token"
+        user_id = "error_in_token" + str(e)
 
     # Continuer le traitement de la requête
     response = await call_next(request)
     process_time = (time.time() - start_time) * 1000
     # Logger les informations
     logger.info('Request info', extra={
-    "timestamp": datetime.fromtimestamp(start_time).isoformat(),
-    "user_id": user_id,
-    'request_method': request.method,
-    'request_url': str(request.url),
-    'response_status': response.status_code,
-    'process_time_ms': process_time,
+        "timestamp": datetime.fromtimestamp(start_time).isoformat(),
+        "user_id": user_id,
+        'request_method': request.method,
+        'request_url': str(request.url),
+        'response_status': response.status_code,
+        'process_time_ms': process_time,
         'request_body': body_text,
         # Autres informations...
-        })
+    })
 
     return response
 
@@ -360,7 +364,7 @@ async def create_user(user_name: str = Form(),
                       current_user: User = Depends(get_current_active_user)):
     if current_user.role == "admin":
         return mongo_handler.insert_user(user_name, full_name, email, get_password_hash(password),
-                                  current_user.username, role)
+                                         current_user.username, role)
     else:
         return {'response': 'Access denied'}
 
@@ -498,7 +502,7 @@ async def load_meps_file(file: UploadFile, current_user: User = Depends(get_curr
         await meps_handler.load_csv_file(upload_file=file, answer=answer)
 
         if not answer['success']:
-            raise HTTPException(status_code=403, detail=messages.denied_entry)
+            raise HTTPException(status_code=404, detail=messages.nok_string_raw)
         else:
             mongo_handler.from_df_to_mongo_meps(df=answer['df'], collection_name="meps_meetings")
             return {'response': messages.build_ok_action_string(user_name=current_user.username)}
@@ -507,17 +511,74 @@ async def load_meps_file(file: UploadFile, current_user: User = Depends(get_curr
 
 
 @app.get("/meps_file",
-          description="loads a file with the list pression groups meetings of MEPs into the database")
-async def get_meps_file(current_user: User = Depends(get_current_active_user)):
+         description="loads a file with the list pression groups meetings of MEPs into the database")
+async def get_meps_file(mep_name: Optional[str] = None,
+                        national_political_group: Optional[str] = None,
+                        political_group: Optional[str] = None,
+                        title: Optional[str] = None,
+                        place: Optional[str] = None,
+                        meeting_with: Optional[str] = None,
+                        start_date: Optional[datetime] = None,
+                        end_date: Optional[datetime] = None,
+                        current_user: User = Depends(get_current_active_user)):
     if current_user.role in ['admin', 'meps']:
-        mongo_handler.from_mongo_to_xlsx_meps()
-        if mongo_handler.from_mongo_to_xlsx_meps():
-            return FileResponse('meps_fichier.xlsx')
+        db_name = meps_handler.get_mep_db_name()
+        collection_name = meps_handler.get_mep_collection_name()
+
+        def wild_card(word_to_search: str) :
+            word_to_search = re.escape(word_to_search)
+            return {"$regex": ".*" + word_to_search + ".*", "$options": "i"}
+
+        query = {
+            'MEP Name': wild_card(mep_name) if mep_name is not None else wild_card(''),
+            'MEP nationalPoliticalGroup': wild_card(national_political_group) if national_political_group is not None else wild_card(''),
+            'MEP politicalGroup': wild_card(political_group) if political_group is not None else wild_card(''),
+            'Title': wild_card(title) if title is not None else wild_card(''),
+            'Place': wild_card(place) if place is not None else wild_card(''),
+            'Meeting With': wild_card(meeting_with) if meeting_with is not None else wild_card('')
+        }
+
+        if start_date and end_date:
+            query['Date'] = {"$gte": start_date, "$lte": end_date}
+        elif start_date:
+            query['Date'] = {"$gte": start_date}
+        elif end_date:
+            query['Date'] = {"$lte": end_date}
+
+        if mongo_handler.from_mongo_to_xlsx(db_name=db_name, collection_name=collection_name, query=query):
+            return FileResponse('export_file.xlsx')
         else:
-            raise HTTPException(status_code=403, detail=messages.denied_entry)
+            raise HTTPException(status_code=404, detail=messages.nok_string_raw)
     else:
         raise HTTPException(status_code=403, detail=messages.denied_entry)
 
+
+@app.get("/meps_file_fields_values",
+         description="gets a file with the list of the values of fields")
+async def get_meps_file_selected_fields(current_user: User = Depends(get_current_active_user)):
+    if current_user.role in ['admin', 'meps']:
+        db_name = meps_handler.get_mep_db_name()
+        fields = meps_handler.get_mep_field_list()
+        collection_name = meps_handler.get_mep_collection_name()
+
+        try:
+            values = mongo_handler.get_unique_values(db_name=db_name, collection_name=collection_name, fields=fields)
+        except Exception as e:
+            print("get_meps_file_selected_fields : " + str(e), flush=True)
+            raise HTTPException(status_code=404, detail=messages.nok_string_raw)
+
+        if values is not None:
+            try:
+                return values
+            except Exception as e:
+                print("get_meps_file_selected_fields : " + str(e), flush=True)
+                raise HTTPException(status_code=404, detail=messages.nok_string_raw)
+        else:
+            raise HTTPException(status_code=404, detail=messages.nok_string_raw)
+    else:
+        raise HTTPException(status_code=403, detail=messages.denied_entry)
+
+
 @app.post("/logout")
 async def logout():
     return {"message": "Disconnected, please log in again"}

DevOps/apps/stacked_apps/docker-compose.yaml

@@ -15,6 +15,13 @@ services:
 
       'PWD_MONGO': "rootmongopwd"
 
+      'SECRET_KEY': "11088b752484acda51943b487d8657e142e91e085187c110e0967650e7526784"
+
+      'URL_FRONT': 'http://localhost:8080'
+
+    volumes:
+      - $PWD/logs_preprod.log:/usr/src/app/app_logs.json
+
     ports:
       - "5555:8000"