Merge pull request #53 from RobsOnWaves/develop

Develop

Code/libs/meps_handler.py

@@ -0,0 +1,52 @@
+import datetime
+import re
+import threading
+from io import BytesIO
+
+import numpy as np
+import pandas as pd
+from docx import Document
+from fastapi import UploadFile
+from libs.messages import Messages
+from openpyxl import load_workbook
+from openpyxl.styles import Alignment
+from unidecode import unidecode
+import shutil
+import tempfile
+
+
+class MepsHandler:
+    def __init__(self):
+        self.__messages__ = Messages()
+        self.__max_length__ = 1000
+        self.__timeout_duration__ = 60
+
+    class TimeoutException(Exception):
+            pass
+
+    def timeout_handler(self):
+        raise self.TimeoutException()
+
+    async def load_csv_file(self, upload_file: UploadFile, answer: dict = None):
+
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as temp_file:
+            # Copier le contenu de l'objet UploadFile dans le fichier temporaire
+            shutil.copyfileobj(upload_file.file, temp_file)
+            temp_file_path = temp_file.name
+
+        # Extract the table
+        timer = threading.Timer(self.__timeout_duration__, self.timeout_handler)
+        try:
+            timer.start()
+            df = pd.read_csv(temp_file_path)
+            answer["df"] = df
+            answer["success"] = True
+            return answer
+        except self.TimeoutException:
+            answer["df"] = None
+            answer["success"] = False
+            return self.__messages__.nok_string
+        finally:
+            # Arrête le timer
+            timer.cancel()
+

Code/libs/messages.py

@@ -13,10 +13,15 @@ class Messages:
 
     __ok_string_user_modified_right_side__ = "' modified" + emojize(":kiss:", language='alias')
 
+    __ok_string_user_action_ok__ = "' action completed with success" + emojize(":kiss:", language='alias')
+
     denied_entry = emojize(":no_entry:", language="alias") + "you didn't say the magic word"
 
     def build_ok_user_string(self, user_name: str = ""):
         return self.__ok_string_user_left_side__ + user_name + self.__ok_string_user_right_side__
 
+    def build_ok_action_string(self, user_name: str = ""):
+        return self.__ok_string_user_left_side__ + user_name + self.__ok_string_user_action_ok__
+
     def build_ok_user_modified_string(self, user_name: str = ""):
         return self.__ok_string_user_left_side__ + user_name + self.__ok_string_user_modified_right_side__

Code/libs/mongo_db_handler.py

@@ -1,6 +1,8 @@
+import pymongo
 from pymongo import MongoClient
 from libs.ged_file_handler import GedFileHandler
 from libs.messages import Messages
+import pandas as pd
 import datetime
 import copy
 
@@ -24,7 +26,7 @@ def get_gold_coeffs(self):
             return gold_coeffs
 
         except Exception as e:
-            return self.__exception_message__+ str(e)
+            return self.__exception_message__ + str(e)
 
     @staticmethod
     def from_ged_dict_to_mongodb_dict(ged_handler: GedFileHandler = GedFileHandler(),
@@ -174,7 +176,7 @@ def get_users(self):
                     end_cursor = True
 
         except Exception as e:
-            return self.__exception_message__+ str(e)
+            return self.__exception_message__ + str(e)
 
         return users
 
@@ -228,7 +230,7 @@ def get_collections(self):
             return {"collection_names": collection_names}
 
         except Exception as e:
-            return self.__exception_message__+ str(e)
+            return self.__exception_message__ + str(e)
 
     def modify_user_password(self,
                              user_name: str,
@@ -250,3 +252,45 @@ def modify_user_password(self,
 
         return self.__messages__.build_ok_user_modified_string(user_name=user_name) if status.acknowledged else \
             self.__messages__.nok_string
+
+    def from_df_to_mongo_meps(self, collection_name: str, df: pd.DataFrame):
+        db = self.__mongo_client__.MEPS
+
+        collection_handler = getattr(db, collection_name)
+
+        try:
+            collection_handler.create_index(
+                [
+                    ('MEP Name', pymongo.ASCENDING),
+                    ('MEP nationalPoliticalGroup', pymongo.ASCENDING),
+                    ('MEP politicalGroup', pymongo.ASCENDING),
+                    ('Title', pymongo.ASCENDING),
+                    ('Date', pymongo.ASCENDING),
+                    ('Place', pymongo.ASCENDING),
+                    ('Capacity', pymongo.ASCENDING),
+                    ('Meeting With', pymongo.ASCENDING),
+                    ('Meeting Related to Procedure', pymongo.ASCENDING)
+                ], unique = True)
+
+            collection_handler.insert_many(df.to_dict('records'), ordered=False)
+
+        except Exception as e:
+            print("Exception in pushing meps documents in Mongo" + str(e))
+            return {"ged_insert_status": "Exception in pushing meps documents in Mongo" + str(e)}
+
+    def from_mongo_to_xlsx_meps(self):
+        db = self.__mongo_client__.MEPS
+        # Récupération des données
+        collection = db.meps_meetings  # Nom de la collection
+
+        try:
+            data = list(collection.find({}, {'_id': False}))
+            df = pd.DataFrame(data)
+            # Création d'un fichier Excel
+            excel_file_path = 'meps_fichier.xlsx'  # Spécifiez le chemin et le nom de fichier souhaités
+            df.to_excel(excel_file_path, index=False)
+            return True
+
+        except Exception as e:
+            print("Exception in getting meps documents in Mongo" + str(e))
+            return {"ged_insert_status": "Exception in getting meps documents in Mongo" + str(e)}

Code/public_api.py

@@ -13,32 +13,41 @@
 from libs.ged_file_handler import GedFileHandler
 from libs.messages import Messages
 from libs.gold_digger import GoldDigger
+from libs.meps_handler import MepsHandler
 from fastapi.middleware.cors import CORSMiddleware
 import re
 from pathlib import Path
 
+
 class Roles(str, Enum):
     admin = "admin"
     user = "user"
+    gold_digger = "gold_digger"
+    meps = "meps"
 
 
 # to get a string like this run:
 # openssl rand -hex 32
-if os.environ["SECRET_KEY"]:
+try:
     SECRET_KEY = os.environ["SECRET_KEY"]
-else:
+    print("SECRET_KEY set, using it")
+except KeyError:
     SECRET_KEY = "11088b752484acda51943b487d8657e142e91e085187c110e0967650e7526784"
+    print("SECRET_KEY not set, using default")
+except Exception as e:
+    print("Error getting SECRET_KEY")
+    print(e)
 
 ALGORITHM = "HS256"
 ACCESS_TOKEN_EXPIRE_MINUTES = 30
-
 JSON_EXTENSION = ".json"
 
 
 class Token(BaseModel):
     access_token: str
     token_type: str
     name: str
+    role: str
 
 
 class TokenData(BaseModel):
@@ -88,9 +97,9 @@ def el_parametrizor(mode_debug=False):
 mongo_handler = MongoDbGed(address=os.environ['URL_MONGO'], user=os.environ['USR_MONGO'],
                            password=os.environ['PWD_MONGO'])
 
-
 messages = Messages()
 gold_handler = GoldDigger()
+meps_handler = MepsHandler()
 
 
 def time_window_control(date_start: datetime, date_end: datetime, current_user: User):
@@ -104,7 +113,7 @@ def time_window_control(date_start: datetime, date_end: datetime, current_user:
         time_window_validated = False
 
     elif date_start > date_end:
-        status_date = "hi " + str(current_user.username) + messages.nok_string +\
+        status_date = "hi " + str(current_user.username) + messages.nok_string + \
                       " you can't finish before you start"
 
         time_window_validated = False
@@ -116,7 +125,7 @@ def time_window_control(date_start: datetime, date_end: datetime, current_user:
         time_window_validated = False
 
     elif date_start > datetime.utcnow() or date_end > datetime.utcnow():
-        status_date = "hi " + str(current_user.username) + messages.nok_string +\
+        status_date = "hi " + str(current_user.username) + messages.nok_string + \
                       " ged-handler is futuristic but does not accept dates in the future"
         time_window_validated = False
     else:
@@ -133,6 +142,7 @@ def sanitize_filename(filename: str):
     """
     return re.sub(r'[^a-zA-Z0-9_.-]', '_', filename)
 
+
 def verify_password(plain_password, hashed_password):
     return pwd_context.verify(plain_password, hashed_password)
 
@@ -144,6 +154,7 @@ def secure_file_path(filename: str, directory="tmp/"):
     sanitized_filename = sanitize_filename(filename)
     return os.path.join(directory, sanitized_filename)
 
+
 def get_password_hash(password):
     return pwd_context.hash(password)
 
@@ -228,7 +239,7 @@ async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends(
     access_token = create_access_token(
         data={"sub": user.username}, expires_delta=access_token_expires
     )
-    return {"access_token": access_token, "name": user.full_name, "token_type": "bearer"}
+    return {"access_token": access_token, "name": user.full_name, "token_type": "bearer", "role": user.role}
 
 
 @app.get("/users/me/", response_model=User, description="Returns information about the current logged in user")
@@ -268,15 +279,15 @@ async def upload_ged_file(file: UploadFile,
 @app.get("/ged_stored_collection_to_json_answer", description="Returns a JSON answer from a stored collection")
 async def ged_stored_collection_to_json_answer(ged_collection_name: str,
                                                current_user: User = Depends(get_current_active_user)):
-
     if current_user.role in ['admin', 'user']:
         return mongo_handler.from_mongo_to_ged_list_dict(collection_name=ged_collection_name)
     else:
         return {'response': messages.nok_string}
 
 
 @app.get("/ged_stored_collection_to_json_file", description="Returns a JSON file from a stored collection")
-async def ged_stored_collection_to_json_file(ged_collection_name: str, current_user: User = Depends(get_current_active_user)):
+async def ged_stored_collection_to_json_file(ged_collection_name: str,
+                                             current_user: User = Depends(get_current_active_user)):
     if current_user.role in ['admin', 'user']:
         safe_path = secure_file_path(ged_collection_name + JSON_EXTENSION)
         with open(safe_path, 'w') as convert_file:
@@ -293,7 +304,6 @@ async def ged_stored_collection_to_json_file(ged_collection_name: str, current_u
                                                   " without storing it in the database")
 async def ged_collection_to_json_answer(file: UploadFile,
                                         current_user: User = Depends(get_current_active_user)):
-
     if current_user.role in ['admin', 'user']:
 
         ged_handler = GedFileHandler()
@@ -307,7 +317,6 @@ async def ged_collection_to_json_answer(file: UploadFile,
 
 @app.get("/ged_stored_collections", description="Returns a list of all stored collections")
 async def ged_stored_collections(current_user: User = Depends(get_current_active_user)):
-
     if current_user.role in ['admin', 'user']:
 
         return mongo_handler.get_collections()
@@ -321,7 +330,6 @@ async def ged_stored_collections(current_user: User = Depends(get_current_active
 async def ged_collection_to_json_file(file: UploadFile,
                                       current_user: User = Depends(get_current_active_user)
                                       ):
-
     if current_user.role in ['admin', 'user']:
 
         ged_handler = GedFileHandler()
@@ -342,11 +350,10 @@ async def ged_collection_to_json_file(file: UploadFile,
 
 @app.post("/modify_user_password", description="Modify an exiting user password, restricted to admin privileges")
 async def modify_user_password(
-                                          user_name: str = Form(description="user name that needs its password to "
-                                                                            "be modified"),
-                                          password: str = Form(min_length=10, description="mini. 10 characters"),
-                                          current_user: User = Depends(get_current_active_user)):
-
+        user_name: str = Form(description="user name that needs its password to "
+                                          "be modified"),
+        password: str = Form(min_length=10, description="mini. 10 characters"),
+        current_user: User = Depends(get_current_active_user)):
     if current_user.role in ['admin']:
 
         return {'response': mongo_handler.modify_user_password(user_name=user_name,
@@ -357,8 +364,9 @@ async def modify_user_password(
 
 
 @app.post("/gold_file_converter", description="Returns an Excel with the estimated value in euros")
-async def gold_file_converter(file: UploadFile, price_per_kg: int, current_user: User = Depends(get_current_active_user)):
-    if current_user.role in ['admin', 'user']:
+async def gold_file_converter(file: UploadFile, price_per_kg: int,
+                              current_user: User = Depends(get_current_active_user)):
+    if current_user.role in ['admin', 'gold_digger']:
         coeffs = mongo_handler.get_gold_coeffs()
         # Génération d'un nom de fichier sécurisé pour le fichier Excel
         timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
@@ -370,13 +378,45 @@ async def gold_file_converter(file: UploadFile, price_per_kg: int, current_user:
             raise HTTPException(status_code=400, detail="File already exists")
 
         # Appel à la méthode de calcul en passant le chemin sécurisé
-        await gold_handler.compute_excel_file(upload_file=file, price_per_kg=price_per_kg, gold_coeffs=coeffs, output_file=full_safe_path)
+        await gold_handler.compute_excel_file(upload_file=file, price_per_kg=price_per_kg, gold_coeffs=coeffs,
+                                              output_file=full_safe_path)
 
         return FileResponse(full_safe_path)
     else:
         return {'response': messages.nok_string}
 
 
+@app.post("/meps_file",
+          description="loads a file with the list pression groups meetings of MEPs into the database")
+async def load_meps_file(file: UploadFile, current_user: User = Depends(get_current_active_user)):
+    if current_user.role in ['admin']:
+        # Génération d'un nom de fichier sécurisé pour le fichier Excel
+        answer = {}
+        # Appel à la méthode de calcul en passant le chemin sécurisé
+        await meps_handler.load_csv_file(upload_file=file, answer=answer)
+
+        if not answer['success']:
+            return {'response': messages.nok_string}
+        else:
+            mongo_handler.from_df_to_mongo_meps(df=answer['df'], collection_name="meps_meetings")
+            return {'response': messages.build_ok_action_string(user_name=current_user.username)}
+    else:
+        return {'response': messages.denied_entry}
+
+
+@app.get("/meps_file",
+          description="loads a file with the list pression groups meetings of MEPs into the database")
+async def get_meps_file(current_user: User = Depends(get_current_active_user)):
+    if current_user.role in ['admin', 'meps']:
+        mongo_handler.from_mongo_to_xlsx_meps()
+        if mongo_handler.from_mongo_to_xlsx_meps():
+            return FileResponse('meps_fichier.xlsx')
+        else:
+            return {'response': messages.nok_string}
+    else:
+        return {'response': messages.denied_entry}
+
+
 @app.post("/logout")
 async def logout():
     return {"message": "Disconnected, please log in again"}