From 0538232fe5158b070a52305cd7c8e94f299c68e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Veljko=20Tekelerovi=C4=87?= Date: Fri, 14 May 2021 19:09:03 +0200 Subject: [PATCH] DB modules update (#6) Added services for MySQL and MongoDB Basic routes exposition and simple use case Co-authored-by: salvatore287 Signed-off-by: vexy --- .gitignore | 10 +++ LICENSE | 0 README.md | 31 +++++-- __init__.py | 0 _config.yml | 0 install-dependencies.sh | 0 main-module.py | 41 ++++++++- models/__init__.py | 0 models/user.py | 0 modules/__init__.py | 0 modules/auth.py | 8 +- modules/protected.py | 14 +-- paw_testkit.paw | Bin requirements.txt | 9 +- services/__init__.py | 0 services/mongodb.py | 111 +++++++++++++++++++++++ services/mysql.py | 192 ++++++++++++++++++++++++++++++++++++++++ services/storage.py | 0 services/tokenizer.py | 0 start.sh | 0 20 files changed, 401 insertions(+), 15 deletions(-) mode change 100644 => 100755 .gitignore mode change 100644 => 100755 LICENSE mode change 100644 => 100755 README.md mode change 100644 => 100755 __init__.py mode change 100644 => 100755 _config.yml mode change 100644 => 100755 install-dependencies.sh mode change 100644 => 100755 main-module.py mode change 100644 => 100755 models/__init__.py mode change 100644 => 100755 models/user.py mode change 100644 => 100755 modules/__init__.py mode change 100644 => 100755 modules/auth.py mode change 100644 => 100755 modules/protected.py mode change 100644 => 100755 paw_testkit.paw mode change 100644 => 100755 requirements.txt mode change 100644 => 100755 services/__init__.py create mode 100755 services/mongodb.py create mode 100755 services/mysql.py mode change 100644 => 100755 services/storage.py mode change 100644 => 100755 services/tokenizer.py mode change 100644 => 100755 start.sh diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 index 5bb5b49..8b83fdf --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,13 @@ __pycache__/ # MacOS stuff .DS_Store + +# VS Code Workspace +*.code-workspace +.vscode +_config.yml + +# Virtual Env +bin +pyvenv.cfg +lib diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 6e1d71d..a65496b --- a/README.md +++ b/README.md @@ -9,15 +9,33 @@ Use this template as starting point for more complex projects and requirements. `JSON Web Tokens` - or [JWT](https://jwt.io/) in short - is the foundation authentication principle used in this template. Be sure **not to forget** to encode/decode token generation at your own strategy. Follow code comments for exact place where you could modify or customise this behaviour. -### No database ! -DB layer has been **intentionally omitted** to allow space for your own implementation. In present form, the code handles all tokens **in memory**, making the data available only while the server is running. All registration data (as well as tokens) will disappear after the server shut down. +### Separate Database layer +DB layer has been split into separate `services` folder. It provides basic Python wrappers for `MySql` and `MongoDB` engines. +In basic form, the code handles all authentication tokens **in memory**, making the data available only while the server is running. All registration data (as well as tokens) will disappear after the server shut down. For more convenient mechanism, store your tokens in some form of persistent storage, or reuse them in different way. +Data handling services supported so far: +1. SharedStorage (trivial implementation of in-memory storage) +2. MySQL wrapper +3. MongoDB wrapper + ### Modularised -Template is designed to support modular structure. Main application modules are stored in `modules` folder. If you need more modules, you can place them inside - as long as they are connected in the main module. +Template is designed to support modular structure. +Following structure is used: +``` + - flask-auth-template + | + | + / services (# contain various services) + / modules (# contain various additional modules) + / models (# contain data models used in this template) +``` + +*NOTE:* +Main application modules are stored in `modules` folder. If you need more modules, you can place them inside - as long as they are connected in the main module. Customize your Flask bluperints to support modularized approach you need. ### Different authentication strategies -Presented here is basic HTTP AUTHENTICATION through Authentication field. Note there are **way secure** authentication mechanisms, such as `OAuth`. +Presented here is **basic** HTTP AUTHENTICATION through Authentication field. Note there are **way secure** authentication mechanisms, such as `OAuth`. #### CORS setup For the sake of simplicity, CORS has been enabled completely. Server will accept all origins no matter where the request comes from. Check and/or modify `@app.after_request` directive to further customise desired behaviour (lines [20-24](https://github.com/vexy/flask-auth-template/blob/master/main-module.py#L20-L24) in `main-module.py`). @@ -32,9 +50,12 @@ Then proceed with installing dependencies: ```bash # Run prepacked script $ . install-dependencies.sh -# or + # install manually through pip3 $ pip3 install -r requirements.txt + +# or +python3.8 -m pip instal -r requirements.txt ``` ### Starting server diff --git a/__init__.py b/__init__.py old mode 100644 new mode 100755 diff --git a/_config.yml b/_config.yml old mode 100644 new mode 100755 diff --git a/install-dependencies.sh b/install-dependencies.sh old mode 100644 new mode 100755 diff --git a/main-module.py b/main-module.py old mode 100644 new mode 100755 index 703f5b1..c74963c --- a/main-module.py +++ b/main-module.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- +from services.mongodb import Database from flask import Flask, Blueprint + +# services.* are used as DB hooks below from services.storage import sharedStorage from modules.auth import * @@ -15,8 +18,9 @@ # register app blueprints app.register_blueprint(authRoute) app.register_blueprint(protectedRoute) + # register DB routes later -# make sure this is turned off +# make sure this is turned or configured according to your needs @app.after_request def attachCORSHeader(response): response.headers.set('Access-Control-Allow-Headers', '*') @@ -27,6 +31,8 @@ def attachCORSHeader(response): # ------------------------------ @app.route('/') def home(): + # This route returns a JSON of "users in the system" + # Replace it with your own logic output = [] for user in sharedStorage.asList(): output.append(str(user)) @@ -35,6 +41,39 @@ def home(): 'storage': output }) +# Publicly accessible routes with DB support +# ------------------------------ +@app.route('/mongo_db') +def mongo_db(): + from services.mongodb import Database + + # This route returns a list of data from a "User" collection + # it assumes having valid MongoDB connection + # Replace it with your own logic + mongoClient = Database("localhost", "user", "pwd") + + output = [] + output = mongoClient.filter("mainCollection", "{'name': 'someName'}") + + # format JSON response + response = jsonify({'results': output}) + return response + +@app.route('/sql_db') +def mongo_db(): + from services.mysql import Database + + # This route returns a list of data from a "User" collection + # it assumes having valid MongoDB connection + # Replace it with your own logic + sqlClient = Database() + + output = [] + output = sqlClient.filter("someTable", "user123") + + # format JSON response + response = jsonify({'results': output}) + return response # --------------------------------- # Server start procedure diff --git a/models/__init__.py b/models/__init__.py old mode 100644 new mode 100755 diff --git a/models/user.py b/models/user.py old mode 100644 new mode 100755 diff --git a/modules/__init__.py b/modules/__init__.py old mode 100644 new mode 100755 diff --git a/modules/auth.py b/modules/auth.py old mode 100644 new mode 100755 index 6c1b3fb..2939dc4 --- a/modules/auth.py +++ b/modules/auth.py @@ -9,6 +9,7 @@ # public blueprint exposure authRoute = Blueprint('auth', __name__) +# 👇 implement your strategy here 👇 @authRoute.route('/login', methods=['POST']) def login(): # get authorization field from HTTP request, early exit if it's not present @@ -36,12 +37,14 @@ def login(): return make_response("Wrong credentials.", 401) +# 👇 implement your strategy here 👇 @authRoute.route('/logout') def logout(): current_app.logger.info("Someone logged out") - # remove token from the storage + # eg. remove/invalidate token from our storage return "You have been logged out.. But who are you ??" +# 👇 implement your strategy here 👇 @authRoute.route('/register', methods=['POST']) def registration(): ''' @@ -56,10 +59,11 @@ def registration(): body = request.json if body: username = body['username'] - pwd = body['password'] # 👈 add password hashing strategy here + pwd = body['password'] email = body['email'] # add to our storage + # 👈 add password hashing strategy here before saving to DB newUser = User(username, pwd, email) current_app.logger.info(f" Adding new user: {newUser.username}, email: {newUser.email}") diff --git a/modules/protected.py b/modules/protected.py old mode 100644 new mode 100755 index fd92393..97bdf46 --- a/modules/protected.py +++ b/modules/protected.py @@ -2,9 +2,11 @@ from flask import Flask, Blueprint from flask import jsonify, request, make_response from flask import current_app + from services.tokenizer import Tokenizer -from services.storage import sharedStorage -from models.user import User +from services.storage import sharedStorage # CONNECT WITH DIFFERENT DB SERVICES HERE + +from models.user import User # REPLACE WITH YOUR OWN MODELS IF NEEDED from functools import wraps # public blueprint exposure @@ -14,7 +16,7 @@ def token_access_required(f): @wraps(f) def decorated(*args, **kwargs): - token = request.args.get('token') #get token from URL + token = request.args.get('token') # get token from URL if not token: return jsonify({'message': 'Protected area. Valid access token required'}), 403 @@ -35,14 +37,14 @@ def decorated(*args, **kwargs): # Protected ROUTES DEFINITION: (split further to standalone Blueprints) # ----------------------------- -@protectedRoute.route('/protected') +@protectedRoute.route('/protected1') @token_access_required def protected(): - resp_body = jsonify({'message': 'Welcome to protected area, you made it'}) + resp_body = jsonify({'message': 'Welcome to protected area 1, you made it'}) return resp_body @protectedRoute.route('/protected2') @token_access_required def protected2(): - resp_body = jsonify({'message': 'Welcome to protected area, you made it'}) + resp_body = jsonify({'message': 'Welcome to protected area 2, you made it'}) return resp_body diff --git a/paw_testkit.paw b/paw_testkit.paw old mode 100644 new mode 100755 diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 index c250825..41662d3 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,9 @@ -Flask==1.1.1 +click==7.1.2 +Flask==1.1.2 +itsdangerous==1.1.0 +Jinja2==2.11.3 +MarkupSafe==1.1.1 PyJWT==1.7.1 +pymongo==3.11.2 +Werkzeug==1.0.1 +PyMySQL==1.0.2 diff --git a/services/__init__.py b/services/__init__.py old mode 100644 new mode 100755 diff --git a/services/mongodb.py b/services/mongodb.py new file mode 100755 index 0000000..9fac971 --- /dev/null +++ b/services/mongodb.py @@ -0,0 +1,111 @@ +from pymongo import MongoClient +import datetime + +class DBError(Exception): + """ raised on critical db errors """ + pass + +class DBTypeError(Exception): + """ raised on db type errors """ + pass + +class Database: + client = None + db = None + + def __init__(self, host, user, pwd, data, port=27017): + if not isinstance(host, str): + raise DBTypeError("'host' param must be type of 'str'.") + if not isinstance(user, str): + raise DBTypeError("'user' param must be type of 'str'.") + if not isinstance(pwd, str): + raise DBTypeError("'pwd' param must be type of 'str'.") + if not isinstance(data, str): + raise DBTypeError("'data' param must be type of 'str'.") + if not isinstance(port, int): + raise DBTypeError("'port' param must be type of 'int'.") + try: + self.client = MongoClient(f"mongodb://{user}:{pwd}@{host}:{port}/") + self.db = self.client[data] + print(self.client) + print(self.db) + except: + raise DBError("Failed to connect.") + + def close(self): + if self.db is not None: + self.db.close() + + def insert(self, collection, fields): + if not isinstance(collection, str): + raise DBTypeError("'collection' param must be type of 'str'.") + if not isinstance(fields, dict): + raise DBTypeError("'fields' param must be type of 'dict'.") + try: + col = self.db[collection] + return col.insert_one(fields) + except: + return None + + def update(self, collection, objectID, fields): + if not isinstance(collection, str): + raise DBTypeError("'collection' param must be type of 'str'.") + if not isinstance(fields, dict): + raise DBTypeError("'fields' param must be type of 'dict'.") + + try: + col = self.db[collection] + res = col.update_many({"id": objectID}, {"$set": fields}) + return res.modified_count + except: + return None + + def delete(self, collection, objectID): + if not isinstance(collection, str): + raise DBTypeError("'collection' param must be type of 'str'.") + if not isinstance(objectID, str): + raise DBTypeError("'objectID' param must be type of 'str' or 'ObjectID'.") + try: + col = self.db[collection] + res = col.delete_many({"id": objectID}) + return res.deleted_count + except: + return -1 + + def filter(self, collection, filterCriteria): + if not isinstance(collection, str): + raise DBTypeError("'collection' param must be type of 'str'.") + + try: + col = self.db[collection] + res = col.find(filterCriteria) + return res + except: + return None + + +# +# Example usage as a standalone module +# +if __name__ == "__main__": + HOST = "" + PORT = 27017 + USER = "Admin" + PASS = "Pwd" + DATA = "admin" + + db = Database(HOST, USER, PASS, DATA, PORT) + x = db.filter("users", "testUser1") + print(f"Find: {x}") + x = db.insert("users", {"name": "testUser1", "number": 3, "number2": 3.5, "addedOn": datetime.datetime.now()}) + print(f"Insert: {x}") #x.inserted_id + x = db.filter("users", "testUser1") + print(f"Find: {x}") + x = db.update("users", "testUser1", {"number": 5, "number2": 7.75}) + print(f"Update: {x}") + x = db.filter("users", "{user: 'testUser1'}") + print(f"Find: {x}") + x = db.delete("users", "testUser1") + print(f"Delete: {x}") + x = db.filter("users", "testUser1") + print(f"Find: {x}") diff --git a/services/mysql.py b/services/mysql.py new file mode 100755 index 0000000..6235a59 --- /dev/null +++ b/services/mysql.py @@ -0,0 +1,192 @@ +import pymysql as mysql +import datetime +import time + +class DBError(Exception): + # raised when there's a fatal database error + pass + +class DBTypeError(TypeError): + # raised when there's a database type error (currently supports 'str', 'float', 'int', 'None', 'bool' and datetime.datetime) + pass + +class Database: + db = None + dbc = None + autocommit = True + autoreconnect = True + + def __init__(self, host, user, pwd, data, port=3306, autocommit=True, autoreconnect=True): + if not isinstance(host, str): + raise DBTypeError("'host' param must be type of 'str'.") + if not isinstance(user, str): + raise DBTypeError("'user' param must be type of 'str'.") + if not isinstance(pwd, str): + raise DBTypeError("'pwd' param must be type of 'str'.") + if not isinstance(data, str): + raise DBTypeError("'data' param must be type of 'str'.") + if not isinstance(port, int): + raise DBTypeError("'port' param must be type of 'int'.") + if not isinstance(autocommit, bool): + raise DBTypeError("'port' param must be type of 'bool'.") + if not isinstance(autoreconnect, bool): + raise DBTypeError("'port' param must be type of 'bool'.") + try: + self.db = mysql.connect(host=host, user=user, password=pwd, database=data, port=port) + self.dbc = self.db.cursor() + self.autocommit = autocommit + self.db.ping(reconnect=autoreconnect) + except Exception as e: + raise DBError(f"Exception while trying to connect to the database: {e}") + + def close(self): + if self.db is not None: + self.db.close() + + def insert(self, table, fields): + if not isinstance(table, str): + raise DBTypeError("'table' param must be type of 'str'.") + if not isinstance(fields, dict): + raise DBTypeError("'fields' param must be type of 'dict', ie. Dict['field'] = value") + length = len(fields) + if length == 0: + raise DBTypeError("'fields' parameter must not be empty.") + table = _escape(table) + query = f"INSERT INTO {table} (" + i = 0 + querypt1 = '' + querypt2 = '' + for key in fields: + i += 1 + key = _escape(key) + if not isinstance(key, str): + raise DBTypeError("A 'fields' key must be type of 'str' only.") + if isinstance(fields[key], str): + fields[key] = _escape(fields[key]) + querypt1 += f"{key}" + querypt2 += f"'{fields[key]}'" + elif isinstance(fields[key], int) or isinstance(fields[key], float) or isinstance(fields[key], bool): + querypt1 += f"{key}" + querypt2 += f"{fields[key]}" + elif fields[key] is None: + querypt1 += f"{key}" + querypt2 += f"NULL" + elif isinstance(fields[key], datetime.datetime): + querypt1 += f"{key}" + querypt2 += f"'{fields[key].strftime('%Y%m%d%H%M%S')}'" + else: + raise DBTypeError("A 'fields' value must be type of 'int', 'float', 'bool', 'str', 'datetime.datetime' or 'None' only.") + if i != length: + querypt1 += "," + querypt2 += "," + try: + query += querypt1 + ") VALUES (" + querypt2 + ")" + self.dbc.execute(query) + return True + except Exception as e: + print(e) + return False + + def update(self, table, userID, fields): + if not isinstance(table, str): + raise DBTypeError("'table' param must be type of 'str'.") + if not isinstance(fields, dict): + raise DBTypeError("'fields' param must be type of 'dict', ie. Dict['field'] = value") + if not isinstance(userID, int): + raise DBTypeError("'userID' param must be type of 'int'.") + length = len(fields) + table = _escape(table) + if length == 0: + raise DBTypeError("'fields' parameter must not be empty.") + query = f"UPDATE {table} SET " + i = 0 + for key in fields: + i += 1 + key = _escape(key) + if not isinstance(key, str): + raise DBTypeError("A 'fields' key must be type of 'str' only.") + if isinstance(fields[key], str): + fields[key] = _escape(fields[key]) + query += f"{key} = '{fields[key]}'" + elif isinstance(fields[key], int) or isinstance(fields[key], float) or isinstance(fields[key], bool): + query += f"{key} = {fields[key]}" + elif isinstance(fields[key], datetime.datetime): + query += f"{fields[key].strftime('%Y%m%d%H%M%S')}" + elif fields[key] is None: + query =+ f"{key} = NULL" + else: + raise DBTypeError("A 'fields' value must be type of 'int', 'float', 'bool', 'str', 'datetime.datetime' or 'None' only.") + if i != length: + query += "," + query += f" WHERE ID = {userID}" + try: + self.dbc.execute(query) + if self.autocommit: + self.db.commit() + return self.dbc.rowcount + except Exception: + return -1 + + def delete(self, table, userID): + if not isinstance(table, str): + raise DBTypeError("'table' param must be type of 'str'.") + if not isinstance(userID, int): + raise DBTypeError("'userID' param must be type of 'int'.") + table = _escape(table) + query = f"DELETE FROM {table} WHERE ID = {userID}" + try: + self.dbc.execute(query) + if self.autocommit: + self.db.commit() + return self.dbc.rowcount + except Exception: + return -1 + + def filter(self, table, userName): + if not isinstance(table, str): + raise DBTypeError("'table' param must be type of 'str'.") + if not isinstance(userName, str): + raise DBTypeError("'userName' param must be type of 'str'.") + userName = _escape(userName) + table = _escape(table) + query = f"SELECT * FROM {table} WHERE name = '{userName}'" + try: + self.dbc.execute(query) + if self.autocommit: + self.db.commit() + x = list(self.dbc.fetchall()) + for i in range(len(x)): + x[i] = list(x[i]) + return x + except Exception: + return None + +def _escape(text): + return text.replace("'", "''") + +# +# Example usage as a standalone module +# +if __name__ == "__main__": + USER = "" + HOST = "" + PASS = "" + DATA = "" + db = Database(HOST, USER, PASS, DATA, autocommit=True, autoreconnect=True) + x = db.filter("users", "test8") + print(f"Select: {x}") + d = datetime.datetime(year=2020, month=3, day=5, hour=17, minute=10, second=59) + x = db.insert("users", {"name":"test8","pwd":"x","salt":"y","token":"z","number":310,"number2":3.13421,"date":d}) + print(f"Insert: {x}") + x = db.filter("users", "test8") + print(f"Select: {x}") + _id = x[0][0] + x = db.update("users", _id, {"number": 72, "number2": 127.4}) + print(f"Update: {x}") + x = db.filter("users", "test8") + print(f"Select: {x}") + x = db.delete("users", _id) + print(f"Delete: {x}") + x = db.filter("users", "test8") + print(f"Select: {x}") + db.close() diff --git a/services/storage.py b/services/storage.py old mode 100644 new mode 100755 diff --git a/services/tokenizer.py b/services/tokenizer.py old mode 100644 new mode 100755 diff --git a/start.sh b/start.sh old mode 100644 new mode 100755