diff --git a/.env_example b/.env_example index b1ee515..c7e2866 100644 --- a/.env_example +++ b/.env_example @@ -7,6 +7,10 @@ SQLALCHEMY_DATABASE_URI=postgresql+asyncpg://demo:91fb8e9e009f5b9ce1854d947e6fe4 REDIS_DSN=redis://:cfe1c2c4703abb205d71abdc07cc3f3d@localhost:6379 APP_ENV=PROD +RUNNING_MODE=uvicorn +WORKERS=1 +LISTENING_HOST=0.0.0.0 +LISTENING_PORT=8000 # docker compose DEFAULT_DB_PASSWORD=91fb8e9e009f5b9ce1854d947e6fe4a3 diff --git a/pyproject.toml b/pyproject.toml index bd90f52..ae5f529 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "sqladmin>=0.16.1", "cryptography>=42.0.7", "bcrypt>=4.1.3", + "gunicorn>=22.0.0", ] readme = "README.md" requires-python = ">= 3.12" @@ -60,7 +61,7 @@ target-version = "py312" [tool.ruff.lint] select = ["ALL"] -ignore = ["D", "G002", "DTZ003", "ANN401", "ANN101", "ANN102", "EM101", "PD901", "COM812", "ISC001", "FBT", "A003", "PLR0913", "G004"] +ignore = ["D", "G002", "DTZ003", "ANN401", "ANN101", "ANN102", "EM101", "PD901", "COM812", "ISC001", "FBT", "A003", "PLR0913", "G004", "E501"] fixable = ["ALL"] @@ -69,7 +70,7 @@ fixable = ["ALL"] "tests/*.py" = ["S101", "ANN201"] "*exceptions.py" = ["ARG001"] "models.py" = ["RUF012"] -"restapi.py" = ["A002", "B008"] +"api.py" = ["A002", "B008"] "deps.py" = ["B008"] "src/internal/api.py" = ["ARG001"] "src/auth/schemas.py" = ["N815"] # frontend menu diff --git a/requirements-dev.lock b/requirements-dev.lock index 5724968..948852c 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -48,6 +48,8 @@ filelock==3.13.1 # via virtualenv greenlet==3.0.3 # via sqlalchemy +gunicorn==22.0.0 + # via fastapi-enterprise-template h11==0.14.0 # via httpcore # via uvicorn @@ -82,6 +84,7 @@ numpy==1.26.3 # via pandas packaging==23.2 # via black + # via gunicorn # via pytest pandas==2.1.4 # via fastapi-enterprise-template diff --git a/requirements.lock b/requirements.lock index cac96c2..c2f5c7d 100644 --- a/requirements.lock +++ b/requirements.lock @@ -34,6 +34,8 @@ fastapi==0.108.0 # via fastapi-enterprise-template greenlet==3.0.3 # via sqlalchemy +gunicorn==22.0.0 + # via fastapi-enterprise-template h11==0.14.0 # via httpcore # via uvicorn @@ -56,6 +58,8 @@ markupsafe==2.1.3 # via wtforms numpy==1.26.3 # via pandas +packaging==24.1 + # via gunicorn pandas==2.1.4 # via fastapi-enterprise-template phonenumbers==8.13.27 diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 0000000..dfc9a7a --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,65 @@ +from typing import Any + +from fastapi import FastAPI + +from src.app import app +from src.core.config import settings +from src.loggers import LOGGING, configure_logger + + +def gunicorn_options() -> dict[str, Any]: + from gunicorn import glogging + + class GunicornLogger(glogging.Logger): + def setup(self, cfg: Any) -> None: # noqa: ARG002 + configure_logger() + + return { + "bind": f"{settings.LISTENING_HOST}:{settings.LISTENING_PORT}", + "workers": settings.WORKERS, + "worker_class": "uvicorn.workers.UvicornWorker", + "preload": "-", + "forwarded_allow_ips": "*", + "accesslog": "-", + "errorlog": "-", + "logger_class": GunicornLogger, + } + + +if __name__ == "__main__": + if settings.RUNNING_MODE == "uvicorn": + import uvicorn + + uvicorn.run( + app, + host=settings.LISTENING_HOST, + port=settings.LISTENING_PORT, + log_config=LOGGING, + proxy_headers=True, + forwarded_allow_ips="*", + loop="uvloop", + http="httptools", + ) + + else: + from gunicorn.app.base import BaseApplication + + class StandaloneApplication(BaseApplication): + def __init__(self, app: FastAPI, options: dict | None = None) -> None: + self.options = options or {} + self.application: FastAPI = app + super().__init__() + + def load_config(self) -> None: + assert self.cfg is not None # noqa: S101 + # Filter out options that are not recognized by gunicorn + filtered_options = {key: value for key, value in self.options.items() if key in self.cfg.settings} + + # Set the filtered options + for key, value in filtered_options.items(): + self.cfg.set(key.lower(), value) + + def load(self) -> FastAPI: + return self.application + + StandaloneApplication(app, gunicorn_options()).run() diff --git a/src/app.py b/src/app.py index 3693458..86eb2c5 100644 --- a/src/app.py +++ b/src/app.py @@ -4,13 +4,14 @@ import redis.asyncio as aioreids import sentry_sdk from fastapi import FastAPI +from fastapi.responses import HTMLResponse from starlette.middleware.cors import CORSMiddleware from starlette.middleware.errors import ServerErrorMiddleware from src.core.config import _Env, settings from src.core.errors.auth_exceptions import default_exception_handler, exception_handlers, sentry_ignore_errors from src.libs.redis import cache -from src.openapi import openapi_description +from src.openapi import get_open_api_intro, get_stoplight_elements_html from src.register.middlewares import RequestMiddleware from src.register.routers import router @@ -38,9 +39,31 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # noqa: ARG001 title=settings.PROJECT_NAME, version=settings.VERSION, summary=settings.DESCRIPTION, - description=openapi_description, + description=get_open_api_intro(), lifespan=lifespan, + docs_url="/api/docs", + redoc_url="/api/redoc", + openapi_url="/api/openapi.json", ) + + @app.get( + "/api/health", include_in_schema=False, tags=["Internal"], operation_id="e7372032-61c5-4e3d-b2f1-b788fe1c52ba" + ) + def health() -> dict[str, str]: + return {"status": "ok"} + + @app.get( + "/api/version", include_in_schema=False, tags=["Internal"], operation_id="47918987-15d9-4eea-8c29-e73cb009a4d5" + ) + def version() -> dict[str, str]: + return {"version": settings.VERSION} + + @app.get( + "/api/elements", include_in_schema=False, tags=["Docs"], operation_id="1a4987dd-6c38-4502-a879-3fe35050ae38" + ) + def get_stoplight_elements() -> HTMLResponse: + return get_stoplight_elements_html(openapi_url="/api/openapi.json", title=settings.PROJECT_NAME) + app.include_router(router, prefix="/api") for handler in exception_handlers: app.add_exception_handler(exc_class_or_status_code=handler["exception"], handler=handler["handler"]) diff --git a/src/core/config.py b/src/core/config.py index af762c6..a50b424 100644 --- a/src/core/config.py +++ b/src/core/config.py @@ -1,6 +1,7 @@ import tomllib from enum import StrEnum from pathlib import Path +from typing import Literal from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -44,6 +45,10 @@ class Settings(BaseSettings): REDIS_DSN: str = Field(default="redis://:cfe1c2c4703abb205d71abdc07cc3f3d@localhost:6379") ENV: str = _Env.DEV.name + RUNNING_MODE: Literal["uvicorn", "gunicorn"] | None = Field(default="uvicorn") + WORKERS: int | None = Field(default=1, gt=0) + LISTENING_HOST: str = Field(default="0.0.0.0") # noqa: S104 + LISTENING_PORT: int = Field(default=8000, gt=0, le=65535) model_config = SettingsConfigDict(env_file=f"{PROJECT_DIR}/.env", case_sensitive=True, extra="allow") diff --git a/src/core/utils/i18n.py b/src/core/utils/i18n.py index 5394b6e..3f5e4f4 100644 --- a/src/core/utils/i18n.py +++ b/src/core/utils/i18n.py @@ -6,7 +6,7 @@ from src.core.utils.context import locale_ctx from src.core.utils.singleton import singleton -from src.openapi import translations +from src.core.utils.translations import translations @singleton @@ -36,5 +36,5 @@ def _find(self, language: str, path: str) -> dict | str: return f"missing translation for {language}" -_i18n: I18n = I18n() +_i18n = I18n() _: Callable[..., dict | str] = _i18n.gettext diff --git a/src/core/utils/processors.py b/src/core/utils/processors.py new file mode 100644 index 0000000..1689403 --- /dev/null +++ b/src/core/utils/processors.py @@ -0,0 +1,45 @@ +import io +from pathlib import Path + +import pandas as pd + + +def bytes_to_human_readable(bytes_: int, format_string: str = "%(value).1f%(symbol)s") -> str: + symbols = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") + prefix = {} + for i, s in enumerate(symbols[1:]): + prefix[s] = 1 << (i + 1) * 10 + for symbol in reversed(symbols[1:]): + if bytes_ >= prefix[symbol]: + value = float(bytes_) / prefix[symbol] + return format_string % locals() + return format_string % {"symbol": symbols[0], "value": bytes_} + + +def export_csv(data: list[dict]) -> str: + df = pd.DataFrame(data) + binary_data = io.StringIO() + df.to_csv(binary_data, index=False, encoding="utf-8") + return binary_data.getvalue() + + +def get_file_size(path: str) -> str: + return bytes_to_human_readable(Path(path).stat().st_size) + + +def format_duration(seconds: float) -> str: + duration_units = [ + ("week", 60 * 60 * 24 * 7), + ("day", 60 * 60 * 24), + ("hour", 60 * 60), + ("minute", 60), + ("second", 1), + ] + + parts = [] + for unit, duration in duration_units: + count, seconds = divmod(int(seconds), duration) + if count: + parts.append(f'{count} {unit}{"s" if count > 1 else ""}') + + return ", ".join(parts) if parts else "0 seconds" diff --git a/src/core/utils/translations.py b/src/core/utils/translations.py new file mode 100644 index 0000000..95266cb --- /dev/null +++ b/src/core/utils/translations.py @@ -0,0 +1,4 @@ +translations = { + "en_US": {}, + "zh_CN": {}, +} diff --git a/src/features/admin/restapi.py b/src/features/admin/api.py similarity index 98% rename from src/features/admin/restapi.py rename to src/features/admin/api.py index 039a10b..d0026a3 100644 --- a/src/features/admin/restapi.py +++ b/src/features/admin/api.py @@ -13,8 +13,8 @@ from src.deps import auth, get_session from src.features.admin import schemas from src.features.admin.models import Group, Permission, Role, User -from src.features.admin.repository import group_repo, menu_repo, permission_repo, role_repo, user_repo from src.features.admin.security import generate_access_token_response +from src.features.admin.services import group_repo, menu_repo, permission_repo, role_repo, user_repo router = APIRouter() diff --git a/src/features/admin/repository.py b/src/features/admin/repository.py deleted file mode 100644 index 9064d2f..0000000 --- a/src/features/admin/repository.py +++ /dev/null @@ -1,75 +0,0 @@ -from collections.abc import Sequence - -from fastapi.security import OAuth2PasswordRequestForm -from sqlalchemy import or_, select -from sqlalchemy.ext.asyncio import AsyncSession - -from src.core.errors.auth_exceptions import NotFoundError, PermissionDenyError -from src.core.repositories import BaseRepository -from src.core.utils.context import locale_ctx -from src.features.admin import schemas -from src.features.admin.models import Group, Menu, Permission, Role, User -from src.features.admin.security import verify_password - - -class UserRepo(BaseRepository[User, schemas.UserCreate, schemas.UserUpdate, schemas.UserQuery]): - async def verify_user(self, session: AsyncSession, user: OAuth2PasswordRequestForm) -> User: - stmt = self._get_base_stmt().where(or_(self.model.email == user.username, self.model.phone == user.username)) - db_user = await session.scalar(stmt) - if not db_user: - raise NotFoundError(self.model.__visible_name__[locale_ctx.get()], "username", user.username) - if not verify_password(user.password, db_user.password): - raise PermissionDenyError - return db_user - - -class PermissionRepo( - BaseRepository[Permission, schemas.PermissionCreate, schemas.PermissionUpdate, schemas.PermissionQuery] -): - async def create( - self, - session: AsyncSession, - obj_in: schemas.PermissionCreate, - excludes: set[str] | None = None, - exclude_unset: bool = False, - exclude_none: bool = False, - commit: bool | None = True, - ) -> Permission: - raise NotImplementedError - - async def update( - self, - session: AsyncSession, - db_obj: Permission, - obj_in: schemas.PermissionUpdate, - excludes: set[str] | None = None, - commit: bool | None = True, - ) -> Permission: - raise NotImplementedError - - async def delete(self, session: AsyncSession, db_obj: Permission) -> None: - raise NotImplementedError - - -class MenuRepo(BaseRepository[Menu, schemas.MenuCreate, schemas.MenuUpdate, schemas.MenuQuery]): - async def get_all(self, session: AsyncSession) -> Sequence[Menu]: - return (await session.scalars(select(self.model))).all() - - @staticmethod - def menu_tree_transform(menus: Sequence[Menu]) -> list[dict]: - ... - - -class GroupRepo(BaseRepository[Group, schemas.GroupCreate, schemas.GroupUpdate, schemas.GroupQuery]): - ... - - -class RoleRepo(BaseRepository[Role, schemas.RoleCreate, schemas.RoleUpdate, schemas.RoleQuery]): - ... - - -user_repo = UserRepo(User) -permission_repo = PermissionRepo(Permission) -menu_repo = MenuRepo(Menu) -group_repo = GroupRepo(Group) -role_repo = RoleRepo(Role) diff --git a/src/features/admin/services.py b/src/features/admin/services.py index e69de29..9064d2f 100644 --- a/src/features/admin/services.py +++ b/src/features/admin/services.py @@ -0,0 +1,75 @@ +from collections.abc import Sequence + +from fastapi.security import OAuth2PasswordRequestForm +from sqlalchemy import or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.errors.auth_exceptions import NotFoundError, PermissionDenyError +from src.core.repositories import BaseRepository +from src.core.utils.context import locale_ctx +from src.features.admin import schemas +from src.features.admin.models import Group, Menu, Permission, Role, User +from src.features.admin.security import verify_password + + +class UserRepo(BaseRepository[User, schemas.UserCreate, schemas.UserUpdate, schemas.UserQuery]): + async def verify_user(self, session: AsyncSession, user: OAuth2PasswordRequestForm) -> User: + stmt = self._get_base_stmt().where(or_(self.model.email == user.username, self.model.phone == user.username)) + db_user = await session.scalar(stmt) + if not db_user: + raise NotFoundError(self.model.__visible_name__[locale_ctx.get()], "username", user.username) + if not verify_password(user.password, db_user.password): + raise PermissionDenyError + return db_user + + +class PermissionRepo( + BaseRepository[Permission, schemas.PermissionCreate, schemas.PermissionUpdate, schemas.PermissionQuery] +): + async def create( + self, + session: AsyncSession, + obj_in: schemas.PermissionCreate, + excludes: set[str] | None = None, + exclude_unset: bool = False, + exclude_none: bool = False, + commit: bool | None = True, + ) -> Permission: + raise NotImplementedError + + async def update( + self, + session: AsyncSession, + db_obj: Permission, + obj_in: schemas.PermissionUpdate, + excludes: set[str] | None = None, + commit: bool | None = True, + ) -> Permission: + raise NotImplementedError + + async def delete(self, session: AsyncSession, db_obj: Permission) -> None: + raise NotImplementedError + + +class MenuRepo(BaseRepository[Menu, schemas.MenuCreate, schemas.MenuUpdate, schemas.MenuQuery]): + async def get_all(self, session: AsyncSession) -> Sequence[Menu]: + return (await session.scalars(select(self.model))).all() + + @staticmethod + def menu_tree_transform(menus: Sequence[Menu]) -> list[dict]: + ... + + +class GroupRepo(BaseRepository[Group, schemas.GroupCreate, schemas.GroupUpdate, schemas.GroupQuery]): + ... + + +class RoleRepo(BaseRepository[Role, schemas.RoleCreate, schemas.RoleUpdate, schemas.RoleQuery]): + ... + + +user_repo = UserRepo(User) +permission_repo = PermissionRepo(Permission) +menu_repo = MenuRepo(Menu) +group_repo = GroupRepo(Group) +role_repo = RoleRepo(Role) diff --git a/src/loggers.py b/src/loggers.py index cf5155e..d3cc0de 100644 --- a/src/loggers.py +++ b/src/loggers.py @@ -19,30 +19,30 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: "formatters": { "default": { "format": ( - "%(asctime)s | %(levelname)s | %(request_id)% | %(filename)s:%(funcName)s:%(lineno)d | %(message)s" - ), - "celery": { - "format": ( - "%(asctime)s | %(levelname)s | %(request_id)% [%(celery_parent_id)s-%(celery_current_id)s] |" - " %(filename)s:%(funcName)s:%(lineno)d | %(message)s" - ), - }, + "%(asctime)s | %(levelname)s | %(request_id)s | %(filename)s:%(funcName)s:%(lineno)d | %(message)s" + ) + }, + "celery": { + "format": ( + "%(asctime)s | %(levelname)s | %(request_id)s [%(celery_parent_id)s-%(celery_current_id)s] |" + " %(filename)s:%(funcName)s:%(lineno)d | %(message)s" + ) }, }, "handlers": { "stdout": { - "level": "DEBUG", + "level": "INFO", "class": "logging.StreamHandler", "formatter": "default", "stream": "ext://sys.stderr", - }, + } }, "loggers": { - "gunicorn.access": {"handlers": ["stdout"], "propagate": True, "level": "DEBUG"}, - "guncorn.error": {"handlers": ["stdout"], "propagate": True, "level": "DEBUG"}, - "celery": {"handlers": ["stdout"], "propagate": True, "level": "DEBUG"}, - "celery.app.trace": {"handlers": ["stdout"], "propagate": False, "level": "DEBUG"}, - "": {"handlers": ["stdout"], "propagate": False, "level": "DEBUG"}, + "gunicorn.access": {"handlers": ["stdout"], "propagate": True, "level": "INFO"}, + "guncorn.error": {"handlers": ["stdout"], "propagate": True, "level": "ERROR"}, + "celery": {"handlers": ["stdout"], "propagate": False, "level": "INFO"}, + "celery.app.trace": {"handlers": ["stdout"], "propagate": False, "level": "INFO"}, + "": {"handlers": ["stdout"], "propagate": False, "level": "INFO"}, }, } diff --git a/src/openapi.py b/src/openapi.py index 1e0cdea..299d36f 100644 --- a/src/openapi.py +++ b/src/openapi.py @@ -1,8 +1,72 @@ -openapi_description = """ +from enum import Enum -""" +from fastapi.responses import HTMLResponse -translations = { - "en_US": {}, - "zh_CN": {}, -} + +class TryItCredentialPolicyOptions(Enum): + OMIT = "omit" + include = "include" + SAME_ORIGIN = "same-origin" + + +class LayoutOptions(Enum): + SIDEBAR = "sidebar" + STACKED = "stacked" + + +class RouterOptions(Enum): + HISTORY = "history" + HASH = "hash" + MEMORY = "memory" + STATIC = "static" + + +def get_stoplight_elements_html( + *, + openapi_url: str, + title: str, + stoplight_elements_js_url: str = "https://unpkg.com/@stoplight/elements/web-components.min.js", + stoplight_elements_css_url: str = "https://unpkg.com/@stoplight/elements/styles.min.css", + stoplight_elements_favicon_url: str = "https://fastapi.tiangolo.com/img/favicon.png", + api_description_document: str = "", + base_path: str = "", + hide_internal: bool = False, + hide_try_it: bool = False, + try_it_cors_proxy: str = "", + try_it_credential_policy: TryItCredentialPolicyOptions = TryItCredentialPolicyOptions.OMIT, + layout: LayoutOptions = LayoutOptions.SIDEBAR, + logo: str = "", + router: RouterOptions = RouterOptions.HISTORY, +) -> HTMLResponse: + html = f""" + + + + + + {title} + + + + + + + + + """ + return HTMLResponse(html) + + +def get_open_api_intro() -> str: + ... diff --git a/src/register/middlewares.py b/src/register/middlewares.py index c14b3bf..3488f9d 100644 --- a/src/register/middlewares.py +++ b/src/register/middlewares.py @@ -1,56 +1,66 @@ -import io import json import time import uuid -from collections.abc import Callable -from dataclasses import dataclass, field +from dataclasses import dataclass from datetime import UTC, datetime -import pandas as pd from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint from starlette.requests import Request from starlette.responses import Response, StreamingResponse from starlette.types import ASGIApp from src.core.utils.context import locale_ctx, request_id_ctx +from src.core.utils.processors import export_csv @dataclass class RequestMiddleware(BaseHTTPMiddleware): app: ASGIApp - dispatch_func: Callable = field(init=False) csv_mime: str = "text/csv" time_header = "x-request-time" id_header = "x-request-id" - content_type: str = "Content-Type" - - def __post_init__(self) -> None: - self.dispatch_func = self.dispatch async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response: start_time = time.time() request_id = str(uuid.uuid4()) request_id_ctx.set(request_id) - language = request.headers.get(locale_ctx.name, locale_ctx.get()) - locale_ctx.set(language) - content_type = request.headers.get(self.content_type, None) - if all((content_type, content_type == self.csv_mime, request.method == "GET")): - response: StreamingResponse = await call_next(request) - async for _res in response.body_iterator: - response_data = _res.decode() - if response_data: - response_data = json.loads(response_data) - csv_result = response_data.get("results", []) - df = pd.DataFrame(csv_result) - output = io.StringIO() - df.to_csv(output, encoding="utf-8", index=False) - filename = f"exporting_data_{datetime.now(tz=UTC).strftime('%Y%m%d %H%M%S')}.csv" - csv_resp = StreamingResponse(iter([output.getvalue()]), media_type="application/otect-stream") - csv_resp.headers["Content-Disposition"] = f'attachment; filename="{filename}.csv"' - csv_resp.headers[self.id_header] = request_id - return csv_resp - response = await call_next(request) + locale_ctx.set(request.headers.get(locale_ctx.name, locale_ctx.get())) + + response = ( + await self._process_csv_response(request, call_next, request_id, start_time) + if self._is_csv_response(request) + else await call_next(request) + ) + response.headers[self.id_header] = request_id - process_time = str(time.time() - start_time) - response.headers[self.time_header] = process_time + response.headers[self.time_header] = str(time.time() - start_time) + return response + + def _is_csv_response(self, request: Request) -> bool: + return request.headers.get("Content-Type") == self.csv_mime and request.method == "GET" + + async def _process_csv_response( + self, request: Request, call_next: RequestResponseEndpoint, request_id: str, start_time: float + ) -> Response: + response: StreamingResponse = await call_next(request) # type: ignore # noqa: PGH003 + csv_data = await self._read_stream_data(response) + csv_result = json.loads(csv_data).get("results", []) + output = export_csv(csv_result) + + csv_resp = StreamingResponse(iter([output]), media_type="application/octet-stream") + csv_resp.headers.update( + { + "Content-Disposition": f'attachment; filename="exporting_data_{datetime.now(tz=UTC).strftime("%Y%m%d %H%M%S")}.csv"', + self.id_header: request_id, + self.time_header: str(time.time() - start_time), + } + ) + + return csv_resp + + async def _read_stream_data(self, response: StreamingResponse) -> str: + body = b"" + async for chunk in response.body_iterator: + body += chunk + return body.decode() diff --git a/src/register/routers.py b/src/register/routers.py index 809ab75..98015d6 100644 --- a/src/register/routers.py +++ b/src/register/routers.py @@ -1,6 +1,6 @@ from fastapi import APIRouter -from src.features.admin.restapi import router as auth_router +from src.features.admin.api import router as auth_router def register_v1_router() -> APIRouter: