feat(bklogin): monitoring (#1397)

src/bk-login/bklogin/common/error_codes.py

@@ -45,6 +45,11 @@ class ErrorCodeCategoryEnum(str, StructuredEnum):
 class ErrorCodes:
     # 通用
     INVALID_ARGUMENT = ErrorCode(_("参数非法"))
+    UNAUTHENTICATED = ErrorCode(
+        _("未认证"),
+        code_category=ErrorCodeCategoryEnum.UNAUTHENTICATED,
+        status_code=HTTPStatus.UNAUTHORIZED,
+    )
     NO_PERMISSION = ErrorCode(
         _("无权限"),
         code_category=ErrorCodeCategoryEnum.NO_PERMISSION,

src/bk-login/bklogin/monitoring/healthz/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""

src/bk-login/bklogin/monitoring/healthz/probes.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from blue_krill.monitoring.probe.mysql import MySQLProbe, transfer_django_db_settings
+from django.conf import settings
+from django.utils.module_loading import import_string
+
+
+def get_default_probes():
+    return [import_string(p) for p in settings.HEALTHZ_PROBES]
+
+
+class MysqlProbe(MySQLProbe):
+    name = "bklogin-mysql"
+    config = transfer_django_db_settings(settings.DATABASES["default"])

src/bk-login/bklogin/monitoring/healthz/urls.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from django.urls import path
+
+from . import views
+
+urlpatterns = [
+    path("ping", views.PingApi.as_view(), name="api.ping"),
+    path("healthz", views.HealthzApi.as_view(), name="api.healthz"),
+]

src/bk-login/bklogin/monitoring/healthz/views.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from blue_krill.monitoring.probe.base import ProbeSet
+from django.conf import settings
+from django.http import HttpResponse
+from django.views.generic import View
+
+from bklogin.common.error_codes import error_codes
+from bklogin.common.response import APISuccessResponse
+
+from .probes import get_default_probes
+
+
+class PingApi(View):
+    """就绪&存活探测 API"""
+
+    def get(self, request, *args, **kwargs):
+        return HttpResponse("pong")
+
+
+class HealthzApi(View):
+    """健康探测 API"""
+
+    def get(self, request, *args, **kwargs):
+        token = request.GET.get("token", "")
+        if not settings.HEALTHZ_TOKEN:
+            raise error_codes.UNAUTHENTICATED.f(
+                "Healthz token was not configured in settings, request denied", replace=True
+            )
+        if not (token and token == settings.HEALTHZ_TOKEN):
+            raise error_codes.UNAUTHENTICATED.f("Please provide valid token", replace=True)
+
+        probe_set = ProbeSet(get_default_probes())
+        diagnosis_list = probe_set.examination()
+
+        if diagnosis_list.is_death:
+            # if something deadly exist, we have to make response non-200 which is easier to be found
+            # by monitor system and make response as a plain text
+            raise error_codes.SYSTEM_ERROR.f("internal server error", replace=True).set_data(
+                diagnosis_list.get_fatal_report()
+            )
+
+        results = [
+            {
+                "system_name": i.system_name,
+                "alive": i.alive,
+                "issues": [{"fatal": j.fatal, "description": j.description} for j in i.issues],
+            }
+            for i in diagnosis_list.items
+        ]
+
+        return APISuccessResponse(data={"results": results})

src/bk-login/bklogin/monitoring/metrics/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""

src/bk-login/bklogin/monitoring/metrics/urls.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from django.conf.urls import url
+
+from . import views
+
+urlpatterns = [url(r"^metrics$", views.metric_view, name="prometheus-django-metrics")]

src/bk-login/bklogin/monitoring/metrics/views.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from django.conf import settings
+from django_prometheus.exports import ExportToDjangoView
+
+from bklogin.common.error_codes import error_codes
+
+
+def metric_view(request):
+    """metric view with basic auth"""
+    token = request.GET.get("token", "")
+    if not settings.METRIC_TOKEN:
+        raise error_codes.UNAUTHENTICATED.f(
+            "Metric token was not configured in settings, request denied", replace=True
+        )
+    if not (token and token == settings.METRIC_TOKEN):
+        raise error_codes.UNAUTHENTICATED.f("Please provide valid token", replace=True)
+
+    return ExportToDjangoView(request)

src/bk-login/bklogin/monitoring/tracing/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""

src/bk-login/bklogin/monitoring/tracing/apps.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from django.apps import AppConfig
+
+from .otel import setup_by_settings
+from .sentry import init_sentry_sdk
+
+
+class TracingConfig(AppConfig):
+    name = "bklogin.monitoring.tracing"
+
+    def ready(self):
+        setup_by_settings()
+        init_sentry_sdk(
+            django_integrated=True,
+            redis_integrated=False,
+            celery_integrated=False,
+        )

src/bk-login/bklogin/monitoring/tracing/hooks.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import json
+from typing import Dict
+
+import requests
+from django.http import HttpRequest, HttpResponse
+from opentelemetry.trace import Span, StatusCode, format_trace_id
+
+
+def handle_api_error(span: Span, result: Dict):
+    """统一处理新版 HTTP API 协议中的错误详情"""
+    if "error" not in result:
+        return
+
+    err = result["error"]
+    span.set_attribute("error_code", err.get("code", ""))
+    span.set_attribute("error_message", err.get("message", ""))
+    span.set_attribute("error_system", err.get("system", ""))
+    # 错误详情若存在，则统一存到一个字段中
+    if err_details := err.get("details", []):
+        span.set_attribute("error_details", json.dumps(err_details))
+
+
+def requests_response_hook(span: Span, request: requests.Request, response: requests.Response):
+    """用于处理 requests 库发起的请求响应，需要兼容支持新旧 esb，apigw，新版 HTTP 协议"""
+    if (
+        # requests 请求异常, 例如访问超时等
+        response is None
+        # 并非所有返回内容都是 json 格式的, 因此需要根据返回头进行判断, 避免处理二进制格式的内容
+        or response.headers.get("Content-Type") != "application/json"
+    ):
+        return
+
+    try:
+        result = json.loads(response.content)
+    except Exception:  # pylint: disable=broad-except
+        return
+    if not isinstance(result, dict):
+        return
+
+    request_id = (
+        # new esb and apigateway
+        response.headers.get("x-bkapi-request-id")
+        # legacy api
+        or response.headers.get("X-Request-Id")
+        # old esb and other
+        or result.get("request_id", "")
+    )
+    if request_id:
+        span.set_attribute("request_id", request_id)
+
+    if "message" in result:
+        span.set_attribute("error_message", result["message"])
+
+    # 旧版本 API 中，code 为 0/'0'/'00' 表示成功
+    code = result.get("code")
+    if code is not None:
+        span.set_attribute("error_code", str(code))
+        if str(code) in ["0", "00"]:
+            span.set_status(StatusCode.OK)
+        else:
+            span.set_status(StatusCode.ERROR)
+
+        # 后续均为处理新版 API 协议逻辑，因此此处直接 return
+        return
+
+    # 根据新版本 HTTP API 协议，处理错误详情
+    handle_api_error(span, result)
+
+    if 200 <= response.status_code <= 299:  # noqa: PLR2004
+        span.set_status(StatusCode.OK)
+    else:
+        span.set_status(StatusCode.ERROR)
+
+
+def django_request_hook(span: Span, request: HttpRequest):
+    """在 request 注入 trace_id，方便获取"""
+    trace_id = span.get_span_context().trace_id
+    request.otel_trace_id = format_trace_id(trace_id)
+
+
+def django_response_hook(span: Span, request: HttpRequest, response: HttpResponse):
+    """处理 Django 响应，因用户管理已经使用新版本 HTTP 协议，因此仅支持新版协议"""
+
+    if (
+        # requests 请求异常, 例如访问超时等
+        response is None
+        # 并非所有返回内容都是 json 格式的, 因此需要根据返回头进行判断, 避免处理二进制格式的内容
+        or response.headers.get("Content-Type") != "application/json"
+    ):
+        return
+
+    # 新版本协议中按照标准 HTTP 协议，200 <= code < 300 的都是正常
+    if 200 <= response.status_code <= 299:  # noqa: PLR2004
+        span.set_status(StatusCode.OK)
+        return
+
+    span.set_status(StatusCode.ERROR)
+    try:
+        result = json.loads(response.content)
+    except Exception:  # pylint: disable=broad-except
+        return
+    if not isinstance(result, dict):
+        return
+
+    # 若能够获取到 request_id，则一并记录
+    request_id = response.headers.get("X-Request-Id") or result.get("request_id")
+    if request_id:
+        span.set_attribute("request_id", request_id)
+
+    handle_api_error(span, result)

src/bk-login/bklogin/monitoring/tracing/instrumentor.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import logging
+from typing import Collection
+
+from django.conf import settings
+from opentelemetry.instrumentation import dbapi
+from opentelemetry.instrumentation.celery import CeleryInstrumentor
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor  # type: ignore
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.instrumentation.redis import RedisInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+
+from .hooks import django_request_hook, django_response_hook, requests_response_hook
+
+logger = logging.getLogger(__name__)
+
+
+class BKLoginInstrumentor(BaseInstrumentor):
+    def instrumentation_dependencies(self) -> Collection[str]:
+        return []
+
+    def _instrument(self, **kwargs):
+        LoggingInstrumentor().instrument()
+        logger.info("otel instructment: logging")
+        RequestsInstrumentor().instrument(response_hook=requests_response_hook)
+        logger.info("otel instructment: requests")
+        DjangoInstrumentor().instrument(request_hook=django_request_hook, response_hook=django_response_hook)
+        logger.info("otel instructment: django")
+        RedisInstrumentor().instrument()
+        logger.info("otel instructment: redis")
+        CeleryInstrumentor().instrument()
+        logger.info("otel instructment: celery")
+
+        if getattr(settings, "OTEL_INSTRUMENT_DB_API", False):
+            import MySQLdb  # noqa
+
+            dbapi.wrap_connect(
+                __name__,
+                MySQLdb,
+                "connect",
+                "mysql",
+                {"database": "db", "port": "port", "host": "host", "user": "user"},
+            )
+            logger.info("otel instructment: database api")
+
+    def _uninstrument(self, **kwargs):
+        for instrumentor in self.instrumentors:
+            logger.info("otel uninstrument %s", instrumentor)
+            instrumentor.uninstrument()