diff --git a/src/bk-user/bin/start_celery_beat.sh b/src/bk-user/bin/start_celery_beat.sh new file mode 100755 index 000000000..eabdc72ba --- /dev/null +++ b/src/bk-user/bin/start_celery_beat.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# 设置环境变量 +CELERY_LOG_LEVEL=${CELERY_LOG_LEVEL:-info} + +# Run! +celery -A bkuser.celery beat -l ${CELERY_LOG_LEVEL} --scheduler django_celery_beat.schedulers:DatabaseScheduler diff --git a/src/bk-user/bkuser/apis/web/data_source/serializers.py b/src/bk-user/bkuser/apis/web/data_source/serializers.py index 5fc488d08..1622dd6f2 100644 --- a/src/bk-user/bkuser/apis/web/data_source/serializers.py +++ b/src/bk-user/bkuser/apis/web/data_source/serializers.py @@ -21,9 +21,10 @@ from bkuser.apps.data_source.constants import FieldMappingOperation from bkuser.apps.data_source.models import DataSource, DataSourcePlugin +from bkuser.apps.sync.constants import DataSourceSyncPeriod from bkuser.apps.tenant.models import TenantUserCustomField, UserBuiltinField from bkuser.biz.data_source_plugin import DefaultPluginConfigProvider -from bkuser.plugins.base import get_plugin_cfg_cls +from bkuser.plugins.base import get_plugin_cfg_cls, is_plugin_exists from bkuser.plugins.constants import DataSourcePluginEnum from bkuser.plugins.local.models import PasswordRuleConfig from bkuser.utils.pydantic import stringify_pydantic_error @@ -73,6 +74,35 @@ class DataSourceFieldMappingSLZ(serializers.Serializer): expression = serializers.CharField(help_text="表达式", required=False) +def _validate_field_mapping_with_tenant_user_fields( + field_mapping: List[Dict[str, str]], tenant_id: str +) -> List[Dict[str, str]]: + target_fields = {m.get("target_field") for m in field_mapping} + + builtin_fields = set(UserBuiltinField.objects.all().values_list("name", flat=True)) + tenant_user_custom_fields = TenantUserCustomField.objects.filter(tenant_id=tenant_id) + + allowed_target_fields = builtin_fields | set(tenant_user_custom_fields.values_list("name", flat=True)) + required_target_fields = builtin_fields | set( + tenant_user_custom_fields.filter(required=True).values_list("name", flat=True) + ) + + if not_allowed_fields := target_fields - allowed_target_fields: + raise ValidationError( + _("字段映射中的目标字段 {} 不属于用户自定义字段或内置字段").format(not_allowed_fields), + ) + if missed_required_fields := required_target_fields - target_fields: + raise ValidationError(_("必填目标字段 {} 缺少字段映射").format(missed_required_fields)) + + return field_mapping + + +class DataSourceSyncConfigSLZ(serializers.Serializer): + """数据源同步配置""" + + sync_period = serializers.ChoiceField(help_text="同步周期", choices=DataSourceSyncPeriod.get_choices()) + + class DataSourceCreateInputSLZ(serializers.Serializer): name = serializers.CharField(help_text="数据源名称", max_length=128) plugin_id = serializers.CharField(help_text="数据源插件 ID") @@ -80,6 +110,7 @@ class DataSourceCreateInputSLZ(serializers.Serializer): field_mapping = serializers.ListField( help_text="用户字段映射", child=DataSourceFieldMappingSLZ(), allow_empty=True, required=False, default=list ) + sync_config = DataSourceSyncConfigSLZ(help_text="数据源同步配置", required=False) def validate_name(self, name: str) -> str: if DataSource.objects.filter(name=name).exists(): @@ -93,28 +124,27 @@ def validate_plugin_id(self, plugin_id: str) -> str: return plugin_id - def validate_field_mapping(self, field_mapping: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - target_fields = {m.get("target_field") for m in field_mapping} - allowed_target_fields = list(UserBuiltinField.objects.all().values_list("name", flat=True)) + list( - TenantUserCustomField.objects.filter(tenant_id=self.context["tenant_id"]).values_list("name", flat=True) - ) - if not_allowed_fields := target_fields - set(allowed_target_fields): - raise ValidationError( - _("字段映射中的目标字段 {} 不属于用户自定义字段或内置字段").format(not_allowed_fields), - ) + def validate_field_mapping(self, field_mapping: List[Dict[str, str]]) -> List[Dict[str, str]]: + # 遇到空的字段映射，直接返回即可，validate() 中会根据插件类型校验是否必须提供字段映射 + if not field_mapping: + return field_mapping - return field_mapping + return _validate_field_mapping_with_tenant_user_fields(field_mapping, self.context["tenant_id"]) def validate(self, attrs: Dict[str, Any]) -> Dict[str, Any]: # 除本地数据源类型外，都需要配置字段映射 - if attrs["plugin_id"] != DataSourcePluginEnum.LOCAL and not attrs["field_mapping"]: - raise ValidationError(_("当前数据源类型必须配置字段映射")) + plugin_id = attrs["plugin_id"] + if plugin_id != DataSourcePluginEnum.LOCAL: + if not attrs["field_mapping"]: + raise ValidationError(_("当前数据源类型必须配置字段映射")) - PluginConfigCls = get_plugin_cfg_cls(attrs["plugin_id"]) # noqa: N806 - # 自定义插件，可能没有对应的配置类，不需要做格式检查 - if not PluginConfigCls: - return attrs + if not attrs.get("sync_config"): + raise ValidationError(_("当前数据源类型必须提供同步配置")) + if not is_plugin_exists(plugin_id): + raise ValidationError(_("数据源插件 {} 不存在").format(plugin_id)) + + PluginConfigCls = get_plugin_cfg_cls(plugin_id) # noqa: N806 try: PluginConfigCls(**attrs["plugin_config"]) except PDValidationError as e: @@ -155,6 +185,7 @@ class DataSourceUpdateInputSLZ(serializers.Serializer): field_mapping = serializers.ListField( help_text="用户字段映射", child=DataSourceFieldMappingSLZ(), allow_empty=True, required=False, default=list ) + sync_config = DataSourceSyncConfigSLZ(help_text="数据源同步配置", required=False) def validate_name(self, name: str) -> str: if DataSource.objects.filter(name=name).exists(): @@ -164,10 +195,6 @@ def validate_name(self, name: str) -> str: def validate_plugin_config(self, plugin_config: Dict[str, Any]) -> Dict[str, Any]: PluginConfigCls = get_plugin_cfg_cls(self.context["plugin_id"]) # noqa: N806 - # 自定义插件，可能没有对应的配置类，不需要做格式检查 - if not PluginConfigCls: - return plugin_config - try: PluginConfigCls(**plugin_config) except PDValidationError as e: @@ -175,24 +202,22 @@ def validate_plugin_config(self, plugin_config: Dict[str, Any]) -> Dict[str, Any return plugin_config - def validate_field_mapping(self, field_mapping: List[Dict]) -> List[Dict]: - # 除本地数据源类型外，都需要配置字段映射 - if self.context["plugin_id"] == DataSourcePluginEnum.LOCAL: + def validate_field_mapping(self, field_mapping: List[Dict[str, str]]) -> List[Dict[str, str]]: + # 遇到空的字段映射，直接返回即可，validate() 中会根据插件类型校验是否必须提供字段映射 + if not field_mapping: return field_mapping - if not field_mapping: - raise ValidationError(_("当前数据源类型必须配置字段映射")) + return _validate_field_mapping_with_tenant_user_fields(field_mapping, self.context["tenant_id"]) - target_fields = {m.get("target_field") for m in field_mapping} - allowed_target_fields = list(UserBuiltinField.objects.all().values_list("name", flat=True)) + list( - TenantUserCustomField.objects.filter(tenant_id=self.context["tenant_id"]).values_list("name", flat=True) - ) - if not_allowed_fields := target_fields - set(allowed_target_fields): - raise ValidationError( - _("字段映射中的目标字段 {} 不属于用户自定义字段或内置字段").format(not_allowed_fields), - ) + def validate(self, attrs: Dict[str, Any]) -> Dict[str, Any]: + if self.context["plugin_id"] != DataSourcePluginEnum.LOCAL: + if not attrs["field_mapping"]: + raise ValidationError(_("当前数据源类型必须配置字段映射")) - return field_mapping + if not attrs.get("sync_config"): + raise ValidationError(_("当前数据源类型必须提供同步配置")) + + return attrs class DataSourceSwitchStatusOutputSLZ(serializers.Serializer): @@ -200,24 +225,45 @@ class DataSourceSwitchStatusOutputSLZ(serializers.Serializer): class RawDataSourceUserSLZ(serializers.Serializer): - id = serializers.CharField(help_text="用户 ID") + code = serializers.CharField(help_text="用户 Code") properties = serializers.JSONField(help_text="用户属性") - leaders = serializers.ListField(help_text="用户 leader ID 列表", child=serializers.CharField()) - departments = serializers.ListField(help_text="用户部门 ID 列表", child=serializers.CharField()) + leaders = serializers.ListField(help_text="用户 leader code 列表", child=serializers.CharField()) + departments = serializers.ListField(help_text="用户部门 code 列表", child=serializers.CharField()) class RawDataSourceDepartmentSLZ(serializers.Serializer): - id = serializers.CharField(help_text="部门 ID") + code = serializers.CharField(help_text="部门 Code") name = serializers.CharField(help_text="部门名称") - parent = serializers.CharField(help_text="父部门 ID") + parent = serializers.CharField(help_text="父部门 Code") -class DataSourceTestConnectionOutputSLZ(serializers.Serializer): - """数据源连通性测试""" +class DataSourceTestConnectionInputSLZ(serializers.Serializer): + plugin_id = serializers.CharField(help_text="数据源插件 ID") + plugin_config = serializers.JSONField(help_text="数据源插件配置") + + def validate(self, attrs: Dict[str, Any]) -> Dict[str, Any]: + plugin_id = attrs["plugin_id"] + if plugin_id == DataSourcePluginEnum.LOCAL: + raise ValidationError(_("本地数据源不支持连通性测试")) + + if not is_plugin_exists(plugin_id): + raise ValidationError(_("数据源插件 {} 不存在").format(plugin_id)) + + PluginConfigCls = get_plugin_cfg_cls(plugin_id) # noqa: N806 + try: + attrs["plugin_config"] = PluginConfigCls(**attrs["plugin_config"]) + except PDValidationError as e: + raise ValidationError(_("插件配置不合法：{}").format(stringify_pydantic_error(e))) + + return attrs + + +class DataSourceTestConnectionOutputSLZ(serializers.Serializer): error_message = serializers.CharField(help_text="错误信息") user = RawDataSourceUserSLZ(help_text="用户") department = RawDataSourceDepartmentSLZ(help_text="部门") + extras = serializers.JSONField(help_text="额外信息") class DataSourceRandomPasswordInputSLZ(serializers.Serializer): @@ -263,8 +309,8 @@ def validate_file(self, file: UploadedFile) -> UploadedFile: return file -class LocalDataSourceImportOutputSLZ(serializers.Serializer): - """本地数据源导入结果""" +class DataSourceImportOrSyncOutputSLZ(serializers.Serializer): + """数据源导入/同步结果""" task_id = serializers.CharField(help_text="任务 ID") status = serializers.CharField(help_text="任务状态") diff --git a/src/bk-user/bkuser/apis/web/data_source/urls.py b/src/bk-user/bkuser/apis/web/data_source/urls.py index 3d050b747..3bbb75947 100644 --- a/src/bk-user/bkuser/apis/web/data_source/urls.py +++ b/src/bk-user/bkuser/apis/web/data_source/urls.py @@ -25,6 +25,12 @@ path("", views.DataSourceListCreateApi.as_view(), name="data_source.list_create"), # 数据源随机密码获取 path("random-passwords/", views.DataSourceRandomPasswordApi.as_view(), name="data_source.random_passwords"), + # 数据源连通性测试 + path( + "test-connection/", + views.DataSourceTestConnectionApi.as_view(), + name="data_source.test_connection", + ), # 数据源更新/获取 path("/", views.DataSourceRetrieveUpdateApi.as_view(), name="data_source.retrieve_update"), # 数据源启/停 @@ -33,12 +39,6 @@ views.DataSourceSwitchStatusApi.as_view(), name="data_source.switch_status", ), - # 连通性测试 - path( - "/operations/test_connection/", - views.DataSourceTestConnectionApi.as_view(), - name="data_source.test_connection", - ), # 获取用户信息导入模板 path( "/operations/download_template/", diff --git a/src/bk-user/bkuser/apis/web/data_source/views.py b/src/bk-user/bkuser/apis/web/data_source/views.py index fa5917588..667fd87dd 100644 --- a/src/bk-user/bkuser/apis/web/data_source/views.py +++ b/src/bk-user/bkuser/apis/web/data_source/views.py @@ -22,6 +22,7 @@ from bkuser.apis.web.data_source.serializers import ( DataSourceCreateInputSLZ, DataSourceCreateOutputSLZ, + DataSourceImportOrSyncOutputSLZ, DataSourcePluginDefaultConfigOutputSLZ, DataSourcePluginOutputSLZ, DataSourceRandomPasswordInputSLZ, @@ -30,10 +31,10 @@ DataSourceSearchInputSLZ, DataSourceSearchOutputSLZ, DataSourceSwitchStatusOutputSLZ, + DataSourceTestConnectionInputSLZ, DataSourceTestConnectionOutputSLZ, DataSourceUpdateInputSLZ, LocalDataSourceImportInputSLZ, - LocalDataSourceImportOutputSLZ, ) from bkuser.apis.web.mixins import CurrentUserTenantMixin from bkuser.apps.data_source.constants import DataSourceStatus @@ -47,7 +48,8 @@ from bkuser.common.passwd import PasswordGenerator from bkuser.common.response import convert_workbook_to_response from bkuser.common.views import ExcludePatchAPIViewMixin, ExcludePutAPIViewMixin -from bkuser.plugins.base import get_plugin_cfg_schema_map +from bkuser.plugins.base import get_plugin_cfg_schema_map, get_plugin_cls +from bkuser.plugins.constants import DataSourcePluginEnum logger = logging.getLogger(__name__) @@ -133,6 +135,7 @@ def post(self, request, *args, **kwargs): plugin=DataSourcePlugin.objects.get(id=data["plugin_id"]), plugin_config=data["plugin_config"], field_mapping=data["field_mapping"], + sync_config=data.get("sync_config") or {}, creator=current_user, updater=current_user, ) @@ -185,6 +188,7 @@ def put(self, request, *args, **kwargs): data_source.name = data["name"] data_source.plugin_config = data["plugin_config"] data_source.field_mapping = data["field_mapping"] + data_source.sync_config = data.get("sync_config") or {} data_source.updater = request.user.username data_source.save() @@ -207,7 +211,7 @@ def post(self, request, *args, **kwargs): return Response(DataSourceRandomPasswordOutputSLZ(instance={"password": passwd}).data) -class DataSourceTestConnectionApi(CurrentUserTenantDataSourceMixin, generics.RetrieveAPIView): +class DataSourceTestConnectionApi(generics.CreateAPIView): """数据源连通性测试""" serializer_class = DataSourceTestConnectionOutputSLZ @@ -215,32 +219,22 @@ class DataSourceTestConnectionApi(CurrentUserTenantDataSourceMixin, generics.Ret @swagger_auto_schema( tags=["data_source"], operation_description="数据源连通性测试", + request_body=DataSourceTestConnectionInputSLZ(), responses={status.HTTP_200_OK: DataSourceTestConnectionOutputSLZ()}, ) - def get(self, request, *args, **kwargs): - data_source = self.get_object() - if data_source.is_local: - raise error_codes.DATA_SOURCE_OPERATION_UNSUPPORTED - - # TODO (su) 实现代码逻辑，需调用数据源插件以确认连通性 - mock_data = { - "error_message": "", - "user": { - "id": "uid_2", - "properties": { - "username": "zhangSan", - }, - "leaders": ["uid_0", "uid_1"], - "departments": ["dept_id_1"], - }, - "department": { - "id": "dept_id_1", - "name": "dept_name", - "parent": "dept_id_0", - }, - } + def post(self, request, *args, **kwargs): + slz = DataSourceTestConnectionInputSLZ(data=request.data) + slz.is_valid(raise_exception=True) + data = slz.validated_data + + plugin_id = data["plugin_id"] + if plugin_id == DataSourcePluginEnum.LOCAL: + raise error_codes.DATA_SOURCE_OPERATION_UNSUPPORTED.f("本地数据源插件不支持连通性测试") + + PluginCls = get_plugin_cls(plugin_id) # noqa: N806 + result = PluginCls(data["plugin_config"]).test_connection() - return Response(DataSourceTestConnectionOutputSLZ(instance=mock_data).data) + return Response(DataSourceTestConnectionOutputSLZ(instance=result).data) class DataSourceSwitchStatusApi(CurrentUserTenantDataSourceMixin, ExcludePutAPIViewMixin, generics.UpdateAPIView): @@ -314,7 +308,7 @@ class DataSourceImportApi(CurrentUserTenantDataSourceMixin, generics.CreateAPIVi tags=["data_source"], operation_description="本地数据源用户数据导入", request_body=LocalDataSourceImportInputSLZ(), - responses={status.HTTP_200_OK: LocalDataSourceImportOutputSLZ()}, + responses={status.HTTP_200_OK: DataSourceImportOrSyncOutputSLZ()}, ) def post(self, request, *args, **kwargs): """从 Excel 导入数据源用户数据""" @@ -330,13 +324,14 @@ def post(self, request, *args, **kwargs): try: workbook = openpyxl.load_workbook(data["file"]) except Exception: # pylint: disable=broad-except - logger.exception("本地数据源导入失败") + logger.exception("本地数据源 %s 导入失败", data_source.id) raise error_codes.DATA_SOURCE_IMPORT_FAILED.f(_("文件格式异常")) options = DataSourceSyncOptions( operator=request.user.username, overwrite=data["overwrite"], incremental=data["incremental"], + # FIXME (su) 本地数据源导入也要改成异步行为，但是要解决 excel 如何传递的问题 async_run=False, trigger=SyncTaskTrigger.MANUAL, ) @@ -349,16 +344,38 @@ def post(self, request, *args, **kwargs): raise error_codes.DATA_SOURCE_IMPORT_FAILED.f(str(e)) return Response( - LocalDataSourceImportOutputSLZ( + DataSourceImportOrSyncOutputSLZ( instance={"task_id": task.id, "status": task.status, "summary": task.summary} ).data ) -class DataSourceSyncApi(generics.CreateAPIView): +class DataSourceSyncApi(CurrentUserTenantDataSourceMixin, generics.CreateAPIView): """数据源同步""" + @swagger_auto_schema( + tags=["data_source"], + operation_description="数据源数据同步", + responses={status.HTTP_200_OK: DataSourceImportOrSyncOutputSLZ()}, + ) def post(self, request, *args, **kwargs): """触发数据源同步任务""" - # TODO (su) 实现代码逻辑，注意：本地数据源应该使用导入，而不是同步 - return Response() + data_source = self.get_object() + if data_source.is_local: + raise error_codes.DATA_SOURCE_OPERATION_UNSUPPORTED.f(_("本地数据源不支持同步，请使用导入功能")) + + # 同步策略：手动点击页面按钮，会触发全量覆盖的同步，且该同步是异步行为 + options = DataSourceSyncOptions( + operator=request.user.username, + overwrite=True, + incremental=False, + async_run=True, + trigger=SyncTaskTrigger.MANUAL, + ) + + task = DataSourceSyncManager(data_source, options).execute() + return Response( + DataSourceImportOrSyncOutputSLZ( + instance={"task_id": task.id, "status": task.status, "summary": task.summary} + ).data + ) diff --git a/src/bk-user/bkuser/apps/data_source/management/__init__.py b/src/bk-user/bkuser/apps/data_source/management/__init__.py new file mode 100644 index 000000000..1060b7bf4 --- /dev/null +++ b/src/bk-user/bkuser/apps/data_source/management/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/src/bk-user/bkuser/apps/data_source/management/commands/__init__.py b/src/bk-user/bkuser/apps/data_source/management/commands/__init__.py new file mode 100644 index 000000000..1060b7bf4 --- /dev/null +++ b/src/bk-user/bkuser/apps/data_source/management/commands/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/src/bk-user/bkuser/apps/data_source/management/commands/register_data_source_plugin.py b/src/bk-user/bkuser/apps/data_source/management/commands/register_data_source_plugin.py new file mode 100644 index 000000000..99c9b2afa --- /dev/null +++ b/src/bk-user/bkuser/apps/data_source/management/commands/register_data_source_plugin.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import os + +from django.conf import settings +from django.core.management.base import BaseCommand +from django.utils.module_loading import import_string + +from bkuser.apps.data_source.models import DataSourcePlugin +from bkuser.plugins.constants import CUSTOM_PLUGIN_ID_PREFIX, MAX_LOGO_SIZE +from bkuser.utils.base64 import load_image_as_base64 + + +class Command(BaseCommand): + """向数据库中写入数据源插件信息""" + + def add_arguments(self, parser): + parser.add_argument("--dir_name", dest="dir_name", required=True, help="插件目录名称") + + def handle(self, dir_name, *args, **options): + plugin_base_dir = settings.BASE_DIR / "bkuser" / "plugins" + # 1. 检查指定的插件目录是否存在 + if not os.path.isdir(plugin_base_dir / dir_name): + raise RuntimeError(f"plugin directory [{dir_name}] not found in bkuser/plugins!") + + # 2. 检查自定义插件是否配置 Metadata + try: + metadata = import_string(f"bkuser.plugins.{dir_name}.METADATA") + except ImportError: + raise RuntimeError("custom data source plugin must set metadata!") + + # 3. 确保自定义插件的 ID 符合规范 + if not metadata.id.startswith(CUSTOM_PLUGIN_ID_PREFIX): + raise RuntimeError(f"custom plugin's id must start with `{CUSTOM_PLUGIN_ID_PREFIX}`") + + logo_path = plugin_base_dir / f"{dir_name}/logo.png" + + # 4. 如果发现有 logo，还需要检查下尺寸大小，避免有性能问题 + if os.path.exists(logo_path) and os.path.getsize(logo_path) > MAX_LOGO_SIZE: + raise RuntimeError(f"plugin logo size must be less than {MAX_LOGO_SIZE/1024}KB!") + + # 5. 尝试获取下 logo，取不到就用默认的 + try: + logo = load_image_as_base64(logo_path) + except Exception: + self.stdout.write("failed to load plugin logo, use default logo...") + logo = "" + + # 6. 如果同名插件已存在，更新，否则创建 + DataSourcePlugin.objects.update_or_create( + id=metadata.id, + defaults={ + "name": metadata.name, + "description": metadata.description, + "logo": logo, + }, + ) + + # 7. 注册到 DB 成功要给提示 + self.stdout.write(f"register data source plugin [{metadata.id}] into database successfully.") diff --git a/src/bk-user/bkuser/apps/data_source/migrations/0002_inbuild_data_source_plugin.py b/src/bk-user/bkuser/apps/data_source/migrations/0002_inbuild_data_source_plugin.py index d948216b7..934e6aa62 100644 --- a/src/bk-user/bkuser/apps/data_source/migrations/0002_inbuild_data_source_plugin.py +++ b/src/bk-user/bkuser/apps/data_source/migrations/0002_inbuild_data_source_plugin.py @@ -1,21 +1,34 @@ -# Generated by Django 3.2.20 on 2023-08-11 07:10 - +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.conf import settings from django.db import migrations from bkuser.plugins.constants import DataSourcePluginEnum +from bkuser.utils.base64 import load_image_as_base64 def forwards_func(apps, schema_editor): """初始化本地数据源插件""" DataSourcePlugin = apps.get_model("data_source", "DataSourcePlugin") - # FIXME: 待数据源插件确定后，重新初始化 & 国际化，且需要考虑存储 base64 编码的 logo - if not DataSourcePlugin.objects.filter(id=DataSourcePluginEnum.LOCAL).exists(): - DataSourcePlugin.objects.create( - id=DataSourcePluginEnum.LOCAL, - name=DataSourcePluginEnum.get_choice_label(DataSourcePluginEnum.LOCAL), - description="支持用户和部门的增删改查，以及用户的登录认证", - ) + if DataSourcePlugin.objects.filter(id=DataSourcePluginEnum.LOCAL).exists(): + return + + # TODO 插件名称，描述国际化 + DataSourcePlugin.objects.create( + id=DataSourcePluginEnum.LOCAL, + name=DataSourcePluginEnum.get_choice_label(DataSourcePluginEnum.LOCAL), + description="支持用户和部门的增删改查，以及用户的登录认证", + logo=load_image_as_base64(settings.BASE_DIR / "bkuser/plugins/local/logo.png"), + ) class Migration(migrations.Migration): diff --git a/src/bk-user/bkuser/apps/data_source/migrations/0007_inbuild_data_source_plugin.py b/src/bk-user/bkuser/apps/data_source/migrations/0007_inbuild_data_source_plugin.py new file mode 100644 index 000000000..1e93ac0eb --- /dev/null +++ b/src/bk-user/bkuser/apps/data_source/migrations/0007_inbuild_data_source_plugin.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.conf import settings +from django.db import migrations + +from bkuser.plugins.constants import DataSourcePluginEnum +from bkuser.utils.base64 import load_image_as_base64 + + +def forwards_func(apps, schema_editor): + """初始化通用数据源插件""" + + DataSourcePlugin = apps.get_model("data_source", "DataSourcePlugin") + if DataSourcePlugin.objects.filter(id=DataSourcePluginEnum.GENERAL).exists(): + return + + # TODO 插件名称，描述国际化 + DataSourcePlugin.objects.create( + id=DataSourcePluginEnum.GENERAL, + name=DataSourcePluginEnum.get_choice_label(DataSourcePluginEnum.GENERAL), + description="支持对接通用 HTTP 数据源的插件，用户需要在服务方提供 `用户数据` 及 `部门数据` API", + logo=load_image_as_base64(settings.BASE_DIR / "bkuser/plugins/general/logo.png"), + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('data_source', '0006_departmentrelationmptttree'), + ] + + operations = [ + migrations.RunPython(forwards_func) + ] diff --git a/src/bk-user/bkuser/apps/data_source/models.py b/src/bk-user/bkuser/apps/data_source/models.py index c08e03d11..211f169cb 100644 --- a/src/bk-user/bkuser/apps/data_source/models.py +++ b/src/bk-user/bkuser/apps/data_source/models.py @@ -92,7 +92,7 @@ class LocalDataSourceIdentityInfo(TimestampedModel): """ user = models.OneToOneField(DataSourceUser, on_delete=models.CASCADE) - # FIXME (su) 使用加盐的非对称加密方式来存储密码 + # FIXME (su) 使用加盐的方式来存储密码 password = EncryptField(verbose_name="用户密码", null=True, blank=True, default="", max_length=255) password_updated_at = models.DateTimeField("密码最后更新时间", null=True, blank=True) password_expired_at = models.DateTimeField("密码过期时间", null=True, blank=True) diff --git a/src/bk-user/bkuser/apps/sync/constants.py b/src/bk-user/bkuser/apps/sync/constants.py index ed01297f8..5fe4dac22 100644 --- a/src/bk-user/bkuser/apps/sync/constants.py +++ b/src/bk-user/bkuser/apps/sync/constants.py @@ -12,6 +12,19 @@ from django.utils.translation import gettext_lazy as _ +class DataSourceSyncPeriod(int, StructuredEnum): + """数据源自动同步周期""" + + PER_30_MIN = EnumField(30, label=_("每 30 分钟")) + PER_1_HOUR = EnumField(60, label=_("每 1 小时")) + PER_3_HOUR = EnumField(3 * 60, label=_("每 3 小时")) + PER_6_HOUR = EnumField(6 * 60, label=_("每 6 小时")) + PER_12_HOUR = EnumField(12 * 60, label=_("每 12 小时")) + PER_1_DAY = EnumField(24 * 60, label=_("每 1 天")) + PER_7_DAY = EnumField(7 * 24 * 60, label=_("每 7 天")) + PER_30_DAY = EnumField(30 * 24 * 60, label=_("每 30 天")) + + class SyncTaskTrigger(str, StructuredEnum): """同步任务触发器枚举""" diff --git a/src/bk-user/bkuser/apps/sync/converters.py b/src/bk-user/bkuser/apps/sync/converters.py index b8ef8cc69..e6a20bcca 100644 --- a/src/bk-user/bkuser/apps/sync/converters.py +++ b/src/bk-user/bkuser/apps/sync/converters.py @@ -67,8 +67,8 @@ def convert(self, user: RawDataSourceUser) -> DataSourceUser: code=user.code, username=props[mapping["username"]], full_name=props[mapping["full_name"]], - email=props[mapping["email"]], - phone=props[mapping["phone"]], + email=props.get(mapping["email"]) or "", + phone=props.get(mapping["phone"]) or "", phone_country_code=props.get(mapping["phone_country_code"], settings.DEFAULT_PHONE_COUNTRY_CODE), extras={f.name: props.get(f.name, f.default) for f in self.custom_fields}, ) diff --git a/src/bk-user/bkuser/apps/sync/data_models.py b/src/bk-user/bkuser/apps/sync/data_models.py index cfc4c2b63..803ce5501 100644 --- a/src/bk-user/bkuser/apps/sync/data_models.py +++ b/src/bk-user/bkuser/apps/sync/data_models.py @@ -10,7 +10,7 @@ """ from pydantic import BaseModel -from bkuser.apps.sync.constants import SyncTaskTrigger +from bkuser.apps.sync.constants import DataSourceSyncPeriod, SyncTaskTrigger class DataSourceSyncOptions(BaseModel): @@ -37,3 +37,9 @@ class TenantSyncOptions(BaseModel): async_run: bool = True # 同步任务触发方式 trigger: SyncTaskTrigger = SyncTaskTrigger.SIGNAL + + +class DataSourceSyncConfig(BaseModel): + """数据源同步配置""" + + sync_period: DataSourceSyncPeriod = DataSourceSyncPeriod.PER_1_DAY diff --git a/src/bk-user/bkuser/apps/sync/exceptions.py b/src/bk-user/bkuser/apps/sync/exceptions.py new file mode 100644 index 000000000..fcbc2b086 --- /dev/null +++ b/src/bk-user/bkuser/apps/sync/exceptions.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + + +class DataSourceSyncError(Exception): + """数据源同步失败""" + + +class UserLeaderNotExists(DataSourceSyncError): + """直接上级数据不存在""" + + +class UserDepartmentNotExists(DataSourceSyncError): + """部门数据不存在""" diff --git a/src/bk-user/bkuser/apps/sync/handlers.py b/src/bk-user/bkuser/apps/sync/handlers.py index 79c3aaae4..b85c0ecbe 100644 --- a/src/bk-user/bkuser/apps/sync/handlers.py +++ b/src/bk-user/bkuser/apps/sync/handlers.py @@ -8,14 +8,22 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import json +import logging + +from django.db.models.signals import post_save from django.dispatch import receiver +from django_celery_beat.models import IntervalSchedule, PeriodicTask from bkuser.apps.data_source.models import DataSource from bkuser.apps.data_source.tasks import initialize_identity_info_and_send_notification -from bkuser.apps.sync.data_models import TenantSyncOptions +from bkuser.apps.sync.data_models import DataSourceSyncConfig, TenantSyncOptions from bkuser.apps.sync.managers import TenantSyncManager +from bkuser.apps.sync.names import gen_data_source_sync_periodic_task_name from bkuser.apps.sync.signals import post_sync_data_source +logger = logging.getLogger(__name__) + @receiver(post_sync_data_source) def sync_identity_infos_and_notify(sender, data_source: DataSource, **kwargs): @@ -28,3 +36,41 @@ def sync_tenant_departments_users(sender, data_source: DataSource, **kwargs): """同步租户数据（部门 & 用户）""" # TODO (su) 目前没有跨租户协同，因此只要往数据源所属租户同步即可 TenantSyncManager(data_source, data_source.owner_tenant_id, TenantSyncOptions()).execute() + + +@receiver(post_save, sender=DataSource) +def set_data_source_sync_periodic_task(sender, instance: DataSource, **kwargs): + """在创建/修改数据源后，需要设置定时同步的任务""" + data_source = instance + if data_source.is_local: + logger.info("skip set sync periodic task for local data source %s", data_source.id) + return + + periodic_task_name = gen_data_source_sync_periodic_task_name(data_source.id) + # 没有同步配置，抛出 warning 并且跳过 + if not data_source.sync_config: + logger.warning("data source %s hasn't sync_config, remove sync periodic task...", data_source.id) + PeriodicTask.objects.filter(name=periodic_task_name).delete() + return + + cfg = DataSourceSyncConfig(**data_source.sync_config) + interval_schedule, _ = IntervalSchedule.objects.get_or_create( + every=cfg.sync_period, + period=IntervalSchedule.MINUTES, + ) + periodic_task, task_created = PeriodicTask.objects.get_or_create( + name=periodic_task_name, + defaults={ + "interval": interval_schedule, + "task": "bkuser.apps.sync.periodic_tasks.build_and_run_data_source_sync_task", + "kwargs": json.dumps({"data_source_id": data_source.id}), + }, + ) + if not task_created: + logger.info( + "update data source %s sync periodic task's period as %s", + data_source.id, + cfg.sync_period, + ) + periodic_task.interval = interval_schedule + periodic_task.save() diff --git a/src/bk-user/bkuser/apps/sync/names.py b/src/bk-user/bkuser/apps/sync/names.py new file mode 100644 index 000000000..552168992 --- /dev/null +++ b/src/bk-user/bkuser/apps/sync/names.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + + +def gen_data_source_sync_periodic_task_name(data_source_id: int) -> str: + """生成数据源同步周期任务名称""" + return f"data_source_{data_source_id}_sync_periodic_task" diff --git a/src/bk-user/bkuser/apps/sync/periodic_tasks.py b/src/bk-user/bkuser/apps/sync/periodic_tasks.py new file mode 100644 index 000000000..11217b661 --- /dev/null +++ b/src/bk-user/bkuser/apps/sync/periodic_tasks.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging + +from bkuser.apps.data_source.constants import DataSourceStatus +from bkuser.apps.data_source.models import DataSource +from bkuser.apps.sync.constants import SyncTaskTrigger +from bkuser.apps.sync.data_models import DataSourceSyncOptions +from bkuser.apps.sync.managers import DataSourceSyncManager +from bkuser.celery import app +from bkuser.common.task import BaseTask + +logger = logging.getLogger(__name__) + + +@app.task(base=BaseTask, ignore_result=True) +def build_and_run_data_source_sync_task(data_source_id: int): + """同步数据源数据""" + logger.info("[celery-beat] receive build and run data source %s sync task", data_source_id) + + data_source = DataSource.objects.get(id=data_source_id) + if data_source.is_local: + logger.error("why local data source %s has periodic task?", data_source_id) + return + + if data_source.status != DataSourceStatus.ENABLED: + logger.warning("data source %s isn't enabled, skip...", data_source_id) + return + + sync_opts = DataSourceSyncOptions( + overwrite=True, + incremental=False, + # 注：现在就在异步任务中，不需要 async_run=True + async_run=False, + trigger=SyncTaskTrigger.CRONTAB, + ) + DataSourceSyncManager(data_source, sync_opts).execute() diff --git a/src/bk-user/bkuser/apps/sync/runners.py b/src/bk-user/bkuser/apps/sync/runners.py index 27a931463..cdd58de31 100644 --- a/src/bk-user/bkuser/apps/sync/runners.py +++ b/src/bk-user/bkuser/apps/sync/runners.py @@ -60,10 +60,8 @@ def run(self): def _initial_plugin(self, plugin_init_extra_kwargs: Dict[str, Any]): """初始化数据源插件""" - plugin_config = self.data_source.plugin_config PluginCfgCls = get_plugin_cfg_cls(self.data_source.plugin_id) # noqa: N806 - if PluginCfgCls is not None: - plugin_config = PluginCfgCls(**plugin_config) + plugin_config = PluginCfgCls(**self.data_source.plugin_config) PluginCls = get_plugin_cls(self.data_source.plugin_id) # noqa: N806 self.plugin = PluginCls(plugin_config, **plugin_init_extra_kwargs) diff --git a/src/bk-user/bkuser/apps/sync/syncers.py b/src/bk-user/bkuser/apps/sync/syncers.py index 84d353ed8..be9952c52 100644 --- a/src/bk-user/bkuser/apps/sync/syncers.py +++ b/src/bk-user/bkuser/apps/sync/syncers.py @@ -12,6 +12,7 @@ from typing import Dict, List, Set from django.utils import timezone +from django.utils.translation import gettext_lazy as _ from bkuser.apps.data_source.models import ( DataSource, @@ -23,6 +24,7 @@ DepartmentRelationMPTTTree, ) from bkuser.apps.sync.converters import DataSourceUserConverter +from bkuser.apps.sync.exceptions import UserDepartmentNotExists, UserLeaderNotExists from bkuser.apps.sync.models import DataSourceSyncTask, TenantSyncTask from bkuser.apps.tenant.models import Tenant, TenantDepartment, TenantUser from bkuser.common.constants import PERMANENT_TIME @@ -175,10 +177,43 @@ def __init__(self, task: DataSourceSyncTask, data_source: DataSource, raw_users: self.converter = DataSourceUserConverter(data_source) def sync(self): + self._validate_users() self._sync_users() self._sync_user_leader_relations() self._sync_user_department_relations() + def _validate_users(self): + """对用户数据进行校验（插件提供的数据不一定是合法的）""" + exists_user_codes = set( + DataSourceUser.objects.filter(data_source=self.data_source).values_list("code", flat=True) + ) + + # 检查本次同步的用户数据中，所有的 leader 是否已经存在 + raw_leader_codes = {leader_code for user in self.raw_users for leader_code in user.leaders} + + user_codes = {user.code for user in self.raw_users} + # 如果是增量同步，则 DB 中已经存在的用户，也可以作为 leader + if self.incremental: + user_codes |= exists_user_codes + + # Q: 提示信息使用 user_code 是否影响可读性 + # A：本地数据源用户 code 即为用户名，因此不会有可读性问题 + # 非本地数据源，因为本身插件提供的用户 Leader 信息即 code 列表，因此是可映射回实际数据的 + if not_exists_leaders := raw_leader_codes - user_codes: + raise UserLeaderNotExists(_("缺少用户上级：{} 信息").format(", ".join(not_exists_leaders))) + + # 数据源部门会先于用户同步，因此这里取到的就是所有可用的数据源部门 code + exists_dept_codes = set( + DataSourceDepartment.objects.filter(data_source=self.data_source).values_list("code", flat=True) + ) + raw_user_dept_codes = {dept_code for user in self.raw_users for dept_code in user.departments} + # 需要确保待同步的 用户-部门 关系中的部门都是存在的 + # Q: 提示信息使用 dept_code 是否影响可读性 + # A：尽管本地数据源使用 Hash 值作为部门 code，但是组织路径中的部门都会被创建，理论上不会触发该处异常 + # 非本地数据源，因为本身插件提供的用户部门信息即 code 列表，因此是可映射回实际的部门数据的 + if not_exists_depts := raw_user_dept_codes - exists_dept_codes: + raise UserDepartmentNotExists(_("缺少用户部门：{} 信息").format(", ".join(not_exists_depts))) + def _sync_users(self): user_codes = set(DataSourceUser.objects.filter(data_source=self.data_source).values_list("code", flat=True)) raw_user_codes = {user.code for user in self.raw_users} diff --git a/src/bk-user/bkuser/apps/tenant/data_models.py b/src/bk-user/bkuser/apps/tenant/data_models.py index 6aefef2de..796db1c41 100644 --- a/src/bk-user/bkuser/apps/tenant/data_models.py +++ b/src/bk-user/bkuser/apps/tenant/data_models.py @@ -20,4 +20,5 @@ class Option(BaseModel): class TenantUserCustomFieldOptions(BaseModel): """用户自定义字段-options字段""" + options: List[Option] diff --git a/src/bk-user/bkuser/biz/data_source_organization.py b/src/bk-user/bkuser/biz/data_source_organization.py index cd92fd9c6..71dad2c74 100644 --- a/src/bk-user/bkuser/biz/data_source_organization.py +++ b/src/bk-user/bkuser/biz/data_source_organization.py @@ -22,7 +22,6 @@ DataSourceUserLeaderRelation, ) from bkuser.apps.tenant.models import Tenant, TenantUser -from bkuser.plugins.local.utils import gen_code from bkuser.utils.uuid import generate_uuid @@ -80,7 +79,7 @@ def create_user( with transaction.atomic(): # 创建数据源用户 user = DataSourceUser.objects.create( - data_source=data_source, code=gen_code(base_user_info.username), **base_user_info.model_dump() + data_source=data_source, code=base_user_info.username, **base_user_info.model_dump() ) # 批量创建数据源用户-部门关系 diff --git a/src/bk-user/bkuser/biz/data_source_plugin.py b/src/bk-user/bkuser/biz/data_source_plugin.py index 256848414..306e48348 100644 --- a/src/bk-user/bkuser/biz/data_source_plugin.py +++ b/src/bk-user/bkuser/biz/data_source_plugin.py @@ -13,6 +13,8 @@ from pydantic import BaseModel from bkuser.plugins.constants import DataSourcePluginEnum +from bkuser.plugins.general.constants import AuthMethod, PageSize +from bkuser.plugins.general.models import AuthConfig, GeneralDataSourcePluginConfig, ServerConfig from bkuser.plugins.local.constants import ( NotificationMethod, NotificationScene, @@ -33,9 +35,14 @@ class DefaultPluginConfigProvider: def get(self, plugin_id: str) -> Optional[BaseModel]: """获取指定插件类型的默认插件配置""" + # 本地数据源 if plugin_id == DataSourcePluginEnum.LOCAL: return self._get_default_local_plugin_config() + # 通用 HTTP 数据源 + if plugin_id == DataSourcePluginEnum.GENERAL: + return self._get_default_general_plugin_config() + return None def _get_default_local_plugin_config(self) -> BaseModel: @@ -217,3 +224,18 @@ def _get_default_local_plugin_config(self) -> BaseModel: ), ), ) + + def _get_default_general_plugin_config(self) -> BaseModel: + return GeneralDataSourcePluginConfig( + server_config=ServerConfig( + server_base_url="https://bk.example.com", + user_api_path="/api/v1/users", + department_api_path="/api/v1/departments", + page_size=PageSize.CNT_100, + request_timeout=30, + retries=3, + ), + auth_config=AuthConfig( + method=AuthMethod.BASIC_AUTH, + ), + ) diff --git a/src/bk-user/bkuser/biz/tenant.py b/src/bk-user/bkuser/biz/tenant.py index 0ab073757..83da08107 100644 --- a/src/bk-user/bkuser/biz/tenant.py +++ b/src/bk-user/bkuser/biz/tenant.py @@ -26,7 +26,6 @@ DataSourceUserHandler, ) from bkuser.plugins.local.models import PasswordInitialConfig -from bkuser.plugins.local.utils import gen_code from bkuser.utils.uuid import generate_uuid @@ -302,7 +301,7 @@ def create_with_managers( for i in managers: # 创建数据源用户 data_source_user = DataSourceUser.objects.create( - data_source=data_source, code=gen_code(i.username), **i.model_dump() + data_source=data_source, code=i.username, **i.model_dump() ) # 创建对应的租户用户 tenant_user = TenantUser.objects.create( diff --git a/src/bk-user/bkuser/biz/validators.py b/src/bk-user/bkuser/biz/validators.py index e4d3ad9a5..d61f25cb4 100644 --- a/src/bk-user/bkuser/biz/validators.py +++ b/src/bk-user/bkuser/biz/validators.py @@ -22,5 +22,5 @@ def validate_data_source_user_username(value): if not re.fullmatch(DATA_SOURCE_USERNAME_REGEX, value): raise ValidationError( - _("{} 不符合 用户名 的命名规范: 由3-32位字母、数字、下划线(_)、点(.)、连接符(-)字符组成，以字母或数字开头").format(value), # noqa: E501 + _("{} 不符合 用户名 的命名规范: 由3-32位字母、数字、下划线(_)、点(.)、连接符(-)字符组成，以字母或数字开头及结尾").format(value), # noqa: E501 ) diff --git a/src/bk-user/bkuser/celery.py b/src/bk-user/bkuser/celery.py index f1ac43a46..276c62168 100644 --- a/src/bk-user/bkuser/celery.py +++ b/src/bk-user/bkuser/celery.py @@ -26,6 +26,7 @@ # Load task modules from all registered Django apps. app.autodiscover_tasks() +app.autodiscover_tasks(related_name="periodic_tasks") # set queue ha policy if use rabbitmq # default queue name is bkuser @@ -34,3 +35,5 @@ ] app.conf.task_default_queue = "bkuser" + +app.conf.beat_scheduler = "django_celery_beat.schedulers:DatabaseScheduler" diff --git a/src/bk-user/bkuser/component/http.py b/src/bk-user/bkuser/component/http.py index 223a7c915..34b5c2b9e 100644 --- a/src/bk-user/bkuser/component/http.py +++ b/src/bk-user/bkuser/component/http.py @@ -20,13 +20,6 @@ # 定义慢请求耗时，单位毫秒 SLOW_REQUEST_LATENCY = 100 - -def _gen_header(): - return { - "Content-Type": "application/json", - } - - session = requests.Session() adapter = HTTPAdapter(pool_connections=settings.REQUESTS_POOL_CONNECTIONS, pool_maxsize=settings.REQUESTS_POOL_MAXSIZE) session.mount("https://", adapter) diff --git a/src/bk-user/bkuser/plugins/README.md b/src/bk-user/bkuser/plugins/README.md index 4a10882f2..face0a165 100644 --- a/src/bk-user/bkuser/plugins/README.md +++ b/src/bk-user/bkuser/plugins/README.md @@ -1,3 +1,109 @@ -# 数据源插件开发指南 +# 自定义数据源插件开发指南 -TODO (su) 补充开发指南，需要说明架构设计，插件能力抽象，注意事项，通用 API 协议等等 +## 目录 & 文件说明 + +假设你要开发名为 Fox 的数据源插件，则需要在 `bkuser/plugins` 下新建 fox 文件夹，目录示例如下： + +``` +fox +├── __init__.py +├── exceptions.py +├── logo.png +├── models.py +└── plugin.py +``` + +### exceptions.py + +`exceptions.py` +存储该插件可能抛出的各类异常，需要注意的是，所有异常必须继承自 `bkuser.plugins.exceptions.BaseDataSourcePluginError` +，更推荐的做法是每种插件拥有自己的 `BaseException`，具体示例如下： + +```python +from bkuser.plugins.exceptions import BaseDataSourcePluginError + + +class FoxDataSourcePluginError(BaseDataSourcePluginError): + """Fox 数据源插件基础异常""" + + +class XXXError(FoxDataSourcePluginError): + """Fox 数据源插件 xxx 异常""" +``` + +### models.py + +`models.py` 存储该插件的配置类建模，我们采用 pydantic(v2) +来对配置类进行建模，所有的配置类模型都需要继承自 `pydantic.BaseModel`，且需要通过 `model_validator` 等方式，支持使用该模型对用户的输入进行校验。 + +建模示例可参考 [models.py](./local/models.py) + +### plugin.py + +`plugin.py` 是插件的入口，所有插件类需要继承自 `bkuser.plugins.base.BaseDataSourcePlugin` 并实现对应的方法。 + +插件设计示例及说明如下： + +```python +from bkuser.plugins.base import BaseDataSourcePlugin + + +class FoxDataSourcePlugin(BaseDataSourcePlugin): + """Fox 数据源插件""" + + # 注：非内置插件请使用字符串作为 ID，且需要以 `custom_` 为前缀 + id = "custom_fox" + # 插件配置类建模 + config_class = FoxDataSourcePluginConfig + + def __init__(self, plugin_config: FoxDataSourcePluginConfig): + self.plugin_config = plugin_config + + def fetch_departments(self) -> List[RawDataSourceDepartment]: + """获取部门信息""" + ... + + def fetch_users(self) -> List[RawDataSourceUser]: + """获取用户信息""" + ... + + def test_connection(self) -> TestConnectionResult: + """测试连通性""" + ... +``` + +### \_\_init\_\_.py + +在插件编写完成后，还需要在 `__init__.py` 中调用 register_plugin 以注册插件，示例如下： + +```python +from bkuser.plugins.base import register_plugin + +from .plugin import FoxDataSourcePlugin + +register_plugin(FoxDataSourcePlugin) + +# 注意：如果这是一个自定义插件（非蓝鲸官方内置），还需要设置插件 Metadata 信息 +from bkuser.plugins.models import PluginMetadata + +METADATA = PluginMetadata( + # 插件唯一 ID + id=FoxDataSourcePlugin.id, + # 插件展示用名称 + name="FoxIsNotDonkey", + # 插件展示用描述 + description="The fox is a nimble, smart creature known for its unique red-brown fur and bushy tail." +) +``` + +### logo.png + +数据源插件 Logo，仅支持 PNG 格式（推荐尺寸 1024 * 1024）；如果未提供则会使用默认 Logo。 + +## 注意事项 + +在将数据源插件实现的源代码目录挂载到 `bkuser/plugins` 目录下后，还需要在服务运行起来后，执行以下命令以在 DB 中添加插件信息： + +```shell +python manage.py register_data_source_plugin --dir_name=fox +``` diff --git a/src/bk-user/bkuser/plugins/__init__.py b/src/bk-user/bkuser/plugins/__init__.py index 1060b7bf4..07836735d 100644 --- a/src/bk-user/bkuser/plugins/__init__.py +++ b/src/bk-user/bkuser/plugins/__init__.py @@ -8,3 +8,20 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import os +from importlib import import_module + +from django.conf import settings + + +def load_plugins(): + plugin_base_dir = settings.BASE_DIR / "bkuser" / "plugins" + for name in os.listdir(plugin_base_dir): + if not os.path.isdir(plugin_base_dir / name): + continue + + # NOTE: 需要先在各个插件的 __init__.py 文件中调用 register_plugin 注册插件 + import_module(f"bkuser.plugins.{name}") + + +load_plugins() diff --git a/src/bk-user/bkuser/plugins/base.py b/src/bk-user/bkuser/plugins/base.py index e39edc80e..58ffd48df 100644 --- a/src/bk-user/bkuser/plugins/base.py +++ b/src/bk-user/bkuser/plugins/base.py @@ -8,21 +8,25 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import logging from abc import ABC, abstractmethod from typing import Dict, List, Type from drf_yasg import openapi from pydantic import BaseModel -from bkuser.plugins.constants import DataSourcePluginEnum +from bkuser.plugins.constants import CUSTOM_PLUGIN_ID_PREFIX, DataSourcePluginEnum from bkuser.plugins.models import RawDataSourceDepartment, RawDataSourceUser, TestConnectionResult from bkuser.utils.pydantic import gen_openapi_schema +logger = logging.getLogger(__name__) + class BaseDataSourcePlugin(ABC): """数据源插件基类""" - config_class: Type[BaseModel] | None + id: str | DataSourcePluginEnum + config_class: Type[BaseModel] @abstractmethod def __init__(self, *args, **kwargs): @@ -47,11 +51,29 @@ def test_connection(self) -> TestConnectionResult: _plugin_cls_map: Dict[str | DataSourcePluginEnum, Type[BaseDataSourcePlugin]] = {} -def register_plugin(plugin_id: str | DataSourcePluginEnum, plugin_cls: Type[BaseDataSourcePlugin]): - """注册插件""" +def register_plugin(plugin_cls: Type[BaseDataSourcePlugin]): + """注册数据源插件""" + plugin_id = plugin_cls.id + + if not plugin_id: + raise RuntimeError(f"plugin {plugin_cls} not provide id") + + if not plugin_cls.config_class: + raise RuntimeError(f"plugin {plugin_cls} not provide config_class") + + if not (isinstance(plugin_id, DataSourcePluginEnum) or plugin_id.startswith(CUSTOM_PLUGIN_ID_PREFIX)): + raise RuntimeError(f"custom plugin's id must start with `{CUSTOM_PLUGIN_ID_PREFIX}`") + + logger.info("register data source plugin: %s", plugin_id) + _plugin_cls_map[plugin_id] = plugin_cls +def is_plugin_exists(plugin_id: str | DataSourcePluginEnum) -> bool: + """判断插件是否存在""" + return plugin_id in _plugin_cls_map + + def get_plugin_cls(plugin_id: str | DataSourcePluginEnum) -> Type[BaseDataSourcePlugin]: """获取指定插件类""" if plugin_id not in _plugin_cls_map: @@ -60,7 +82,7 @@ def get_plugin_cls(plugin_id: str | DataSourcePluginEnum) -> Type[BaseDataSource return _plugin_cls_map[plugin_id] -def get_plugin_cfg_cls(plugin_id: str | DataSourcePluginEnum) -> Type[BaseModel] | None: +def get_plugin_cfg_cls(plugin_id: str | DataSourcePluginEnum) -> Type[BaseModel]: """获取指定插件的配置类""" return get_plugin_cls(plugin_id).config_class @@ -70,5 +92,4 @@ def get_plugin_cfg_schema_map() -> Dict[str, openapi.Schema]: return { f"plugin_config:{plugin_id}": gen_openapi_schema(model.config_class) for plugin_id, model in _plugin_cls_map.items() - if model.config_class is not None } diff --git a/src/bk-user/bkuser/plugins/constants.py b/src/bk-user/bkuser/plugins/constants.py index dcb9120b4..142ef4e0f 100644 --- a/src/bk-user/bkuser/plugins/constants.py +++ b/src/bk-user/bkuser/plugins/constants.py @@ -11,12 +11,18 @@ from blue_krill.data_types.enum import EnumField, StructuredEnum from django.utils.translation import gettext_lazy as _ +# 非内置插件，必须以指定前缀开头 +CUSTOM_PLUGIN_ID_PREFIX = "custom_" + +# Logo 限制 64KB，过大的 logo 会导致性能下降，1024 * 1024 约 37KB +MAX_LOGO_SIZE = 64 * 1024 + class DataSourcePluginEnum(str, StructuredEnum): """数据源插件枚举""" LOCAL = EnumField("local", label=_("本地数据源")) - GENERAL = EnumField("general", label=_("通用数据源")) + GENERAL = EnumField("general", label=_("通用 HTTP 数据源")) WECOM = EnumField("wecom", label=_("企业微信")) LDAP = EnumField("ldap", label=_("OpenLDAP")) MAD = EnumField("mad", label=_("MicrosoftActiveDirectory")) diff --git a/src/bk-user/bkuser/plugins/general/README.md b/src/bk-user/bkuser/plugins/general/README.md new file mode 100644 index 000000000..5e3038236 --- /dev/null +++ b/src/bk-user/bkuser/plugins/general/README.md @@ -0,0 +1,111 @@ +# 通用 HTTP 数据源 API 协议 + +## 协议说明 + +蓝鲸用户管理内置了对通用 HTTP 数据源的插件支持，用户需要在服务方提供 `用户数据` 及 `部门数据` API，具体请求参数 & 响应规范如下。 + +## 鉴权模式 + +通用 HTTP 插件目前支持使用 `BearerToken` 或 `BasicAuth` 进行鉴权，后续将陆续支持其他鉴权方式。当前示例请求等效于： + +```shell +# BearerToken +curl -H 'Authorization: Bearer ${BearerToken}' http://bk.example.com/apis/v1/users?page=1&page_size=100 + +# BasicAuth +curl -H "Authorization: Basic dXNlcm5hbWU6cGFzc3dvcmQ=" http://bk.example.com/apis/v1/departments?page=1&page_size=100 +``` + +## 用户数据 API + +### Request 参数 + +| 参数名称 | 描述 | 必须支持 | 默认值 | +|------------|----------|----------|-------| +| page | 页码 | ✓ | 1 | +| page_size | 每页数量 | ✓ | 100 | + +### Response 规范 + +```json5 +{ + "count": 3, // 总数量 + "results": [ + { + "id": "100", // 用户（本数据源内）唯一 ID + "username": "sanzhang", // 用户名（英文名） + "full_name": "张三", // 用户全名（姓名） + "email": "1234567891@qq.com", // 邮箱 + "phone": "12345678901", // 手机号码 + "phone_country_code": "86", // 手机区号 + "extras": { // 自定义字段信息 + "gender": "male" + }, + "leaders": [], // 用户直接上级唯一 ID + "departments": ["company"] // 用户所属部门唯一 ID + }, + { + "id": "101", + "username": "sili", + "full_name": "李四", + "email": "1234567892@qq.com", + "phone": "12345678902", + "phone_country_code": "86", + "extras": { + "gender": "female" + }, + "leaders": ["100"], + "departments": ["dept_a"] + }, + { + "id": "102", + "username": "wuwang", + "full_name": "王五", + "email": "1234567893@qq.com", + "phone": "12345678903", + "phone_country_code": "86", + "extras": { + "gender": "male" + }, + "leaders": ["100", "101"], + "departments": ["center_aa"] + } + ] +} +``` + +## 部门数据 API + +### Request 参数 + +| 参数名称 | 描述 | 必须支持 | 默认值 | +|------------|----------|----------|-------| +| page | 页码 | ✓ | 1 | +| page_size | 每页数量 | ✓ | 100 | + +### Response 规范 + +```json5 +{ + "count": 3, // 总数量 + "results": [ + { + "id": "company", // 部门（当前数据源内）唯一 ID + "name": "总公司", // 部门展示用名称 + "parent": null // 父部门唯一 ID，若为根部门则为 null + }, + { + "id": "dept_a", + "name": "部门A", + "parent": "company" + }, + { + "id": "center_aa", + "name": "中心AA", + "parent": "dept_a" + } + ] +} +``` + +> 注意：蓝鲸用户管理将通过分页的方式，分多次拉取全量用户 & 部门数据；若指定范围超过总数量，则返回结果中 results 字段需为空列表。 diff --git a/src/bk-user/bkuser/plugins/general/__init__.py b/src/bk-user/bkuser/plugins/general/__init__.py new file mode 100644 index 000000000..df169d035 --- /dev/null +++ b/src/bk-user/bkuser/plugins/general/__init__.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from bkuser.plugins.base import register_plugin + +from .plugin import GeneralDataSourcePlugin + +register_plugin(GeneralDataSourcePlugin) diff --git a/src/bk-user/bkuser/plugins/general/constants.py b/src/bk-user/bkuser/plugins/general/constants.py new file mode 100644 index 000000000..e1c5dbf0e --- /dev/null +++ b/src/bk-user/bkuser/plugins/general/constants.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from blue_krill.data_types.enum import EnumField, StructuredEnum + +# 服务基础 URL 正则 +BASE_URL_REGEX = r"^https?://[a-zA-Z0-9-\.]+(:\d+)?$" + +# API 路径正则 +API_URL_PATH_REGEX = r"^\/[\w-]+(\/[\w-]+)*\/?$" + +# 最小请求超时时间 +MIN_REQ_TIMEOUT = 5 +# 最大请求超时时间 +MAX_REQ_TIMEOUT = 120 +# 默认请求超时时间 +DEFAULT_REQ_TIMEOUT = 30 + +# 最小重试次数 +MIN_RETRIES = 0 +# 最大重试次数 +MAX_RETRIES = 3 +# 默认重试次数 +DEFAULT_RETRIES = 1 + +# 默认页码 +DEFAULT_PAGE = 1 +# 获取首条数据用的每页数量 +PAGE_SIZE_FOR_FETCH_FIRST = 1 +# 最大拉取总数量 100w +MAX_TOTAL_COUNT = 10**6 + + +class AuthMethod(str, StructuredEnum): + """鉴权方式""" + + BEARER_TOKEN = EnumField("bearer_token", label="BearerToken") + BASIC_AUTH = EnumField("basic_auth", label="BasicAuth") + + +class PageSize(int, StructuredEnum): + """每页数量""" + + CNT_100 = EnumField(100, label="100") + CNT_200 = EnumField(200, label="200") + CNT_500 = EnumField(500, label="500") + CNT_1000 = EnumField(1000, label="1000") + CNT_2000 = EnumField(2000, label="2000") + CNT_5000 = EnumField(5000, label="5000") diff --git a/src/bk-user/bkuser/plugins/general/exceptions.py b/src/bk-user/bkuser/plugins/general/exceptions.py new file mode 100644 index 000000000..af9e9acf8 --- /dev/null +++ b/src/bk-user/bkuser/plugins/general/exceptions.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from bkuser.plugins.exceptions import BaseDataSourcePluginError + + +class GeneralDataSourcePluginError(BaseDataSourcePluginError): + """通用 HTTP 数据源插件基础异常""" + + +class RequestApiError(GeneralDataSourcePluginError): + """请求数据源 API 异常""" + + +class RespDataFormatError(GeneralDataSourcePluginError): + """数据源 API 响应格式错误""" diff --git a/src/bk-user/bkuser/plugins/general/http.py b/src/bk-user/bkuser/plugins/general/http.py new file mode 100644 index 000000000..36bcb59ec --- /dev/null +++ b/src/bk-user/bkuser/plugins/general/http.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import base64 +import logging +from typing import Any, Dict, List + +import requests +from django.utils.translation import gettext_lazy as _ +from requests.adapters import HTTPAdapter, Retry +from requests.exceptions import JSONDecodeError + +from bkuser.plugins.general.constants import DEFAULT_PAGE, MAX_TOTAL_COUNT, PAGE_SIZE_FOR_FETCH_FIRST, AuthMethod +from bkuser.plugins.general.exceptions import RequestApiError, RespDataFormatError +from bkuser.plugins.general.models import AuthConfig + +logger = logging.getLogger(__name__) + + +def gen_headers(cfg: AuthConfig) -> Dict[str, str]: + headers = {"Content-Type": "application/json"} + + if cfg.method == AuthMethod.BEARER_TOKEN: + # BearerToken + headers["Authorization"] = f"Bearer {cfg.bearer_token}" + elif cfg.method == AuthMethod.BASIC_AUTH: + # BasicAuth + credentials = base64.b64encode(f"{cfg.username}:{cfg.password}".encode("utf-8")).decode("utf-8") + headers["Authorization"] = f"Basic {credentials}" + + return headers + + +def fetch_all_data( + url: str, headers: Dict[str, str], page_size: int, timeout: int, retries: int +) -> List[Dict[str, Any]]: + """ + 根据指定配置，请求数据源 API 以获取用户 / 部门数据 + + :param url: 数据源 URL，如 https://bk.example.com/apis/v1/users + :param headers: 请求头，包含认证信息等 + :param timeout: 单次请求超时时间 + :param retries: 请求失败重试次数 + :returns: API 返回结果，应符合通用 HTTP 数据源 API 协议 + """ + with requests.Session() as session: + adapter = HTTPAdapter( + max_retries=Retry( + total=retries, + backoff_factor=1, + status_forcelist=[429, 500, 502, 503, 504], + ) + ) + session.mount("https://", adapter) + session.mount("http://", adapter) + + cur_page, max_page = DEFAULT_PAGE, MAX_TOTAL_COUNT / page_size + total_cnt, items = 0, [] + while True: + params = {"page": cur_page, "page_size": page_size} + resp = session.get(url, headers=headers, params=params, timeout=timeout) + if not resp.ok: + raise RequestApiError( + _("请求数据源 API {} 参数 {} 异常，状态码：{} 响应内容 {}").format( + url, + params, + resp.status_code, + resp.content, + ) + ) + + try: + resp_data = resp.json() + except JSONDecodeError: # noqa: PERF203 + raise RespDataFormatError( + _("数据源 API {} 参数 {} 返回非 Json 格式，响应内容").format(url, params, resp.content) + ) # noqa: E501 + + total_cnt = resp_data.get("count", 0) + cur_req_results = resp_data.get("results", []) + items.extend(cur_req_results) + + logger.info( + "request data source api %s, params %s, get %d items, total count is %d", + url, + params, + len(cur_req_results), + total_cnt, + ) + + if cur_page * page_size >= total_cnt: + break + + # 理论拉取数量超过最大上限，强制退出 + if cur_page >= max_page: + logger.warning("request data source api %s, exceed max page %d, force break...", url, max_page) + break + + cur_page += 1 + + return items + + +def fetch_first_item(url: str, headers: Dict[str, str], timeout: int) -> Dict[str, Any] | None: + """ + 根据指定配置，请求数据源 API 以获取用户 / 部门第一条数据（测试连通性用） + + :param url: 数据源 URL，如 https://bk.example.com/apis/v1/users + :param headers: 请求头，包含认证信息等 + :param timeout: 单次请求超时时间 + :returns: API 返回结果，应符合通用 HTTP 数据源 API 协议 + """ + params = {"page": DEFAULT_PAGE, "page_size": PAGE_SIZE_FOR_FETCH_FIRST} + resp = requests.get(url, headers=headers, params=params, timeout=timeout) + if not resp.ok: + raise RequestApiError(_("请求数据源 API {} 异常：{}").format(url, resp.content)) + + try: + resp_data = resp.json() + except JSONDecodeError: + raise RespDataFormatError(_("数据源 API {} 返回非 Json 格式").format(url)) + + results = resp_data.get("results", []) + if not results: + return None + + return results[0] diff --git a/src/bk-user/bkuser/plugins/general/logo.png b/src/bk-user/bkuser/plugins/general/logo.png new file mode 100644 index 000000000..32d9ce8f6 Binary files /dev/null and b/src/bk-user/bkuser/plugins/general/logo.png differ diff --git a/src/bk-user/bkuser/plugins/general/models.py b/src/bk-user/bkuser/plugins/general/models.py new file mode 100644 index 000000000..a7c749d92 --- /dev/null +++ b/src/bk-user/bkuser/plugins/general/models.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from pydantic import BaseModel, Field + +from bkuser.plugins.general.constants import ( + API_URL_PATH_REGEX, + BASE_URL_REGEX, + DEFAULT_REQ_TIMEOUT, + DEFAULT_RETRIES, + MAX_REQ_TIMEOUT, + MAX_RETRIES, + MIN_REQ_TIMEOUT, + MIN_RETRIES, + AuthMethod, + PageSize, +) + + +class ServerConfig(BaseModel): + """数据服务相关配置""" + + # 服务地址 + server_base_url: str = Field(pattern=BASE_URL_REGEX) + # 用户数据 API 路径 + user_api_path: str = Field(pattern=API_URL_PATH_REGEX) + # 部门数据 API 路径 + department_api_path: str = Field(pattern=API_URL_PATH_REGEX) + # 单次分页请求数量 + page_size: PageSize = PageSize.CNT_100 + # 单次请求超时时间 + request_timeout: int = Field(ge=MIN_REQ_TIMEOUT, le=MAX_REQ_TIMEOUT, default=DEFAULT_REQ_TIMEOUT) + # 请求失败重试次数 + retries: int = Field(ge=MIN_RETRIES, le=MAX_RETRIES, default=DEFAULT_RETRIES) + + +class AuthConfig(BaseModel): + """认证配置""" + + method: AuthMethod + # bearer token 配置 + bearer_token: str | None = None + # basic auth 配置 + username: str | None = None + password: str | None = None + + +class GeneralDataSourcePluginConfig(BaseModel): + """通用 HTTP 数据源插件配置""" + + # 服务配置 + server_config: ServerConfig + # 认证配置 + auth_config: AuthConfig diff --git a/src/bk-user/bkuser/plugins/general/plugin.py b/src/bk-user/bkuser/plugins/general/plugin.py new file mode 100644 index 000000000..6e3cdd203 --- /dev/null +++ b/src/bk-user/bkuser/plugins/general/plugin.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from typing import Any, Dict, List + +from django.utils.translation import gettext_lazy as _ + +from bkuser.plugins.base import BaseDataSourcePlugin +from bkuser.plugins.constants import DataSourcePluginEnum +from bkuser.plugins.general.exceptions import RequestApiError, RespDataFormatError +from bkuser.plugins.general.http import fetch_all_data, fetch_first_item, gen_headers +from bkuser.plugins.general.models import GeneralDataSourcePluginConfig +from bkuser.plugins.models import ( + RawDataSourceDepartment, + RawDataSourceUser, + TestConnectionResult, +) + +logger = logging.getLogger(__name__) + + +class GeneralDataSourcePlugin(BaseDataSourcePlugin): + """通用 HTTP 数据源插件""" + + id = DataSourcePluginEnum.GENERAL + config_class = GeneralDataSourcePluginConfig + + def __init__(self, plugin_config: GeneralDataSourcePluginConfig): + self.plugin_config = plugin_config + + def fetch_departments(self) -> List[RawDataSourceDepartment]: + """获取部门信息""" + cfg = self.plugin_config.server_config + depts = fetch_all_data( + cfg.server_base_url + cfg.department_api_path, + gen_headers(self.plugin_config.auth_config), + cfg.page_size, + cfg.request_timeout, + cfg.retries, + ) + return [self._gen_raw_dept(d) for d in depts] + + def fetch_users(self) -> List[RawDataSourceUser]: + """获取用户信息""" + cfg = self.plugin_config.server_config + users = fetch_all_data( + cfg.server_base_url + cfg.user_api_path, + gen_headers(self.plugin_config.auth_config), + cfg.page_size, + cfg.request_timeout, + cfg.retries, + ) + return [self._gen_raw_user(u) for u in users] + + def test_connection(self) -> TestConnectionResult: + """连通性测试""" + cfg = self.plugin_config.server_config + err_msg, user, dept = "", None, None + user_data, dept_data = None, None + try: + user_data = fetch_first_item( + cfg.server_base_url + cfg.user_api_path, + gen_headers(self.plugin_config.auth_config), + cfg.request_timeout, + ) + + dept_data = fetch_first_item( + cfg.server_base_url + cfg.department_api_path, + gen_headers(self.plugin_config.auth_config), + cfg.request_timeout, + ) + except (RequestApiError, RespDataFormatError) as e: + err_msg = str(e) + except Exception as e: + logger.exception("general data source plugin test connection error") + err_msg = str(e) + + if not (user_data and dept_data): + err_msg = _("获取到的用户/部门数据为空，请检查数据源 API 服务") + else: + try: + user = self._gen_raw_user(user_data) + dept = self._gen_raw_dept(dept_data) + except Exception: + err_msg = _("解析用户/部门数据失败，请确保 API 返回数据符合协议规范") + + return TestConnectionResult( + error_message=str(err_msg), + user=user, + department=dept, + extras={"user_data": user_data, "department_data": dept_data}, + ) + + @staticmethod + def _gen_raw_user(user: Dict[str, Any]) -> RawDataSourceUser: + return RawDataSourceUser( + code=user["id"], + properties={ + "username": user["username"], + "full_name": user["full_name"], + "email": user["email"], + "phone": user["phone"], + "phone_country_code": user["phone_country_code"], + **user["extras"], + }, + leaders=user["leaders"], + departments=user["departments"], + ) + + @staticmethod + def _gen_raw_dept(dept: Dict[str, Any]) -> RawDataSourceDepartment: + return RawDataSourceDepartment( + code=dept["id"], + name=dept["name"], + parent=dept["parent"], + ) diff --git a/src/bk-user/bkuser/plugins/local/__init__.py b/src/bk-user/bkuser/plugins/local/__init__.py index 08cc6bd7e..cd04006dc 100644 --- a/src/bk-user/bkuser/plugins/local/__init__.py +++ b/src/bk-user/bkuser/plugins/local/__init__.py @@ -10,8 +10,7 @@ """ from bkuser.plugins.base import register_plugin -from bkuser.plugins.constants import DataSourcePluginEnum from .plugin import LocalDataSourcePlugin -register_plugin(DataSourcePluginEnum.LOCAL, LocalDataSourcePlugin) +register_plugin(LocalDataSourcePlugin) diff --git a/src/bk-user/bkuser/plugins/local/constants.py b/src/bk-user/bkuser/plugins/local/constants.py index e83d72e85..a5574f0e1 100644 --- a/src/bk-user/bkuser/plugins/local/constants.py +++ b/src/bk-user/bkuser/plugins/local/constants.py @@ -38,7 +38,7 @@ MAX_RESERVED_PREVIOUS_PASSWORD_COUNT = 5 # 数据源用户名规则 -USERNAME_REGEX = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{2,31}") +USERNAME_REGEX = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{1,30}[a-zA-Z0-9]$") class PasswordGenerateMethod(str, StructuredEnum): diff --git a/src/bk-user/bkuser/plugins/local/exceptions.py b/src/bk-user/bkuser/plugins/local/exceptions.py index d7763bcda..444cac211 100644 --- a/src/bk-user/bkuser/plugins/local/exceptions.py +++ b/src/bk-user/bkuser/plugins/local/exceptions.py @@ -35,9 +35,13 @@ class RequiredFieldIsEmpty(LocalDataSourcePluginError): """待导入文件中必填字段为空""" -class DuplicateUsername(LocalDataSourcePluginError): - """待导入文件中存在重复用户""" +class InvalidLeader(LocalDataSourcePluginError): + """待导入的用不上级信息不合法""" + +class InvalidUsername(LocalDataSourcePluginError): + """待导入的用户名非法""" -class UserLeaderInvalid(LocalDataSourcePluginError): - """待导入文件中直接上级数据有误""" + +class DuplicateUsername(LocalDataSourcePluginError): + """待导入文件中存在重复用户""" diff --git a/src/bk-user/bkuser/plugins/local/logo.png b/src/bk-user/bkuser/plugins/local/logo.png new file mode 100644 index 000000000..736c90df5 Binary files /dev/null and b/src/bk-user/bkuser/plugins/local/logo.png differ diff --git a/src/bk-user/bkuser/plugins/local/parser.py b/src/bk-user/bkuser/plugins/local/parser.py index 99fbb5c6f..d6586cc0f 100644 --- a/src/bk-user/bkuser/plugins/local/parser.py +++ b/src/bk-user/bkuser/plugins/local/parser.py @@ -16,13 +16,15 @@ from django.utils.translation import gettext_lazy as _ from openpyxl.workbook import Workbook +from bkuser.plugins.local.constants import USERNAME_REGEX from bkuser.plugins.local.exceptions import ( CustomColumnNameInvalid, DuplicateColumnName, DuplicateUsername, + InvalidLeader, + InvalidUsername, RequiredFieldIsEmpty, SheetColumnsNotMatch, - UserLeaderInvalid, UserSheetNotExists, ) from bkuser.plugins.local.utils import gen_code @@ -120,25 +122,30 @@ def _validate_and_prepare(self): # noqa: C901 if duplicate_col_names := [n for n, cnt in Counter(sheet_col_names).items() if cnt > 1]: raise DuplicateColumnName(_("待导入文件中存在重复列名：{}").format(", ".join(duplicate_col_names))) - usernames, leaders = [], [] - # 5. 检查所有必填字段是否有值 + all_usernames = [] for row in self.sheet.iter_rows(min_row=self.user_data_min_row_idx): info = dict(zip(self.all_field_names, [cell.value for cell in row], strict=True)) + # 5. 检查所有必填字段是否有值 for field_name in self.required_field_names: if not info.get(field_name): raise RequiredFieldIsEmpty(_("待导入文件中必填字段 {} 存在空值").format(field_name)) - usernames.append(info["username"]) - if user_leaders := info["leaders"]: - leaders.extend([ld.strip() for ld in user_leaders.split(",") if ld]) + username = info["username"] + # 6. 检查用户名是否合法 + if not USERNAME_REGEX.fullmatch(username): + raise InvalidUsername( + _("用户名 {} 不符合命名规范: 由3-32位字母、数字、下划线(_)、点(.)、连接符(-)字符组成，以字母或数字开头").format(username) # noqa: E501 + ) - # 6. 检查用户名是否有重复的 - if duplicate_usernames := [n for n, cnt in Counter(usernames).items() if cnt > 1]: - raise DuplicateUsername(_("待导入文件中存在重复用户名：{}").format(", ".join(duplicate_usernames))) + # 7. 检查用户不能是自己的 leader + if (leaders := info.get("leaders")) and username in [ld.strip() for ld in leaders.split(",")]: + raise InvalidLeader(_("待导入文件中用户 {} 不能是自己的直接上级").format(username)) - # 7. 检查 leaders 是不是都存在 - if not_exists_leaders := set(leaders) - set(usernames): - raise UserLeaderInvalid(_("待导入文件中不存在用户上级信息：{}").format(", ".join(not_exists_leaders))) + all_usernames.append(username) + + # 8. 检查用户名是否有重复的 + if duplicate_usernames := [n for n, cnt in Counter(all_usernames).items() if cnt > 1]: + raise DuplicateUsername(_("待导入文件中存在重复用户名：{}").format(", ".join(duplicate_usernames))) def _parse_departments(self): organizations = set() @@ -164,12 +171,17 @@ def _parse_users(self): for row in self.sheet.iter_rows(min_row=self.user_data_min_row_idx): properties = dict(zip(self.all_field_names, [cell.value for cell in row], strict=True)) - department_codes, leader_codes = [], [] + departments, leaders = [], [] if organizations := properties.pop("organizations"): - department_codes = [gen_code(org.strip()) for org in organizations.split(",") if org] + for org in organizations.split(","): + if org := org.strip(): + departments.append(gen_code(org)) # noqa: PERF401 - if leaders := properties.pop("leaders"): - leader_codes = [gen_code(ld.strip()) for ld in leaders.split(",") if ld] + if leader_names := properties.pop("leaders"): + for ld in leader_names.split(","): + if ld := ld.strip(): + # xlsx 中填写的是 leader 的 username，但在本地数据源中，username 就是 code + leaders.append(ld) # noqa: PERF401 phone_number = str(properties.pop("phone_number")) # 默认认为是不带国际代码的 @@ -184,9 +196,10 @@ def _parse_users(self): properties = {k: str(v) for k, v in properties.items() if v is not None} self.users.append( RawDataSourceUser( - code=gen_code(properties["username"]), + # 本地数据源用户，code 就是 username + code=properties["username"], properties=properties, - leaders=leader_codes, - departments=department_codes, + leaders=leaders, + departments=departments, ) ) diff --git a/src/bk-user/bkuser/plugins/local/plugin.py b/src/bk-user/bkuser/plugins/local/plugin.py index e93db54bd..3d67869c1 100644 --- a/src/bk-user/bkuser/plugins/local/plugin.py +++ b/src/bk-user/bkuser/plugins/local/plugin.py @@ -14,6 +14,7 @@ from openpyxl.workbook import Workbook from bkuser.plugins.base import BaseDataSourcePlugin +from bkuser.plugins.constants import DataSourcePluginEnum from bkuser.plugins.local.models import LocalDataSourcePluginConfig from bkuser.plugins.local.parser import LocalDataSourceDataParser from bkuser.plugins.models import ( @@ -26,6 +27,7 @@ class LocalDataSourcePlugin(BaseDataSourcePlugin): """本地数据源插件""" + id = DataSourcePluginEnum.LOCAL config_class = LocalDataSourcePluginConfig def __init__(self, plugin_config: LocalDataSourcePluginConfig, workbook: Workbook): diff --git a/src/bk-user/bkuser/plugins/local/utils.py b/src/bk-user/bkuser/plugins/local/utils.py index 967453d8a..82d78055f 100644 --- a/src/bk-user/bkuser/plugins/local/utils.py +++ b/src/bk-user/bkuser/plugins/local/utils.py @@ -11,7 +11,8 @@ from hashlib import sha256 -def gen_code(username_or_org: str) -> str: - # 本地数据源数据没有提供用户及部门 code 的方式， +def gen_code(org: str) -> str: + # 本地数据源数据没有提供部门 code 的方式， # 因此使用 sha256 计算以避免冲突，也便于后续插入 DB 时进行比较 - return sha256(username_or_org.encode("utf-8")).hexdigest() + # 注意：本地数据源用户 code 就是 username，不需要额外计算 code + return sha256(org.encode("utf-8")).hexdigest() diff --git a/src/bk-user/bkuser/plugins/models.py b/src/bk-user/bkuser/plugins/models.py index e54aed27b..152bb6aed 100644 --- a/src/bk-user/bkuser/plugins/models.py +++ b/src/bk-user/bkuser/plugins/models.py @@ -8,11 +8,19 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from typing import Dict, List +from typing import Any, Dict, List from pydantic import BaseModel +class PluginMetadata(BaseModel): + """插件基本信息""" + + id: str + name: str + description: str + + class RawDataSourceUser(BaseModel): """原始数据源用户信息""" @@ -20,9 +28,9 @@ class RawDataSourceUser(BaseModel): code: str # 用户名，邮箱，手机号等个人信息 properties: Dict[str, str] - # 直接上级信息 + # 直接上级信息（code） leaders: List[str] - # 所属部门信息 + # 所属部门信息（code） departments: List[str] @@ -40,6 +48,11 @@ class RawDataSourceDepartment(BaseModel): class TestConnectionResult(BaseModel): """连通性测试结果，包含示例数据""" + # 连通性测试错误信息，空则表示正常 error_message: str - user: RawDataSourceUser - department: RawDataSourceDepartment + # 获取到的首个数据源用户 + user: RawDataSourceUser | None + # 获取到的首个数据源部门 + department: RawDataSourceDepartment | None + # 可能便于排查问题的额外数据 + extras: Dict[str, Any] | None = None diff --git a/src/bk-user/bkuser/utils/base64.py b/src/bk-user/bkuser/utils/base64.py new file mode 100644 index 000000000..25815ee00 --- /dev/null +++ b/src/bk-user/bkuser/utils/base64.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import base64 +from pathlib import Path + + +def load_image_as_base64(image_path: Path) -> str: + """加载指定的 PNG 图片文件，并转换成 base64 字符串""" + if image_path.suffix != ".png": + raise ValueError("only PNG image supported") + + with open(image_path, "rb") as f: + content = base64.b64encode(f.read()).decode("utf-8") + + return "data:image/png;base64," + content diff --git a/src/bk-user/pyproject.toml b/src/bk-user/pyproject.toml index 419c10c35..0cb2c3c9a 100644 --- a/src/bk-user/pyproject.toml +++ b/src/bk-user/pyproject.toml @@ -171,6 +171,19 @@ layers = [ "bkuser.apps.data_source | bkuser.apps.idp", ] +# apos.sync 分层 +[[tool.importlinter.contracts]] +name = "Apps sync Layers contract" +type = "layers" +layers = [ + "bkuser.apps.sync.periodic_tasks", + "bkuser.apps.sync.managers", + "bkuser.apps.sync.tasks", + "bkuser.apps.sync.runners", + "bkuser.apps.sync.syncers", + "bkuser.apps.sync.models", +] + # biz 分层 [[tool.importlinter.contracts]] name = "Biz Layers contract" diff --git a/src/bk-user/tests/apis/web/data_source/test_data_source.py b/src/bk-user/tests/apis/web/data_source/test_data_source.py index f9f4edc25..ec9f84dad 100644 --- a/src/bk-user/tests/apis/web/data_source/test_data_source.py +++ b/src/bk-user/tests/apis/web/data_source/test_data_source.py @@ -8,9 +8,10 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +from typing import Any, Dict, List import pytest -from bkuser.apps.data_source.constants import DataSourceStatus +from bkuser.apps.data_source.constants import DataSourceStatus, FieldMappingOperation from bkuser.apps.data_source.models import DataSource from bkuser.plugins.constants import DataSourcePluginEnum from django.urls import reverse @@ -23,7 +24,7 @@ @pytest.fixture() -def data_source(request, local_ds_plugin, local_ds_plugin_config): +def data_source(request, local_ds_plugin, local_ds_plugin_cfg) -> DataSource: # 支持检查是否使用 random_tenant fixture 以生成不属于默认租户的数据源 tenant_id = DEFAULT_TENANT if "random_tenant" in request.fixturenames: @@ -33,10 +34,26 @@ def data_source(request, local_ds_plugin, local_ds_plugin_config): name=generate_random_string(), owner_tenant_id=tenant_id, plugin=local_ds_plugin, - plugin_config=local_ds_plugin_config, + plugin_config=local_ds_plugin_cfg, ) +@pytest.fixture() +def field_mapping(request) -> List[Dict]: + """字段映射，不含自定义字段""" + fields = ["username", "full_name", "phone_country_code", "phone", "email"] + if "tenant_user_custom_fields" in request.fixturenames: + fields += [f.name for f in request.getfixturevalue("tenant_user_custom_fields")] + + return [{"source_field": f, "mapping_operation": "direct", "target_field": f} for f in fields] + + +@pytest.fixture() +def sync_config() -> Dict[str, Any]: + """数据源同步配置""" + return {"sync_period": 30} + + class TestDataSourcePluginListApi: def test_list(self, api_client): resp = api_client.get(reverse("data_source_plugin.list")) @@ -56,13 +73,13 @@ def test_retrieve_not_exists(self, api_client): class TestDataSourceCreateApi: - def test_create_local_data_source(self, api_client, local_ds_plugin_config): + def test_create_local_data_source(self, api_client, local_ds_plugin_cfg): resp = api_client.post( reverse("data_source.list_create"), data={ "name": generate_random_string(), "plugin_id": DataSourcePluginEnum.LOCAL, - "plugin_config": local_ds_plugin_config, + "plugin_config": local_ds_plugin_cfg, # 本地数据源不需要字段映射配置 }, ) @@ -99,97 +116,140 @@ def test_create_without_plugin_config(self, api_client): assert resp.status_code == status.HTTP_400_BAD_REQUEST assert "plugin_config: 该字段是必填项。" in resp.data["message"] - def test_create_with_broken_plugin_config(self, api_client, local_ds_plugin_config): - local_ds_plugin_config["password_initial"] = None + def test_create_with_broken_plugin_config(self, api_client, local_ds_plugin_cfg): + local_ds_plugin_cfg["password_initial"] = None resp = api_client.post( reverse("data_source.list_create"), data={ "name": generate_random_string(), "plugin_id": DataSourcePluginEnum.LOCAL, - "plugin_config": local_ds_plugin_config, + "plugin_config": local_ds_plugin_cfg, }, ) assert resp.status_code == status.HTTP_400_BAD_REQUEST assert "密码生成规则、初始密码设置、密码到期设置均不能为空" in resp.data["message"] - def test_create_with_invalid_notification_template(self, api_client, local_ds_plugin_config): - local_ds_plugin_config["password_expire"]["notification"]["templates"][0]["title"] = None + def test_create_with_invalid_notification_template(self, api_client, local_ds_plugin_cfg): + local_ds_plugin_cfg["password_expire"]["notification"]["templates"][0]["title"] = None resp = api_client.post( reverse("data_source.list_create"), data={ "name": generate_random_string(), "plugin_id": DataSourcePluginEnum.LOCAL, - "plugin_config": local_ds_plugin_config, + "plugin_config": local_ds_plugin_cfg, }, ) assert resp.status_code == status.HTTP_400_BAD_REQUEST assert "邮件通知模板需要提供标题" in resp.data["message"] - def test_create_with_invalid_plugin_config(self, api_client, local_ds_plugin_config): - local_ds_plugin_config.pop("enable_account_password_login") + def test_create_with_invalid_plugin_config(self, api_client, local_ds_plugin_cfg): + local_ds_plugin_cfg.pop("enable_account_password_login") resp = api_client.post( reverse("data_source.list_create"), data={ "name": generate_random_string(), "plugin_id": DataSourcePluginEnum.LOCAL, - "plugin_config": local_ds_plugin_config, + "plugin_config": local_ds_plugin_cfg, }, ) assert resp.status_code == status.HTTP_400_BAD_REQUEST assert "插件配置不合法：enable_account_password_login: Field required" in resp.data["message"] - # TODO (su) 支持通用 HTTP 数据源后启用 - # def test_create_with_field_mappings(self, api_client, general_ds_plugin_config, tenant_user_custom_fields): - # """非本地数据源，需要提供字段映射""" - # resp = api_client.post( - # reverse("data_source.list_create"), - # data={ - # "name": generate_random_string(), - # "plugin_id": DataSourcePluginEnum.GENERAL, - # "plugin_config": general_ds_plugin_config, - # "field_mappings": [ - # { - # "source_field": "uname", - # "mapping_operation": FieldMappingOperation.DIRECT, - # "target_field": "username", - # } - # ], - # }, - # ) - # assert resp.status_code == status.HTTP_201_CREATED - # - # def test_create_without_required_field_mapping(self, api_client, general_ds_plugin_config): - # """非本地数据源，需要字段映射配置""" - # resp = api_client.post( - # reverse("data_source.list_create"), - # data={ - # "name": generate_random_string(), - # "plugin_id": DataSourcePluginEnum.GENERAL, - # "plugin_config": general_ds_plugin_config, - # "field_mappings": [], - # }, - # ) - # assert resp.status_code == status.HTTP_400_BAD_REQUEST - # assert "当前数据源类型必须配置字段映射" in resp.data["message"] - # - # def test_create_with_invalid_field_mappings(self, api_client, general_ds_plugin_config): - # resp = api_client.post( - # reverse("data_source.list_create"), - # data={ - # "name": generate_random_string(), - # "plugin_id": DataSourcePluginEnum.GENERAL, - # "plugin_config": general_ds_plugin_config, - # "field_mappings": [ - # { - # "source_field": "uname", - # "mapping_operation": FieldMappingOperation.DIRECT, - # "target_field": "xxx_username", - # } - # ], - # }, - # ) - # assert resp.status_code == status.HTTP_400_BAD_REQUEST - # assert "字段映射中的目标字段 xxx_username 不属于用户自定义字段或内置字段" in resp.data["message"] + def test_create_general_data_source( + self, api_client, general_ds_plugin_cfg, tenant_user_custom_fields, field_mapping, sync_config + ): + """非本地数据源，需要提供字段映射""" + resp = api_client.post( + reverse("data_source.list_create"), + data={ + "name": generate_random_string(), + "plugin_id": DataSourcePluginEnum.GENERAL, + "plugin_config": general_ds_plugin_cfg, + "field_mapping": field_mapping, + "sync_config": sync_config, + }, + ) + assert resp.status_code == status.HTTP_201_CREATED + + def test_create_without_required_field_mapping(self, api_client, general_ds_plugin_cfg, sync_config): + """非本地数据源，需要字段映射配置""" + resp = api_client.post( + reverse("data_source.list_create"), + data={ + "name": generate_random_string(), + "plugin_id": DataSourcePluginEnum.GENERAL, + "plugin_config": general_ds_plugin_cfg, + "field_mapping": [], + "sync_config": sync_config, + }, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + assert "当前数据源类型必须配置字段映射" in resp.data["message"] + + def test_create_with_invalid_field_mapping_case_not_allowed_field(self, api_client, general_ds_plugin_cfg): + resp = api_client.post( + reverse("data_source.list_create"), + data={ + "name": generate_random_string(), + "plugin_id": DataSourcePluginEnum.GENERAL, + "plugin_config": general_ds_plugin_cfg, + "field_mapping": [ + { + "source_field": "uname", + "mapping_operation": FieldMappingOperation.DIRECT, + "target_field": "xxx_username", + } + ], + }, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + assert "字段映射中的目标字段 {'xxx_username'} 不属于用户自定义字段或内置字段" in resp.data["message"] + + def test_create_with_invalid_field_mapping_case_missed_field(self, api_client, general_ds_plugin_cfg): + resp = api_client.post( + reverse("data_source.list_create"), + data={ + "name": generate_random_string(), + "plugin_id": DataSourcePluginEnum.GENERAL, + "plugin_config": general_ds_plugin_cfg, + "field_mapping": [ + { + "source_field": "uname", + "mapping_operation": FieldMappingOperation.DIRECT, + "target_field": "username", + } + ], + }, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + assert "缺少字段映射" in resp.data["message"] + + def test_create_without_sync_config(self, api_client, general_ds_plugin_cfg, field_mapping): + resp = api_client.post( + reverse("data_source.list_create"), + data={ + "name": generate_random_string(), + "plugin_id": DataSourcePluginEnum.GENERAL, + "plugin_config": general_ds_plugin_cfg, + "field_mapping": field_mapping, + }, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + assert "当前数据源类型必须提供同步配置" in resp.data["message"] + + def test_create_with_invalid_sync_config(self, api_client, general_ds_plugin_cfg, field_mapping): + resp = api_client.post( + reverse("data_source.list_create"), + data={ + "name": generate_random_string(), + "plugin_id": DataSourcePluginEnum.GENERAL, + "plugin_config": general_ds_plugin_cfg, + "field_mapping": field_mapping, + "sync_config": {"sync_period": -1}, + }, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + assert "sync_config.sync_period: “-1” 不是合法选项。" in resp.data["message"] class TestDataSourceListApi: @@ -216,12 +276,12 @@ def test_list_other_tenant_data_source(self, api_client, random_tenant, data_sou class TestDataSourceUpdateApi: - def test_update_local_data_source(self, api_client, data_source, local_ds_plugin_config): + def test_update_local_data_source(self, api_client, data_source, local_ds_plugin_cfg): new_data_source_name = generate_random_string() - local_ds_plugin_config["enable_account_password_login"] = False + local_ds_plugin_cfg["enable_account_password_login"] = False resp = api_client.put( reverse("data_source.retrieve_update", kwargs={"id": data_source.id}), - data={"name": new_data_source_name, "plugin_config": local_ds_plugin_config}, + data={"name": new_data_source_name, "plugin_config": local_ds_plugin_cfg}, ) assert resp.status_code == status.HTTP_204_NO_CONTENT @@ -229,18 +289,58 @@ def test_update_local_data_source(self, api_client, data_source, local_ds_plugin assert resp.data["name"] == new_data_source_name assert resp.data["plugin_config"]["enable_account_password_login"] is False - def test_update_with_invalid_plugin_config(self, api_client, data_source, local_ds_plugin_config): - local_ds_plugin_config.pop("enable_account_password_login") + def test_update_with_invalid_plugin_config(self, api_client, data_source, local_ds_plugin_cfg): + local_ds_plugin_cfg.pop("enable_account_password_login") resp = api_client.put( reverse("data_source.retrieve_update", kwargs={"id": data_source.id}), - data={"name": generate_random_string(), "plugin_config": local_ds_plugin_config}, + data={"name": generate_random_string(), "plugin_config": local_ds_plugin_cfg}, ) assert resp.status_code == status.HTTP_400_BAD_REQUEST assert "插件配置不合法：enable_account_password_login: Field required" in resp.data["message"] - def test_update_without_required_field_mapping(self, api_client): + def test_update_general_data_source( + self, api_client, bare_general_data_source, general_ds_plugin_cfg, field_mapping, sync_config + ): + resp = api_client.put( + reverse("data_source.retrieve_update", kwargs={"id": bare_general_data_source.id}), + data={ + "name": generate_random_string(), + "plugin_config": general_ds_plugin_cfg, + "field_mapping": field_mapping, + "sync_config": sync_config, + }, + ) + assert resp.status_code == status.HTTP_204_NO_CONTENT + + def test_update_without_required_field_mapping( + self, api_client, bare_general_data_source, general_ds_plugin_cfg, sync_config + ): """非本地数据源，需要字段映射配置""" - # TODO 需要内置非本地的数据源插件后补全测试用例 + resp = api_client.put( + reverse("data_source.retrieve_update", kwargs={"id": bare_general_data_source.id}), + data={ + "name": generate_random_string(), + "plugin_config": general_ds_plugin_cfg, + "sync_config": sync_config, + }, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + assert resp.data["message"] == "参数校验不通过: 当前数据源类型必须配置字段映射" + + def test_update_without_required_sync_config( + self, api_client, bare_general_data_source, general_ds_plugin_cfg, field_mapping + ): + """非本地数据源，需要同步配置""" + resp = api_client.put( + reverse("data_source.retrieve_update", kwargs={"id": bare_general_data_source.id}), + data={ + "name": generate_random_string(), + "plugin_config": general_ds_plugin_cfg, + "field_mapping": field_mapping, + }, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + assert resp.data["message"] == "参数校验不通过: 当前数据源类型必须提供同步配置" class TestDataSourceRetrieveApi: diff --git a/src/bk-user/tests/apps/sync/conftest.py b/src/bk-user/tests/apps/sync/conftest.py index 3d7d68783..8576ac6ff 100644 --- a/src/bk-user/tests/apps/sync/conftest.py +++ b/src/bk-user/tests/apps/sync/conftest.py @@ -68,7 +68,7 @@ def raw_users() -> List[RawDataSourceUser]: """数据源插件提供的原始用户信息""" return [ RawDataSourceUser( - code="Employee-3", + code="zhangsan", properties={ "username": "zhangsan", "full_name": "张三", @@ -82,7 +82,7 @@ def raw_users() -> List[RawDataSourceUser]: departments=["company"], ), RawDataSourceUser( - code="Employee-4", + code="lisi", properties={ "username": "lisi", "full_name": "李四", @@ -92,11 +92,11 @@ def raw_users() -> List[RawDataSourceUser]: "gender": "female", "region": "shanghai", }, - leaders=["Employee-3"], + leaders=["zhangsan"], departments=["dept_a", "center_aa"], ), RawDataSourceUser( - code="Employee-5", + code="wangwu", properties={ "username": "wangwu", "full_name": "王五", @@ -106,11 +106,11 @@ def raw_users() -> List[RawDataSourceUser]: "gender": "male", "region": "shenzhen", }, - leaders=["Employee-3"], + leaders=["zhangsan"], departments=["dept_a", "dept_b"], ), RawDataSourceUser( - code="Employee-6", + code="zhaoliu", properties={ "username": "zhaoliu", "full_name": "赵六", @@ -120,11 +120,11 @@ def raw_users() -> List[RawDataSourceUser]: "gender": "female", "region": "tianjin", }, - leaders=["Employee-4"], + leaders=["lisi"], departments=["center_aa"], ), RawDataSourceUser( - code="Employee-7", + code="liuqi", properties={ "username": "liuqi", "full_name": "柳七", @@ -134,11 +134,11 @@ def raw_users() -> List[RawDataSourceUser]: "gender": "female", "region": "jiangxi", }, - leaders=["Employee-6"], + leaders=["zhaoliu"], departments=["group_aaa"], ), RawDataSourceUser( - code="Employee-8", + code="maiba", properties={ "username": "maiba", "full_name": "麦八", @@ -148,11 +148,11 @@ def raw_users() -> List[RawDataSourceUser]: "gender": "male", "region": "xinjiang", }, - leaders=["Employee-4", "Employee-5"], + leaders=["lisi", "wangwu"], departments=["center_ab"], ), RawDataSourceUser( - code="Employee-9", + code="yangjiu", properties={ "username": "yangjiu", "full_name": "杨九", @@ -162,11 +162,11 @@ def raw_users() -> List[RawDataSourceUser]: "gender": "male", "region": "guangdong", }, - leaders=["Employee-5"], + leaders=["wangwu"], departments=["center_ab"], ), RawDataSourceUser( - code="Employee-10", + code="lushi", properties={ "username": "lushi", "full_name": "鲁十", @@ -176,11 +176,11 @@ def raw_users() -> List[RawDataSourceUser]: "gender": "male", "region": "jiangsu", }, - leaders=["Employee-8", "Employee-5"], + leaders=["wangwu", "maiba"], departments=["group_aba", "center_ba"], ), RawDataSourceUser( - code="Employee-11", + code="linshiyi", properties={ "username": "linshiyi", "full_name": "林十一", @@ -190,11 +190,11 @@ def raw_users() -> List[RawDataSourceUser]: "gender": "male", "region": "hunan", }, - leaders=["Employee-10"], + leaders=["lushi"], departments=["group_aba"], ), RawDataSourceUser( - code="Employee-12", + code="baishier", properties={ "username": "baishier", "full_name": "白十二", @@ -204,11 +204,11 @@ def raw_users() -> List[RawDataSourceUser]: "gender": "female", "region": "guangdong", }, - leaders=["Employee-10"], + leaders=["lushi"], departments=["group_baa"], ), RawDataSourceUser( - code="Employee-666", + code="freedom", properties={ "username": "freedom", "full_name": "自由人", diff --git a/src/bk-user/tests/apps/sync/test_converters.py b/src/bk-user/tests/apps/sync/test_converters.py index 5375de209..d2853654b 100644 --- a/src/bk-user/tests/apps/sync/test_converters.py +++ b/src/bk-user/tests/apps/sync/test_converters.py @@ -56,7 +56,7 @@ def test_get_field_mapping_from_field_definition(self, bare_local_data_source, t def test_convert_case_1(self, bare_local_data_source, tenant_user_custom_fields): raw_zhangsan = RawDataSourceUser( - code="Employee-3", + code="zhangsan", properties={ "username": "zhangsan", "full_name": "张三", @@ -70,7 +70,7 @@ def test_convert_case_1(self, bare_local_data_source, tenant_user_custom_fields) ) zhangsan = DataSourceUserConverter(bare_local_data_source).convert(raw_zhangsan) - assert zhangsan.code == "Employee-3" + assert zhangsan.code == "zhangsan" assert zhangsan.username == "zhangsan" assert zhangsan.full_name == "张三" assert zhangsan.email == "zhangsan@m.com" @@ -80,7 +80,7 @@ def test_convert_case_1(self, bare_local_data_source, tenant_user_custom_fields) def test_convert_case_2(self, bare_local_data_source, tenant_user_custom_fields): raw_lisi = RawDataSourceUser( - code="Employee-4", + code="lisi", properties={ "username": "lisi", "full_name": "李四", @@ -90,12 +90,12 @@ def test_convert_case_2(self, bare_local_data_source, tenant_user_custom_fields) "age": "28", "gender": "female", }, - leaders=["Employee-3"], + leaders=["zhangsan"], departments=["dept_a", "center_aa"], ) lisi = DataSourceUserConverter(bare_local_data_source).convert(raw_lisi) - assert lisi.code == "Employee-4" + assert lisi.code == "lisi" assert lisi.username == "lisi" assert lisi.full_name == "李四" assert lisi.email == "lisi@m.com" diff --git a/src/bk-user/tests/apps/sync/test_handlers.py b/src/bk-user/tests/apps/sync/test_handlers.py new file mode 100644 index 000000000..0d5ae68f5 --- /dev/null +++ b/src/bk-user/tests/apps/sync/test_handlers.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import pytest +from bkuser.apps.sync.names import gen_data_source_sync_periodic_task_name +from django_celery_beat.models import PeriodicTask + +pytestmark = pytest.mark.django_db + + +def test_set_data_source_sync_periodic_task_with_local(bare_local_data_source): + """本地数据源，不会创建周期任务""" + task_name = gen_data_source_sync_periodic_task_name(bare_local_data_source.id) + assert not PeriodicTask.objects.filter(name=task_name).exists() + + +def test_set_data_source_sync_periodic_task_with_general(bare_general_data_source): + """通用 HTTP 数据源，创建任务并更新，最后删除""" + task_name = gen_data_source_sync_periodic_task_name(bare_general_data_source.id) + task = PeriodicTask.objects.get(name=task_name) + assert task.interval.every == 60 # noqa: PLR2004 + + bare_general_data_source.sync_config = {"sync_period": 30} # noqa: PLR2004 + bare_general_data_source.save() + task = PeriodicTask.objects.get(name=task_name) + assert task.interval.every == 30 # noqa: PLR2004 + + bare_general_data_source.sync_config = {} + bare_general_data_source.save() + assert not PeriodicTask.objects.filter(name=task_name).exists() diff --git a/src/bk-user/tests/apps/sync/test_syncers.py b/src/bk-user/tests/apps/sync/test_syncers.py index 31f72196d..038fa3cae 100644 --- a/src/bk-user/tests/apps/sync/test_syncers.py +++ b/src/bk-user/tests/apps/sync/test_syncers.py @@ -20,6 +20,7 @@ DataSourceUser, DataSourceUserLeaderRelation, ) +from bkuser.apps.sync.exceptions import UserLeaderNotExists from bkuser.apps.sync.syncers import ( DataSourceDepartmentSyncer, DataSourceUserSyncer, @@ -128,7 +129,7 @@ def test_update_with_overwrite( raw_users[0].properties["phone_country_code"] = "63" raw_users[0].properties["age"] = "30" # 2. 修改用户的 code，会导致用户被重建 - lisi_old_code, lisi_new_code = "Employee-4", "Employee-4-1" + lisi_old_code, lisi_new_code = "lisi", "lisi-1" raw_users[1].code = lisi_new_code # 需要更新其他用户的信息，避免 leader 还是用旧的 Code for u in raw_users: @@ -153,7 +154,7 @@ def test_update_with_overwrite( assert all(bool(e) for e in users.values_list("extras", flat=True)) # 验证内置/自定义字段被更新 - zhangsan = users.filter(code="Employee-3").first() + zhangsan = users.filter(code="zhangsan").first() assert zhangsan.username == "zhangsan_rename" assert zhangsan.full_name == "张三的另一个名字" assert zhangsan.email == "zhangsan_rename@m.com" @@ -165,7 +166,7 @@ def test_update_with_overwrite( lisi = users.filter(username="lisi").first() assert lisi.full_name == "李四" assert lisi.email == "lisi@m.com" - assert lisi.code == "Employee-4-1" + assert lisi.code == "lisi-1" # 验证用户部门信息 assert self._gen_user_depts_from_db(users) == self._gen_user_depts_from_raw_users(raw_users) @@ -198,7 +199,7 @@ def test_update_without_overwrite(self, data_source_sync_task, full_local_data_s assert not any(bool(e) for e in users.values_list("extras", flat=True)) # 验证内置/自定义字段都不会被更新，因为没有选择 overwrite - zhangsan = users.filter(code="Employee-3").first() + zhangsan = users.filter(code="zhangsan").first() assert zhangsan.username == "zhangsan" assert zhangsan.full_name == "张三" assert zhangsan.email == "zhangsan@m.com" @@ -219,7 +220,21 @@ def test_update_with_incremental(self, data_source_sync_task, full_local_data_so users = DataSourceUser.objects.filter(data_source=full_local_data_source) assert set(users.values_list("code", flat=True)) == user_codes - def destroy(self, data_source_sync_task, full_local_data_source): + def test_update_with_invalid_leader(self, data_source_sync_task, full_local_data_source, random_raw_user): + """全量同步模式，要求用户的 leader 必须也在数据中""" + random_raw_user.leaders.append("lisi") + with pytest.raises(UserLeaderNotExists): + DataSourceUserSyncer(data_source_sync_task, full_local_data_source, [random_raw_user]).sync() + + def test_update_with_leader_in_db(self, data_source_sync_task, full_local_data_source, random_raw_user): + """增量同步模式，用户的 leader 在 db 中存在也是可以的""" + data_source_sync_task.extra["incremental"] = True + random_raw_user.leaders.append("lisi") + + DataSourceUserSyncer(data_source_sync_task, full_local_data_source, [random_raw_user]).sync() + assert DataSourceUser.objects.filter(code=random_raw_user.code).count() == 1 + + def test_destroy(self, data_source_sync_task, full_local_data_source): raw_users: List[RawDataSourceUser] = [] DataSourceUserSyncer(data_source_sync_task, full_local_data_source, raw_users).sync() @@ -314,7 +329,7 @@ def test_cud(self, tenant_sync_task, full_local_data_source, bare_general_data_s # 另外的数据源同步到租户的数据 other_ds_user = DataSourceUser.objects.create( data_source=bare_general_data_source, - code="Employee-201", + code="libai", username="libai", full_name="李白", email="libai@m.com", @@ -336,11 +351,12 @@ def test_cud(self, tenant_sync_task, full_local_data_source, bare_general_data_s # 更新场景 DataSourceUser.objects.filter( - data_source=full_local_data_source, code__in=["Employee-9", "Employee-10"] + data_source=full_local_data_source, + code__in=["yangjiu", "lushi"], ).delete() DataSourceUser.objects.create( data_source=full_local_data_source, - code="Employee-20", + code="xiaoershi", username="xiaoershi", full_name="萧二十", email="xiaoershi@m.com", diff --git a/src/bk-user/tests/conftest.py b/src/bk-user/tests/conftest.py index e41754062..58371c2f3 100644 --- a/src/bk-user/tests/conftest.py +++ b/src/bk-user/tests/conftest.py @@ -18,9 +18,9 @@ full_general_data_source, full_local_data_source, general_ds_plugin, - general_ds_plugin_config, + general_ds_plugin_cfg, local_ds_plugin, - local_ds_plugin_config, + local_ds_plugin_cfg, ) from tests.fixtures.tenant import tenant_user_custom_fields # noqa: F401 from tests.test_utils.auth import create_user diff --git a/src/bk-user/tests/fixtures/data_source.py b/src/bk-user/tests/fixtures/data_source.py index 5faf8d7ce..70addd44f 100644 --- a/src/bk-user/tests/fixtures/data_source.py +++ b/src/bk-user/tests/fixtures/data_source.py @@ -20,7 +20,7 @@ @pytest.fixture() -def local_ds_plugin_config() -> Dict[str, Any]: +def local_ds_plugin_cfg() -> Dict[str, Any]: return { "enable_account_password_login": True, "password_rule": { @@ -127,13 +127,13 @@ def local_ds_plugin() -> DataSourcePlugin: @pytest.fixture() -def bare_local_data_source(local_ds_plugin_config, local_ds_plugin) -> DataSource: +def bare_local_data_source(local_ds_plugin_cfg, local_ds_plugin) -> DataSource: """裸本地数据源（没有用户，部门等数据）""" return DataSource.objects.create( name=generate_random_string(), owner_tenant_id=DEFAULT_TENANT, plugin=local_ds_plugin, - plugin_config=local_ds_plugin_config, + plugin_config=local_ds_plugin_cfg, ) @@ -145,9 +145,20 @@ def full_local_data_source(bare_local_data_source) -> DataSource: @pytest.fixture() -def general_ds_plugin_config() -> Dict[str, Any]: - # TODO (su) 预设通用 HTTP 数据源的插件配置 - return {"TODO": "TODO"} +def general_ds_plugin_cfg() -> Dict[str, Any]: + return { + "server_config": { + "server_base_url": "http://bk.example.com:8090", + "user_api_path": "/api/v1/users", + "department_api_path": "/api/v1/departments", + "request_timeout": 5, + "retries": 3, + }, + "auth_config": { + "method": "bearer_token", + "bearer_token": "123456", + }, + } @pytest.fixture() @@ -160,13 +171,14 @@ def general_ds_plugin() -> DataSourcePlugin: @pytest.fixture() -def bare_general_data_source(general_ds_plugin_config, general_ds_plugin) -> DataSource: +def bare_general_data_source(general_ds_plugin_cfg, general_ds_plugin) -> DataSource: """裸通用 HTTP 数据源（没有用户，部门等数据）""" return DataSource.objects.create( name=generate_random_string(), owner_tenant_id=DEFAULT_TENANT, plugin=general_ds_plugin, - plugin_config=general_ds_plugin_config, + plugin_config=general_ds_plugin_cfg, + sync_config={"sync_period": 60}, ) diff --git a/src/bk-user/tests/plugins/general/test_plugin.py b/src/bk-user/tests/plugins/general/test_plugin.py new file mode 100644 index 000000000..fbfb0e34a --- /dev/null +++ b/src/bk-user/tests/plugins/general/test_plugin.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from unittest import mock + +import pytest +from bkuser.plugins.general.models import GeneralDataSourcePluginConfig +from bkuser.plugins.general.plugin import GeneralDataSourcePlugin + + +@pytest.fixture() +def general_ds_cfg(general_ds_plugin_cfg) -> GeneralDataSourcePluginConfig: + return GeneralDataSourcePluginConfig(**general_ds_plugin_cfg) + + +def _mocked_fetch_first_item(url, *args, **kwargs): + if url.endswith("/simple_users"): + return { + "id": "100", + "username": "sanzhang", + } + + if url.endswith("/users"): + return { + "id": "101", + "username": "sili", + "full_name": "李四", + "email": "1234567892@qq.com", + "phone": "12345678902", + "phone_country_code": "86", + "extras": { + "gender": "female", + }, + "leaders": ["100"], + "departments": ["dept_a"], + } + + if url.endswith("departments"): + return { + "id": "dept_a", + "name": "部门A", + "parent": "company", + } + + return None + + +class TestGeneralDataSourcePlugin: + @mock.patch( + "bkuser.plugins.general.plugin.fetch_all_data", + return_value=[ + {"id": "company", "name": "总公司", "parent": None}, + {"id": "dept_a", "name": "部门A", "parent": "company"}, + {"id": "center_aa", "name": "中心AA", "parent": "dept_a"}, + ], + ) + def test_get_departments(self, general_ds_cfg): + plugin = GeneralDataSourcePlugin(general_ds_cfg) + assert len(plugin.fetch_departments()) == 3 # noqa: PLR2004 + + @mock.patch( + "bkuser.plugins.general.plugin.fetch_all_data", + return_value=[ + { + "id": "100", + "username": "sanzhang", + "full_name": "张三", + "email": "1234567891@qq.com", + "phone": "12345678901", + "phone_country_code": "86", + "extras": { + "gender": "male", + }, + "leaders": [], + "departments": ["company"], + }, + { + "id": "101", + "username": "sili", + "full_name": "李四", + "email": "1234567892@qq.com", + "phone": "12345678902", + "phone_country_code": "86", + "extras": { + "gender": "female", + }, + "leaders": ["100"], + "departments": ["dept_a"], + }, + { + "id": "102", + "username": "wuwang", + "full_name": "王五", + "email": "1234567893@qq.com", + "phone": "12345678903", + "phone_country_code": "86", + "extras": { + "gender": "male", + }, + "leaders": ["100", "101"], + "departments": ["center_aa"], + }, + ], + ) + def test_get_users(self, general_ds_cfg): + plugin = GeneralDataSourcePlugin(general_ds_cfg) + assert len(plugin.fetch_users()) == 3 # noqa: PLR2004 + + @mock.patch("bkuser.plugins.general.plugin.fetch_first_item", new=_mocked_fetch_first_item) + def test_test_connection(self, general_ds_cfg): + result = GeneralDataSourcePlugin(general_ds_cfg).test_connection() + assert not result.error_message + assert result.user + assert result.department + assert result.extras + + @mock.patch("bkuser.plugins.general.plugin.fetch_first_item", new=_mocked_fetch_first_item) + def test_test_connection_with_simple_users(self, general_ds_cfg): + """API 返回的字段不足，无法完全解析""" + general_ds_cfg.server_config.user_api_path = "/api/v1/simple_users" + result = GeneralDataSourcePlugin(general_ds_cfg).test_connection() + assert result.error_message == "解析用户/部门数据失败，请确保 API 返回数据符合协议规范" + + @mock.patch("bkuser.plugins.general.plugin.fetch_first_item", new=_mocked_fetch_first_item) + def test_test_connection_without_data(self, general_ds_cfg): + """部门 / 用户 API 返回数据为空 -> 数据源不可用""" + general_ds_cfg.server_config.user_api_path = "/api/v1/bad_path" + result = GeneralDataSourcePlugin(general_ds_cfg).test_connection() + assert result.error_message == "获取到的用户/部门数据为空，请检查数据源 API 服务" diff --git a/src/bk-user/tests/plugins/local/conftest.py b/src/bk-user/tests/plugins/local/conftest.py new file mode 100644 index 000000000..565241841 --- /dev/null +++ b/src/bk-user/tests/plugins/local/conftest.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import pytest +from django.conf import settings +from openpyxl.reader.excel import load_workbook +from openpyxl.workbook import Workbook + + +@pytest.fixture() +def user_wk() -> Workbook: + return load_workbook(settings.BASE_DIR / "tests/assets/fake_users.xlsx") diff --git a/src/bk-user/tests/plugins/local/test_parser.py b/src/bk-user/tests/plugins/local/test_parser.py index f2e8d384a..6e8ea62f2 100644 --- a/src/bk-user/tests/plugins/local/test_parser.py +++ b/src/bk-user/tests/plugins/local/test_parser.py @@ -8,27 +8,22 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +from typing import List + import pytest from bkuser.plugins.local.exceptions import ( CustomColumnNameInvalid, DuplicateColumnName, DuplicateUsername, + InvalidLeader, + InvalidUsername, RequiredFieldIsEmpty, SheetColumnsNotMatch, - UserLeaderInvalid, UserSheetNotExists, ) from bkuser.plugins.local.parser import LocalDataSourceDataParser from bkuser.plugins.local.utils import gen_code from bkuser.plugins.models import RawDataSourceDepartment, RawDataSourceUser -from django.conf import settings -from openpyxl.reader.excel import load_workbook -from openpyxl.workbook import Workbook - - -@pytest.fixture() -def user_wk() -> Workbook: - return load_workbook(settings.BASE_DIR / "tests/assets/fake_users.xlsx") class TestLocalDataSourceDataParser: @@ -62,18 +57,30 @@ def test_validate_case_required_field_is_empty(self, user_wk): with pytest.raises(RequiredFieldIsEmpty): LocalDataSourceDataParser(user_wk).parse() + def test_validate_case_invalid_username_chinese(self, user_wk): + # 修改表格数据，导致用户名非法 + user_wk["users"]["A4"].value = "张三" + with pytest.raises(InvalidUsername): + LocalDataSourceDataParser(user_wk).parse() + + def test_validate_case_invalid_username_punctuation(self, user_wk): + # 修改表格数据，导致用户名非法 + user_wk["users"]["A4"].value = "zhangsan@m.com" + with pytest.raises(InvalidUsername): + LocalDataSourceDataParser(user_wk).parse() + + def test_validate_case_invalid_leader(self, user_wk): + # 修改表格数据，导致用户是自己的 leader + user_wk["users"]["F4"].value = "zhangsan, lisi,wangwu" + with pytest.raises(InvalidLeader): + LocalDataSourceDataParser(user_wk).parse() + def test_validate_case_duplicate_username(self, user_wk): # 修改表格数据，导致用户名重复 - user_wk["users"]["A4"].value = "zhangsan" + user_wk["users"]["A6"].value = "zhangsan" with pytest.raises(DuplicateUsername): LocalDataSourceDataParser(user_wk).parse() - def test_validate_case_invalid_leaders(self, user_wk): - # 修改表格数据，导致直接上级不合法 - user_wk["users"]["F3"].value = "not_exists" - with pytest.raises(UserLeaderInvalid): - LocalDataSourceDataParser(user_wk).parse() - def test_get_departments(self, user_wk): parser = LocalDataSourceDataParser(user_wk) parser.parse() @@ -110,9 +117,12 @@ def test_get_users(self, user_wk): parser = LocalDataSourceDataParser(user_wk) parser.parse() + def gen_depts(orgs: List[str]) -> List[str]: + return [gen_code(o) for o in orgs] + assert sorted(parser.get_users(), key=lambda u: u.properties["age"]) == [ RawDataSourceUser( - code=gen_code("zhangsan"), + code="zhangsan", properties={ "username": "zhangsan", "full_name": "张三", @@ -124,10 +134,10 @@ def test_get_users(self, user_wk): "phone_country_code": "86", }, leaders=[], - departments=[gen_code("公司")], + departments=gen_depts(["公司"]), ), RawDataSourceUser( - code=gen_code("lisi"), + code="lisi", properties={ "username": "lisi", "full_name": "李四", @@ -138,11 +148,11 @@ def test_get_users(self, user_wk): "phone": "13512345672", "phone_country_code": "86", }, - leaders=[gen_code("zhangsan")], - departments=[gen_code("公司/部门A"), gen_code("公司/部门A/中心AA")], + leaders=["zhangsan"], + departments=gen_depts(["公司/部门A", "公司/部门A/中心AA"]), ), RawDataSourceUser( - code=gen_code("wangwu"), + code="wangwu", properties={ "username": "wangwu", "full_name": "王五", @@ -153,11 +163,11 @@ def test_get_users(self, user_wk): "phone": "13512345673", "phone_country_code": "63", }, - leaders=[gen_code("zhangsan")], - departments=[gen_code("公司/部门A"), gen_code("公司/部门B")], + leaders=["zhangsan"], + departments=gen_depts(["公司/部门A", "公司/部门B"]), ), RawDataSourceUser( - code=gen_code("zhaoliu"), + code="zhaoliu", properties={ "username": "zhaoliu", "full_name": "赵六", @@ -168,11 +178,11 @@ def test_get_users(self, user_wk): "phone": "13512345674", "phone_country_code": "86", }, - leaders=[gen_code("lisi")], - departments=[gen_code("公司/部门A/中心AA")], + leaders=["lisi"], + departments=gen_depts(["公司/部门A/中心AA"]), ), RawDataSourceUser( - code=gen_code("liuqi"), + code="liuqi", properties={ "username": "liuqi", "full_name": "柳七", @@ -183,11 +193,11 @@ def test_get_users(self, user_wk): "phone": "13512345675", "phone_country_code": "63", }, - leaders=[gen_code("zhaoliu")], - departments=[gen_code("公司/部门A/中心AA/小组AAA")], + leaders=["zhaoliu"], + departments=gen_depts(["公司/部门A/中心AA/小组AAA"]), ), RawDataSourceUser( - code=gen_code("maiba"), + code="maiba", properties={ "username": "maiba", "full_name": "麦八", @@ -198,11 +208,11 @@ def test_get_users(self, user_wk): "phone": "13512345676", "phone_country_code": "86", }, - leaders=[gen_code("lisi"), gen_code("wangwu")], - departments=[gen_code("公司/部门A/中心AB")], + leaders=["lisi", "wangwu"], + departments=gen_depts(["公司/部门A/中心AB"]), ), RawDataSourceUser( - code=gen_code("yangjiu"), + code="yangjiu", properties={ "username": "yangjiu", "full_name": "杨九", @@ -213,11 +223,11 @@ def test_get_users(self, user_wk): "phone": "13512345677", "phone_country_code": "86", }, - leaders=[gen_code("wangwu")], - departments=[gen_code("公司/部门A/中心AB")], + leaders=["wangwu"], + departments=gen_depts(["公司/部门A/中心AB"]), ), RawDataSourceUser( - code=gen_code("lushi"), + code="lushi", properties={ "username": "lushi", "full_name": "鲁十", @@ -228,11 +238,11 @@ def test_get_users(self, user_wk): "phone": "13512345678", "phone_country_code": "86", }, - leaders=[gen_code("wangwu"), gen_code("maiba")], - departments=[gen_code("公司/部门B/中心BA"), gen_code("公司/部门A/中心AB/小组ABA")], + leaders=["wangwu", "maiba"], + departments=gen_depts(["公司/部门B/中心BA", "公司/部门A/中心AB/小组ABA"]), ), RawDataSourceUser( - code=gen_code("linshiyi"), + code="linshiyi", properties={ "username": "linshiyi", "full_name": "林十一", @@ -243,11 +253,11 @@ def test_get_users(self, user_wk): "phone": "13512345679", "phone_country_code": "86", }, - leaders=[gen_code("lushi")], - departments=[gen_code("公司/部门A/中心AB/小组ABA")], + leaders=["lushi"], + departments=gen_depts(["公司/部门A/中心AB/小组ABA"]), ), RawDataSourceUser( - code=gen_code("baishier"), + code="baishier", properties={ "username": "baishier", "full_name": "白十二", @@ -258,11 +268,11 @@ def test_get_users(self, user_wk): "phone": "13512345670", "phone_country_code": "86", }, - leaders=[gen_code("lushi")], - departments=[gen_code("公司/部门B/中心BA/小组BAA")], + leaders=["lushi"], + departments=gen_depts(["公司/部门B/中心BA/小组BAA"]), ), RawDataSourceUser( - code=gen_code("qinshisan"), + code="qinshisan", properties={ "username": "qinshisan", "full_name": "秦十三", @@ -273,11 +283,11 @@ def test_get_users(self, user_wk): "phone": "13512245671", "phone_country_code": "86", }, - leaders=[gen_code("lisi")], - departments=[gen_code("公司/部门C/中心CA/小组CAA")], + leaders=["lisi"], + departments=gen_depts(["公司/部门C/中心CA/小组CAA"]), ), RawDataSourceUser( - code=gen_code("freedom"), + code="freedom", properties={ "username": "freedom", "full_name": "自由人", diff --git a/src/bk-user/tests/plugins/local/test_plugin.py b/src/bk-user/tests/plugins/local/test_plugin.py new file mode 100644 index 000000000..455661add --- /dev/null +++ b/src/bk-user/tests/plugins/local/test_plugin.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import pytest +from bkuser.plugins.local.models import LocalDataSourcePluginConfig +from bkuser.plugins.local.plugin import LocalDataSourcePlugin + + +@pytest.fixture() +def local_ds_cfg(local_ds_plugin_cfg): + return LocalDataSourcePluginConfig(**local_ds_plugin_cfg) + + +class TestLocalDataSourcePlugin: + def test_get_departments(self, local_ds_cfg, user_wk): + plugin = LocalDataSourcePlugin(local_ds_cfg, user_wk) + assert len(plugin.fetch_departments()) == 12 # noqa: PLR2004 + + def test_get_users(self, local_ds_cfg, user_wk): + plugin = LocalDataSourcePlugin(local_ds_cfg, user_wk) + assert len(plugin.fetch_users()) == 12 # noqa: PLR2004 + + def test_test_connection(self, local_ds_cfg, user_wk): + with pytest.raises(NotImplementedError): + LocalDataSourcePlugin(local_ds_cfg, user_wk).test_connection() diff --git a/src/bk-user/tests/plugins/local/test_utils.py b/src/bk-user/tests/plugins/local/test_utils.py index 6361d549a..e45d42fff 100644 --- a/src/bk-user/tests/plugins/local/test_utils.py +++ b/src/bk-user/tests/plugins/local/test_utils.py @@ -16,9 +16,6 @@ ("raw", "excepted"), [ ("", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), - ("zhangsan", "82721d844c04f22502244a87061313e08d49c6e8d3fbee47e12201880a5ce6cb"), - ("lisi", "45ade5c9806fd3585a4ce199adbdf058301a01e625bf514252913e641191edd9"), - ("wangwu", "4ada86d5cca8cf4ac00399a0a16738ee8c944df3767b4af7151ca8b148cfe9e3"), ("公司", "e06ff957ed48e868a41b7e7e4460ce371e398108db542cf1cd1d61795b83e647"), ("公司/部门A", "2da9c820170b44354632bd3fe26ad09f4836b5977d2f6a5ff20afe7b143ac1e1"), ("公司/部门A/中心AA", "63986fb4ef27820413deb3f7c57cc36aef2ea898f03d8355e854d73b5c14e09c"), @@ -26,6 +23,6 @@ ], ) def test_gen_code(raw, excepted): - # 重要：如果该单元测试挂了，说明修改了本地数据源用户 & 部门的 Code 的生成规则 + # 重要：如果该单元测试挂了，说明修改了本地数据源部门的 Code 的生成规则 # 该行为会导致新同步的数据，无法与 DB 中的数据匹配上，将会触发数据重建！！！ assert gen_code(raw) == excepted diff --git a/src/bk-user/tests/test_utils/data_source.py b/src/bk-user/tests/test_utils/data_source.py index e4cda3613..75c6f9787 100644 --- a/src/bk-user/tests/test_utils/data_source.py +++ b/src/bk-user/tests/test_utils/data_source.py @@ -86,7 +86,7 @@ def init_data_source_users_depts_and_relations(ds: DataSource) -> None: # 数据源用户 zhangsan = DataSourceUser.objects.create( - code="Employee-3", + code="zhangsan", username="zhangsan", full_name="张三", email="zhangsan@m.com", @@ -94,7 +94,7 @@ def init_data_source_users_depts_and_relations(ds: DataSource) -> None: data_source=ds, ) lisi = DataSourceUser.objects.create( - code="Employee-4", + code="lisi", username="lisi", full_name="李四", email="lisi@m.com", @@ -102,7 +102,7 @@ def init_data_source_users_depts_and_relations(ds: DataSource) -> None: data_source=ds, ) wangwu = DataSourceUser.objects.create( - code="Employee-5", + code="wangwu", username="wangwu", full_name="王五", email="wangwu@m.com", @@ -110,7 +110,7 @@ def init_data_source_users_depts_and_relations(ds: DataSource) -> None: data_source=ds, ) zhaoliu = DataSourceUser.objects.create( - code="Employee-6", + code="zhaoliu", username="zhaoliu", full_name="赵六", email="zhaoliu@m.com", @@ -118,7 +118,7 @@ def init_data_source_users_depts_and_relations(ds: DataSource) -> None: data_source=ds, ) liuqi = DataSourceUser.objects.create( - code="Employee-7", + code="liuqi", username="liuqi", full_name="柳七", email="liuqi@m.com", @@ -126,7 +126,7 @@ def init_data_source_users_depts_and_relations(ds: DataSource) -> None: data_source=ds, ) maiba = DataSourceUser.objects.create( - code="Employee-8", + code="maiba", username="maiba", full_name="麦八", email="maiba@m.com", @@ -134,7 +134,7 @@ def init_data_source_users_depts_and_relations(ds: DataSource) -> None: data_source=ds, ) yangjiu = DataSourceUser.objects.create( - code="Employee-9", + code="yangjiu", username="yangjiu", full_name="杨九", email="yangjiu@m.com", @@ -142,7 +142,7 @@ def init_data_source_users_depts_and_relations(ds: DataSource) -> None: data_source=ds, ) lushi = DataSourceUser.objects.create( - code="Employee-10", + code="lushi", username="lushi", full_name="鲁十", email="lushi@m.com", @@ -150,7 +150,7 @@ def init_data_source_users_depts_and_relations(ds: DataSource) -> None: data_source=ds, ) linshiyi = DataSourceUser.objects.create( - code="Employee-11", + code="linshiyi", username="linshiyi", full_name="林十一", email="linshiyi@m.com", @@ -158,7 +158,7 @@ def init_data_source_users_depts_and_relations(ds: DataSource) -> None: data_source=ds, ) baishier = DataSourceUser.objects.create( - code="Employee-12", + code="baishier", username="baishier", full_name="白十二", email="baishier@m.com", @@ -167,7 +167,7 @@ def init_data_source_users_depts_and_relations(ds: DataSource) -> None: ) # 不属于任何组织，没有上下级的自由人 DataSourceUser.objects.create( - code="Employee-666", + code="freedom", username="freedom", full_name="自由人", email="freedom@m.com", diff --git a/src/bk-user/tests/utils/test_base64.py b/src/bk-user/tests/utils/test_base64.py new file mode 100644 index 000000000..afc072214 --- /dev/null +++ b/src/bk-user/tests/utils/test_base64.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import base64 +import os +import tempfile +from pathlib import Path + +import pytest +from bkuser.utils.base64 import load_image_as_base64 + + +def test_load_image_as_base64(): + # 生成临时图片文件 + with tempfile.NamedTemporaryFile(suffix=".png") as img: + img.write(os.urandom(40)) + img.flush() + + b64content = load_image_as_base64(Path(img.name)) + with open(img.name, "rb") as f: + expected = "data:image/png;base64," + base64.b64encode(f.read()).decode("utf-8") + + assert b64content == expected + + +def test_load_image_as_base64_with_not_png(): + # 生成临时图片文件 + with tempfile.NamedTemporaryFile(suffix=".jpg") as img: + img.write(os.urandom(40)) + img.flush() + + with pytest.raises(ValueError, match="only PNG image supported"): + load_image_as_base64(Path(img.name))