feat: mask data source plugin config sensitive fields (#1366)

src/bk-user/bkuser/apis/web/data_source/serializers.py

@@ -25,9 +25,12 @@
 from bkuser.apps.sync.models import DataSourceSyncTask
 from bkuser.apps.tenant.models import TenantUserCustomField, UserBuiltinField
 from bkuser.biz.data_source_plugin import DefaultPluginConfigProvider
+from bkuser.common.constants import SENSITIVE_MASK
 from bkuser.plugins.base import get_plugin_cfg_cls, is_plugin_exists
 from bkuser.plugins.constants import DataSourcePluginEnum
 from bkuser.plugins.local.models import PasswordRuleConfig
+from bkuser.plugins.models import BasePluginConfig
+from bkuser.utils import dictx
 from bkuser.utils.pydantic import stringify_pydantic_error
 
 logger = logging.getLogger(__name__)
@@ -147,7 +150,7 @@ def validate(self, attrs: Dict[str, Any]) -> Dict[str, Any]:
 
         PluginConfigCls = get_plugin_cfg_cls(plugin_id)  # noqa: N806
         try:
-            PluginConfigCls(**attrs["plugin_config"])
+            attrs["plugin_config"] = PluginConfigCls(**attrs["plugin_config"])
         except PDValidationError as e:
             raise ValidationError(_("插件配置不合法：{}").format(stringify_pydantic_error(e)))
 
@@ -198,15 +201,19 @@ def validate_name(self, name: str) -> str:
 
         return name
 
-    def validate_plugin_config(self, plugin_config: Dict[str, Any]) -> Dict[str, Any]:
+    def validate_plugin_config(self, plugin_config: Dict[str, Any]) -> BasePluginConfig:
         PluginConfigCls = get_plugin_cfg_cls(self.context["plugin_id"])  # noqa: N806
+
+        # 将敏感信息填充回 plugin_config，一并进行校验
+        for info in self.context["exists_sensitive_infos"]:
+            if dictx.get_items(plugin_config, info.key) == SENSITIVE_MASK:
+                dictx.set_items(plugin_config, info.key, info.value)
+
         try:
-            PluginConfigCls(**plugin_config)
+            return PluginConfigCls(**plugin_config)
         except PDValidationError as e:
             raise ValidationError(_("插件配置不合法：{}").format(stringify_pydantic_error(e)))
 
-        return plugin_config
-
     def validate_field_mapping(self, field_mapping: List[Dict[str, str]]) -> List[Dict[str, str]]:
         # 遇到空的字段映射，直接返回即可，validate() 中会根据插件类型校验是否必须提供字段映射
         if not field_mapping:

src/bk-user/bkuser/apis/web/data_source/views.py

@@ -41,7 +41,7 @@
 )
 from bkuser.apis.web.mixins import CurrentUserTenantMixin
 from bkuser.apps.data_source.constants import DataSourceStatus
-from bkuser.apps.data_source.models import DataSource, DataSourcePlugin
+from bkuser.apps.data_source.models import DataSource, DataSourcePlugin, DataSourceSensitiveInfo
 from bkuser.apps.sync.constants import SyncTaskTrigger
 from bkuser.apps.sync.data_models import DataSourceSyncOptions
 from bkuser.apps.sync.managers import DataSourceSyncManager
@@ -184,18 +184,20 @@ def put(self, request, *args, **kwargs):
                 "plugin_id": data_source.plugin_id,
                 "tenant_id": self.get_current_tenant_id(),
                 "current_name": data_source.name,
+                "exists_sensitive_infos": DataSourceSensitiveInfo.objects.filter(data_source=data_source),
             },
         )
         slz.is_valid(raise_exception=True)
         data = slz.validated_data
 
         with transaction.atomic():
             data_source.name = data["name"]
-            data_source.plugin_config = data["plugin_config"]
             data_source.field_mapping = data["field_mapping"]
             data_source.sync_config = data.get("sync_config") or {}
             data_source.updater = request.user.username
-            data_source.save()
+            data_source.save(update_fields=["name", "field_mapping", "sync_config", "updater", "updated_at"])
+            # 由于需要替换敏感信息，因此需要独立调用 set_plugin_cfg 方法
+            data_source.set_plugin_cfg(data["plugin_config"])
 
         return Response(status=status.HTTP_204_NO_CONTENT)
 

src/bk-user/bkuser/apps/data_source/initializers.py

@@ -48,7 +48,9 @@ def __init__(self, data_source: DataSource):
         if not data_source.is_local:
             return
 
-        self.plugin_cfg = LocalDataSourcePluginConfig(**data_source.plugin_config)
+        self.plugin_cfg = data_source.get_plugin_cfg()
+        assert isinstance(self.plugin_cfg, LocalDataSourcePluginConfig)
+
         if not self.plugin_cfg.enable_account_password_login:
             return
 
@@ -90,7 +92,7 @@ def _can_skip(self) -> bool:
             return True
 
         # 是本地数据源，但是没开启账密登录的，不需要初始化
-        if not self.plugin_cfg.enable_account_password_login:
+        if not self.plugin_cfg.enable_account_password_login:  # type: ignore
             return True
 
         return False

src/bk-user/bkuser/apps/data_source/migrations/0009_datasourcesensitiveinfo.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.20 on 2023-11-04 02:08
+
+import blue_krill.models.fields
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_source', '0008_auto_20231024_0940'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DataSourceSensitiveInfo',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('key', models.CharField(max_length=255, verbose_name='配置字段路径')),
+                ('value', blue_krill.models.fields.EncryptField(max_length=255, verbose_name='敏感配置数据')),
+                ('data_source', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.PROTECT, to='data_source.datasource')),
+            ],
+            options={
+                'unique_together': {('data_source', 'key')},
+            },
+        ),
+    ]

src/bk-user/bkuser/apps/data_source/models.py

@@ -8,15 +8,18 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-
 from blue_krill.models.fields import EncryptField
 from django.conf import settings
-from django.db import models
+from django.db import models, transaction
 from mptt.models import MPTTModel, TreeForeignKey
 
 from bkuser.apps.data_source.constants import DataSourceStatus
+from bkuser.common.constants import SENSITIVE_MASK
 from bkuser.common.models import AuditedModel, TimestampedModel
+from bkuser.plugins.base import get_plugin_cfg_cls
 from bkuser.plugins.constants import DataSourcePluginEnum
+from bkuser.plugins.models import BasePluginConfig
+from bkuser.utils import dictx
 from bkuser.utils.uuid import generate_uuid
 
 
@@ -32,6 +35,22 @@ class DataSourcePlugin(models.Model):
     logo = models.TextField("Logo", null=True, blank=True, default="")
 
 
+class DataSourceManager(models.Manager):
+    """数据源管理器类"""
+
+    @transaction.atomic()
+    def create(self, *args, **kwargs):
+        if "plugin_config" not in kwargs:
+            return super().create(*args, **kwargs)
+
+        plugin_cfg = kwargs.pop("plugin_config")
+        assert isinstance(plugin_cfg, BasePluginConfig)
+
+        data_source: DataSource = super().create(*args, **kwargs)
+        data_source.set_plugin_cfg(plugin_cfg)
+        return data_source
+
+
 class DataSource(AuditedModel):
     name = models.CharField("数据源名称", max_length=128, unique=True)
     owner_tenant_id = models.CharField("归属租户", max_length=64, db_index=True)
@@ -49,6 +68,8 @@ class DataSource(AuditedModel):
     # 字段映射，外部数据源提供商，用户数据字段映射到租户用户数据字段
     field_mapping = models.JSONField("用户字段映射", default=list)
 
+    objects = DataSourceManager()
+
     class Meta:
         ordering = ["id"]
 
@@ -57,6 +78,38 @@ def is_local(self) -> bool:
         """检查类型是否为本地数据源"""
         return self.plugin.id == DataSourcePluginEnum.LOCAL
 
+    def get_plugin_cfg(self) -> BasePluginConfig:
+        """获取插件配置
+
+        注意：使用该方法获取到的配置将会包含敏感信息，不适合通过 API 暴露出去，仅可用于内部逻辑流转
+        API 要获取插件配置请使用 data_source.plugin_config，其中的敏感信息将会被 ******* 取代
+        """
+        plugin_cfg = self.plugin_config
+        for info in DataSourceSensitiveInfo.objects.filter(data_source=self):
+            dictx.set_items(plugin_cfg, info.key, info.value)
+
+        PluginCfgCls = get_plugin_cfg_cls(self.plugin.id)  # noqa: N806
+        return PluginCfgCls(**plugin_cfg)
+
+    def set_plugin_cfg(self, cfg: BasePluginConfig) -> None:
+        """设置插件配置，注意：该方法包含 DB 数据更新，需要在事务中执行"""
+        plugin_cfg = cfg.model_dump()
+
+        # 由于单个插件的敏感字段不会很多，这里不采用批量创建/更新的方式
+        for field in cfg.sensitive_fields:
+            sensitive_val = dictx.get_items(plugin_cfg, field)
+            # 若敏感字段无值，或者已经被替换为掩码，则不需要二次替换
+            if not sensitive_val or sensitive_val == SENSITIVE_MASK:
+                continue
+
+            DataSourceSensitiveInfo.objects.update_or_create(
+                data_source=self, key=field, defaults={"value": sensitive_val}
+            )
+            dictx.set_items(plugin_cfg, field, SENSITIVE_MASK)
+
+        self.plugin_config = plugin_cfg
+        self.save(update_fields=["plugin_config", "updated_at"])
+
 
 class DataSourceUser(TimestampedModel):
     data_source = models.ForeignKey(DataSource, on_delete=models.PROTECT, db_constraint=False)
@@ -181,3 +234,14 @@ class DepartmentRelationMPTTTree(models.Model):
     """部门关系树记录表，用于自增 tree_id 的分配"""
 
     data_source = models.ForeignKey(DataSource, on_delete=models.PROTECT, db_constraint=False)
+
+
+class DataSourceSensitiveInfo(TimestampedModel):
+    """数据源敏感配置信息"""
+
+    data_source = models.ForeignKey(DataSource, on_delete=models.PROTECT, db_constraint=False)
+    key = models.CharField("配置字段路径", max_length=255)
+    value = EncryptField(verbose_name="敏感配置数据", max_length=255)
+
+    class Meta:
+        unique_together = [("data_source", "key")]

src/bk-user/bkuser/apps/data_source/notifier.py

@@ -88,7 +88,9 @@ def __init__(self, data_source: DataSource, scene: NotificationScene):
         if not data_source.is_local:
             return
 
-        plugin_cfg = LocalDataSourcePluginConfig(**data_source.plugin_config)
+        plugin_cfg = data_source.get_plugin_cfg()
+        assert isinstance(plugin_cfg, LocalDataSourcePluginConfig)
+
         if not plugin_cfg.enable_account_password_login:
             return
 

src/bk-user/bkuser/apps/sync/periodic_tasks.py

@@ -36,6 +36,8 @@ def build_and_run_data_source_sync_task(data_source_id: int):
         return
 
     sync_opts = DataSourceSyncOptions(
+        # 定时执行的任务，执行者为最后修改数据源配置的人
+        operator=data_source.updater,
         overwrite=True,
         incremental=False,
         # 注：现在就在异步任务中，不需要 async_run=True

src/bk-user/bkuser/apps/sync/runners.py

@@ -24,7 +24,7 @@
     TenantUserSyncer,
 )
 from bkuser.apps.tenant.models import Tenant
-from bkuser.plugins.base import get_plugin_cfg_cls, get_plugin_cls
+from bkuser.plugins.base import get_plugin_cls
 
 logger = logging.getLogger(__name__)
 
@@ -50,11 +50,10 @@ def run(self):
 
     def _initial_plugin(self, plugin_init_extra_kwargs: Dict[str, Any]):
         """初始化数据源插件"""
-        PluginCfgCls = get_plugin_cfg_cls(self.data_source.plugin_id)  # noqa: N806
-        plugin_config = PluginCfgCls(**self.data_source.plugin_config)
+        plugin_cfg = self.data_source.get_plugin_cfg()
 
         PluginCls = get_plugin_cls(self.data_source.plugin_id)  # noqa: N806
-        self.plugin = PluginCls(plugin_config, **plugin_init_extra_kwargs)
+        self.plugin = PluginCls(plugin_cfg, **plugin_init_extra_kwargs)
 
     def _sync_departments(self, ctx: DataSourceSyncTaskContext):
         """同步部门信息"""

src/bk-user/bkuser/biz/data_source.py

@@ -70,7 +70,7 @@ def create_local_data_source_with_merge_config(
             name=data_source_name,
             owner_tenant_id=owner_tenant_id,
             plugin=DataSourcePlugin.objects.get(id=plugin_id),
-            plugin_config=plugin_config.model_dump(),
+            plugin_config=plugin_config,
         )
 
 

src/bk-user/bkuser/common/constants.py

@@ -20,3 +20,6 @@ class BkLanguageEnum(str, StructuredEnum):
 
 # 永久：2100-01-01 00:00:00
 PERMANENT_TIME = datetime.datetime(year=2100, month=1, day=1, hour=0, minute=0, second=0)
+
+# 敏感信息掩码（7 位 * 是故意的，避免遇到用户输入 6/8 位 * 的情况）
+SENSITIVE_MASK = "*******"

src/bk-user/bkuser/plugins/base.py

@@ -13,10 +13,9 @@
 from typing import Dict, List, Type
 
 from drf_yasg import openapi
-from pydantic import BaseModel
 
 from bkuser.plugins.constants import CUSTOM_PLUGIN_ID_PREFIX, DataSourcePluginEnum
-from bkuser.plugins.models import RawDataSourceDepartment, RawDataSourceUser, TestConnectionResult
+from bkuser.plugins.models import BasePluginConfig, RawDataSourceDepartment, RawDataSourceUser, TestConnectionResult
 from bkuser.utils.pydantic import gen_openapi_schema
 
 logger = logging.getLogger(__name__)
@@ -26,7 +25,7 @@ class BaseDataSourcePlugin(ABC):
     """数据源插件基类"""
 
     id: str | DataSourcePluginEnum
-    config_class: Type[BaseModel]
+    config_class: Type[BasePluginConfig]
 
     @abstractmethod
     def __init__(self, *args, **kwargs):
@@ -82,7 +81,7 @@ def get_plugin_cls(plugin_id: str | DataSourcePluginEnum) -> Type[BaseDataSource
     return _plugin_cls_map[plugin_id]
 
 
-def get_plugin_cfg_cls(plugin_id: str | DataSourcePluginEnum) -> Type[BaseModel]:
+def get_plugin_cfg_cls(plugin_id: str | DataSourcePluginEnum) -> Type[BasePluginConfig]:
     """获取指定插件的配置类"""
     return get_plugin_cls(plugin_id).config_class
 

src/bk-user/bkuser/plugins/general/models.py

@@ -22,6 +22,7 @@
     AuthMethod,
     PageSize,
 )
+from bkuser.plugins.models import BasePluginConfig
 
 
 class QueryParam(BaseModel):
@@ -63,9 +64,14 @@ class AuthConfig(BaseModel):
     password: str | None = None
 
 
-class GeneralDataSourcePluginConfig(BaseModel):
+class GeneralDataSourcePluginConfig(BasePluginConfig):
     """通用 HTTP 数据源插件配置"""
 
+    sensitive_fields = [
+        "auth_config.bearer_token",
+        "auth_config.password",
+    ]
+
     # 服务配置
     server_config: ServerConfig
     # 认证配置

src/bk-user/bkuser/plugins/local/models.py

@@ -26,6 +26,7 @@
     NotificationScene,
     PasswordGenerateMethod,
 )
+from bkuser.plugins.models import BasePluginConfig
 from bkuser.utils.pydantic import stringify_pydantic_error
 
 
@@ -143,9 +144,14 @@ class PasswordExpireConfig(BaseModel):
     notification: NotificationConfig
 
 
-class LocalDataSourcePluginConfig(BaseModel):
+class LocalDataSourcePluginConfig(BasePluginConfig):
     """本地数据源插件配置"""
 
+    # 敏感字段
+    sensitive_fields = [
+        "password_initial.fixed_password",
+    ]
+
     # 是否允许使用账密登录
     enable_account_password_login: bool
     # 密码生成规则

src/bk-user/bkuser/plugins/models.py

@@ -8,7 +8,7 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-from typing import Any, Dict, List
+from typing import Any, ClassVar, Dict, List
 
 from pydantic import BaseModel
 
@@ -21,6 +21,16 @@ class PluginMetadata(BaseModel):
     description: str
 
 
+class BasePluginConfig(BaseModel):
+    """插件配置基类"""
+
+    # 注：敏感字段声明有以下规范
+    # 字段形式如: auth_config.password，
+    # 字段类型为 str 或 (str | None)
+    # 字段路径中不支持列表下标，只能是字典 key
+    sensitive_fields: ClassVar[List[str]] = []
+
+
 class RawDataSourceUser(BaseModel):
     """原始数据源用户信息"""