feature: Agent 2.0 配置模板、配置环境变量管理 (closed #1681)

apps/backend/agent/artifact_builder/agent.py

@@ -22,6 +22,8 @@
 
 class AgentArtifactBuilder(base.BaseArtifactBuilder):
 
+    ENV_FILES = constants.GsePackageEnv.AGENT.value
+    TEMPLATE_PATTERN = constants.GsePackageTemplatePattern.AGENT.value
     NAME = constants.GsePackageCode.AGENT.value
     PKG_DIR = constants.GsePackageDir.AGENT.value
     CERT_FILENAMES = [
@@ -39,5 +41,4 @@ def extract_initial_artifact(self, initial_artifact_local_path: str, extract_dir
         return extract_dir
 
     def _get_support_files_info(self, extract_dir: str) -> typing.Dict[str, typing.Any]:
-        # Agent 包管理实现
-        pass
+        return super()._get_support_files_info(extract_dir=extract_dir)

apps/backend/agent/artifact_builder/base.py

@@ -22,6 +22,8 @@
 from apps.backend import exceptions
 from apps.core.files import core_files_constants
 from apps.core.files.storage import get_storage
+from apps.core.tag.constants import TargetType
+from apps.core.tag.handlers import TagHandler
 from apps.node_man import constants, models
 from apps.utils import cache, files
 
@@ -44,6 +46,16 @@ class BaseArtifactBuilder(abc.ABC):
     # 制品存储的根路径
     BASE_STORAGE_DIR: str = "agent"
 
+    VERSION_FILE_NAME: str = "VERSION"
+
+    SUPPORT_FILES_DIR: str = "support-files"
+    ENV_FILES_DIR: str = os.path.join(SUPPORT_FILES_DIR, "env")
+    TEMPLATE_FILES_DIR: str = os.path.join(SUPPORT_FILES_DIR, "templates")
+    TEMPLATE_PATTERN: str = None
+
+    ENV_FILES: typing.List[str] = []
+    TEMPLATE_FILES: typing.List[str] = []
+
     def __init__(
         self,
         initial_artifact_path: str,
@@ -68,6 +80,10 @@ def __init__(
         self.applied_tmp_dirs = set()
         # 文件源
         self.storage = get_storage(file_overwrite=True)
+        self.agent_pkg_manage_enable = models.GlobalSettings.get_config(
+            key=models.GlobalSettings.KeyEnum.AGENT_PKG_MANAGE_ENABLE.value,
+            default=False,
+        )
 
     @staticmethod
     def download_file(file_path: str, target_path: str):
@@ -145,14 +161,38 @@ def extract_initial_artifact(self, initial_artifact_local_path: str, extract_dir
         """
         raise NotImplementedError
 
-    @abc.abstractmethod
     def _get_support_files_info(self, extract_dir: str) -> typing.Dict[str, typing.Any]:
         """
         获取部署依赖文件合集：配置 / 环境变量等
         :param extract_dir: 解压目录
         :return:
         """
-        raise NotImplementedError
+        env_file_infos: typing.List[typing.Dict] = []
+        template_file_infos: typing.List[typing.Dict] = []
+        # 获取env信息
+        for env_file in os.listdir(os.path.join(extract_dir, self.ENV_FILES_DIR)):
+            if env_file not in self.ENV_FILES:
+                continue
+            env_file_infos.append(
+                {
+                    "file_name": env_file,
+                    "file_absolute_path": os.path.join(extract_dir, self.ENV_FILES_DIR, env_file),
+                }
+            )
+
+        # 获取配置文件信息
+        for template in os.listdir(os.path.join(extract_dir, self.TEMPLATE_FILES_DIR)):
+            template_match: str = self.TEMPLATE_PATTERN.search(template)
+
+            if template_match:
+                template_file_infos.append(
+                    {
+                        "file_name": template_match.group(),
+                        "file_absolute_path": os.path.join(extract_dir, self.TEMPLATE_FILES_DIR, template),
+                    }
+                )
+
+        return {"env": env_file_infos, "templates": template_file_infos}
 
     def _inject_dependencies(self, extract_dir: str):
         """
@@ -240,7 +280,9 @@ def _list_package_dir_infos(self, extract_dir: str) -> typing.List[typing.Dict]:
         :return:
         """
         package_dir_infos: typing.List[typing.Dict] = []
+
         for pkg_dir_name in os.listdir(extract_dir):
+
             # 通过正则提取出插件（plugin）目录名中的插件信息
             re_match = constants.AGENT_PATH_RE.match(pkg_dir_name)
             if re_match is None:
@@ -346,8 +388,8 @@ def _get_version(self, extract_dir: str) -> str:
         :param extract_dir: 解压目录
         :return:
         """
-        # 优先使用覆盖版本号
-        if self.overwrite_version:
+        # TODO： 适配未开启Agent包管理，优先使用覆盖版本号
+        if not self.agent_pkg_manage_enable and self.overwrite_version:
             return self.overwrite_version
 
         version_file_path: str = os.path.join(extract_dir, "VERSION")
@@ -385,7 +427,58 @@ def update_or_create_record(self, artifact_meta_info: typing.Dict[str, typing.An
         """
         pass
 
-    def update_or_create_package_records(self, package_infos: typing.List[typing.Dict]):
+    def update_or_create_tag(self, artifact_meta_info: typing.Dict[str, typing.Any]):
+        """
+        创建或更新标签记录，待 Agent 包管理完善
+        :param artifact_meta_info:
+        :return:
+        """
+        # TODO  写死target_id 后续完善
+        if self.overwrite_version:
+            TagHandler.publish_tag_version(
+                name=self.overwrite_version,
+                target_type=TargetType.AGENT.value,
+                target_id=1,
+                target_version=artifact_meta_info["version"],
+            )
+
+    def update_or_create_support_files(self, package_infos: typing.List[typing.Dict]):
+        """
+        创建或更新support_files记录
+        :param package_infos:
+        :return:
+        """
+        for package_info in package_infos:
+            support_files = package_info["artifact_meta_info"]["support_files_info"]
+            version = package_info["artifact_meta_info"]["version"]
+            package = package_info["package_dir_info"]
+
+            for env_file in support_files["env"]:
+                with open(env_file["file_absolute_path"], "r") as f:
+                    env_value = f.read()
+
+                models.GseConfigEnv.objects.update_or_create(
+                    defaults={"env_value": env_value},
+                    version=version,
+                    os=package["os"],
+                    cpu_arch=package["cpu_arch"],
+                    agent_name=self.NAME,
+                )
+
+            for template in support_files["templates"]:
+                with open(template["file_absolute_path"], "r") as f:
+                    content = f.read()
+
+                models.GseConfigTemplate.objects.update_or_create(
+                    defaults={"content": content},
+                    name=template["file_name"],
+                    version=version,
+                    os=package["os"],
+                    cpu_arch=package["cpu_arch"],
+                    agent_name=self.NAME,
+                )
+
+    def update_or_create_package_records(self, v):
         """
         创建或更新安装包记录，待 Agent 包管理完善
         :param package_infos:
@@ -457,8 +550,11 @@ def make(
             )
 
         artifact_meta_info["operator"] = operator
-        self.update_or_create_record(artifact_meta_info)
-        self.update_or_create_package_records(package_infos)
+        if self.agent_pkg_manage_enable:
+            self.update_or_create_support_files(package_infos)
+            self.update_or_create_tag(artifact_meta_info)
+            self.update_or_create_record(artifact_meta_info)
+            self.update_or_create_package_records(package_infos)
 
     def __enter__(self) -> "BaseArtifactBuilder":
         return self

apps/backend/agent/artifact_builder/proxy.py

@@ -29,7 +29,8 @@ class ProxyArtifactBuilder(base.BaseArtifactBuilder):
     NAME = constants.GsePackageCode.PROXY.value
     PKG_DIR = constants.GsePackageDir.PROXY.value
     CERT_FILENAMES: typing.List[str] = constants.GseCert.list_member_values()
-
+    ENV_FILES = constants.GsePackageEnv.PROXY.value
+    TEMPLATE_PATTERN = constants.GsePackageTemplatePattern.PROXY.value
     # 服务二进制目录
     SERVER_BIN_DIR: str = "server/bin"
     # 所需的二进制文件
@@ -95,5 +96,4 @@ def extract_initial_artifact(self, initial_artifact_local_path: str, extract_dir
         return extract_dir
 
     def _get_support_files_info(self, extract_dir: str) -> typing.Dict[str, typing.Any]:
-        # Agent 包管理实现
-        pass
+        return super()._get_support_files_info(extract_dir=extract_dir)

apps/backend/agent/solution_maker.py

@@ -27,6 +27,8 @@
 from apps.backend.api import constants as backend_api_constants
 from apps.backend.subscription.steps.agent_adapter.base import AgentSetupInfo
 from apps.core.script_manage.base import ScriptHook
+from apps.core.tag.constants import TargetType
+from apps.core.tag.targets import BaseTargetHelper, get_target_helper
 from apps.node_man import constants, models
 from apps.utils import basic
 
@@ -280,7 +282,10 @@ def get_run_cmd_base_params(self) -> typing.List[str]:
 
         # 新版本 Agent 需要补充构件信息
         if not self.agent_setup_info.is_legacy:
-            run_cmd_params.extend([f"-n {self.agent_setup_info.name}", f"-t {self.agent_setup_info.version}"])
+            # 通过标签和类型获取安装版本
+            tag_handler: BaseTargetHelper = get_target_helper(TargetType.AGENT.value)
+            target_version: str = tag_handler.get_tagret_version_by_name(name=self.agent_setup_info.version)
+            run_cmd_params.extend([f"-n {self.agent_setup_info.name}", f"-t {target_version}"])
 
         # 因 bat 脚本逻辑，-R 参数只能放在最后一位
         if self.is_uninstall:

apps/backend/exceptions.py

@@ -88,3 +88,9 @@ class NotSemanticVersionError(BackendBaseException):
 class PkgMetaInfoValidationError(BackendBaseException):
     MESSAGE = _("安装包元数据校验失败")
     ERROR_CODE = 14
+
+
+class AgentConfigTemplateEnvNotExistError(BackendBaseException):
+    MESSAGE = _("配置模板Env不存在")
+    MESSAGE_TPL = _("配置模板Env不存在[{name}-{version}-{os_type}-{cpu_arch}]不存在")
+    ERROR_CODE = 15

apps/backend/subscription/steps/agent_adapter/adapter.py

@@ -18,12 +18,15 @@
 
 from apps.backend import exceptions
 from apps.backend.agent.tools import fetch_proxies
+from apps.core.tag.constants import TargetType
+from apps.core.tag.targets import get_target_helper
 from apps.node_man import constants, models
 from apps.utils import cache
 from env.constants import GseVersion
 
-from . import base, config_templates, legacy
+from . import base, legacy
 from .config_context import context_helper
+from .handlers import GseConfigHandler, get_gse_config_handler_class
 
 logger = logging.getLogger("app")
 
@@ -67,26 +70,6 @@ def config(self) -> OrderedDict:
         """
         return self.validated_data(data=self.subscription_step.config, serializer=AgentStepConfigSerializer)
 
-    @property
-    @cache.class_member_cache()
-    def config_tmpl_obj_gby_os_key(self) -> typing.Dict[str, typing.List[base.AgentConfigTemplate]]:
-        """
-        获取按机型（os_type + cpu_arch）聚合的配置模板
-        :return:
-        """
-        agent_config_templates: typing.List[base.AgentConfigTemplate] = [
-            base.AgentConfigTemplate(name="gse_agent.conf", content=config_templates.GSE_AGENT_CONFIG_TMPL),
-            base.AgentConfigTemplate(name="gse_data_proxy.conf", content=config_templates.GSE_DATA_PROXY_CONFIG_TMPL),
-            base.AgentConfigTemplate(name="gse_file_proxy.conf", content=config_templates.GSE_FILE_PROXY_CONFIG_TEMPL),
-        ]
-        return {
-            # 向 Agent 包管理过渡：AgentConfigTemplate 后续替换为数据模型对象
-            self.get_os_key(constants.OsType.LINUX, constants.CpuType.x86): agent_config_templates,
-            self.get_os_key(constants.OsType.LINUX, constants.CpuType.x86_64): agent_config_templates,
-            self.get_os_key(constants.OsType.WINDOWS, constants.CpuType.x86): agent_config_templates,
-            self.get_os_key(constants.OsType.WINDOWS, constants.CpuType.x86_64): agent_config_templates,
-        }
-
     def get_main_config_filename(self) -> str:
         return ("gse_agent.conf", "agent.conf")[self.is_legacy]
 
@@ -99,24 +82,20 @@ def _get_config(
         proxies: typing.List[models.Host],
         install_channel: typing.Tuple[typing.Optional[models.Host], typing.Dict[str, typing.List]],
     ) -> str:
-        config_tmpl_objs: typing.List[base.AgentConfigTemplate] = self.config_tmpl_obj_gby_os_key.get(
-            self.get_os_key(host.os_type, host.cpu_arch),
-            [
-                base.AgentConfigTemplate(name="gse_agent.conf", content=config_templates.GSE_AGENT_CONFIG_TMPL),
-                base.AgentConfigTemplate(
-                    name="gse_data_proxy.conf", content=config_templates.GSE_DATA_PROXY_CONFIG_TMPL
-                ),
-                base.AgentConfigTemplate(
-                    name="gse_file_proxy.conf", content=config_templates.GSE_FILE_PROXY_CONFIG_TEMPL
-                ),
-            ],
+        # TODO: 第一版通过tag 获取到目标版本，后续版本需要实现动态
+        target_version: str = get_target_helper(TargetType.AGENT.value).get_tagret_version_by_name("stable")
+        gse_config_handler_class: GseConfigHandler = get_gse_config_handler_class(host.node_type)
+        gse_config_handler: GseConfigHandler = gse_config_handler_class(
+            os_type=host.os_type,
+            cpu_arch=host.cpu_arch,
+            target_version=target_version,
         )
-        # 查找机型匹配的第一个配置
-        target_config_tmpl_obj: base.AgentConfigTemplate = next(
-            (config_tmpl_obj for config_tmpl_obj in config_tmpl_objs if config_tmpl_obj.name == filename), None
+
+        config_tmpl_obj: base.AgentConfigTemplate = gse_config_handler.get_template_by_config_name(
+            config_name=filename,
         )
 
-        if not target_config_tmpl_obj:
+        if not config_tmpl_obj:
             logger.error(
                 f"{self.log_prefix} agent config template not exist: name -> {self.config['name']}, "
                 f"filename -> {filename}, version -> {self.config['version']}, "
@@ -135,7 +114,7 @@ def _get_config(
             proxies=proxies,
             install_channel=install_channel,
         )
-        return ch.render(target_config_tmpl_obj.content)
+        return ch.render(config_tmpl_obj.content, gse_config_handler.template_env)
 
     def get_config(
         self,

apps/backend/subscription/steps/agent_adapter/config_context/context_helper.py

@@ -14,6 +14,7 @@
 import re
 import typing
 from dataclasses import asdict, dataclass, field
+from typing import Any, Dict
 
 from django.conf import settings
 
@@ -221,7 +222,7 @@ def __post_init__(self):
         for context in contexts:
             self.context_dict.update(asdict(context, dict_factory=context.dict_factory))
 
-    def render(self, content: str) -> str:
+    def render(self, content: str, template_env: Dict[str, Any] = {}) -> str:
         """
         渲染并返回配置
         :param content: 配置模板内容
@@ -232,5 +233,8 @@ def _double(_matched) -> str:
             _env_name: str = str(_matched.group())
             return "{{ " + _env_name[2:-2] + " }}"
 
+        # 先加载配置模板默认变量配置
+        context_dict = template_env
+        context_dict.update(self.context_dict)
         content = re.sub(r"(__[0-9A-Z_]+__)", _double, content)
-        return nested_render_data(content, self.context_dict)
+        return nested_render_data(content, context_dict)

apps/backend/subscription/steps/agent_adapter/config_parser.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from configparser import ConfigParser, NoSectionError
+
+_UNSET = object()
+
+
+class GseConfigParser(ConfigParser):
+    def optionxform(self, optionstr: str) -> str:
+        # 返回原始大小写
+        return optionstr
+
+    def format_value(self, value):
+        try:
+            value = int(value)
+        except ValueError:
+            try:
+                value = float(value)
+            except ValueError:
+                pass
+        return value
+
+    def items(self, section=_UNSET, raw=False, vars=None):
+        d = self._defaults.copy()
+        try:
+            d.update(self._sections[section])
+        except KeyError:
+            if section != self.default_section:
+                raise NoSectionError(section)
+        # Update with the entry specific variables
+        if vars:
+            for key, value in vars.items():
+                d[self.optionxform(key)] = value
+        value_getter = lambda option: self._interpolation.before_get(self, section, option, d[option], d)  # noqa
+        if raw:
+            value_getter = lambda option: d[option]  # noqa
+        return [(option, self.format_value(value_getter(option))) for option in d.keys()]