From e1b31f6311ddfa613046b89f6983e5e028b5b7f2 Mon Sep 17 00:00:00 2001 From: iSecloud <869820505@qq.com> Date: Thu, 7 Mar 2024 20:03:09 +0800 Subject: [PATCH] =?UTF-8?q?fix(backend):=20sql=E5=AF=BC=E5=85=A5=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E6=9C=80=E5=A4=A7=E6=96=87=E4=BB=B6=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E5=92=8C=E6=9C=80=E5=A4=A7=E6=96=87=E4=BB=B6=E9=A2=84=E8=A7=88?= =?UTF-8?q?=E9=99=90=E5=88=B6=20#3486?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/components/mysql_backup/client.py | 2 +- .../db_services/dbbase/instances/handlers.py | 5 +---- .../db_services/mysql/open_area/handlers.py | 1 + .../db_services/mysql/sql_import/constants.py | 5 +++++ .../db_services/mysql/sql_import/handlers.py | 17 ++++++++++++----- .../db_services/mysql/sql_import/serializers.py | 3 +++ helm-charts/bk-dbm/Chart.lock | 14 +++++++------- helm-charts/bk-dbm/Chart.yaml | 8 ++++---- helm-charts/bk-dbm/charts/dbconfig/Chart.yaml | 4 ++-- helm-charts/bk-dbm/charts/dbm/Chart.yaml | 4 ++-- 10 files changed, 38 insertions(+), 25 deletions(-) diff --git a/dbm-ui/backend/components/mysql_backup/client.py b/dbm-ui/backend/components/mysql_backup/client.py index b35644eb97..c2eacefb14 100644 --- a/dbm-ui/backend/components/mysql_backup/client.py +++ b/dbm-ui/backend/components/mysql_backup/client.py @@ -39,7 +39,7 @@ def __init__(self): method="POST", url="backupapi/client/install", description=_("backup_client下载，同步任务"), - default_timeout=600, + default_timeout=300, max_retry_times=1, ) diff --git a/dbm-ui/backend/db_services/dbbase/instances/handlers.py b/dbm-ui/backend/db_services/dbbase/instances/handlers.py index d4bd790059..248b5c901a 100644 --- a/dbm-ui/backend/db_services/dbbase/instances/handlers.py +++ b/dbm-ui/backend/db_services/dbbase/instances/handlers.py @@ -13,7 +13,6 @@ from django.db.models import F, Q -from backend import env from backend.constants import IP_PORT_DIVIDER from backend.db_meta.models import Machine, ProxyInstance, StorageInstance from backend.db_services.ipchooser.handlers.host_handler import HostHandler @@ -119,9 +118,7 @@ def check_instances(self, query_instances: List[Union[str, Dict]], cluster_ids: db_instances.append(db_inst) # 查询补充主机信息 - host_infos = HostHandler.check( - [{"bk_biz_id": env.DBA_APP_BK_BIZ_ID, "scope_type": "biz"}], [], [], bk_host_ids - ) + host_infos = HostHandler.check([{"bk_biz_id": self.bk_biz_id, "scope_type": "biz"}], [], [], bk_host_ids) host_id_info_map = {host_info["host_id"]: host_info for host_info in host_infos} return [ {**host_id_instance_map[str(db_inst)], **{"host_info": host_id_info_map.get(db_inst.bk_host_id, {})}} diff --git a/dbm-ui/backend/db_services/mysql/open_area/handlers.py b/dbm-ui/backend/db_services/mysql/open_area/handlers.py index 1c8897fbf2..af20f54333 100644 --- a/dbm-ui/backend/db_services/mysql/open_area/handlers.py +++ b/dbm-ui/backend/db_services/mysql/open_area/handlers.py @@ -144,6 +144,7 @@ def __get_openarea_rules_set(cls, config, config_data, operator, cluster_id__clu "cluster_type": cluster_id__cluster[data["cluster_id"]].cluster_type, } for data in config_data + if data.get("authorize_ips") for user in user__dbs_rules.keys() ] return authorize_details diff --git a/dbm-ui/backend/db_services/mysql/sql_import/constants.py b/dbm-ui/backend/db_services/mysql/sql_import/constants.py index 336c1e515b..f10f87fbdf 100644 --- a/dbm-ui/backend/db_services/mysql/sql_import/constants.py +++ b/dbm-ui/backend/db_services/mysql/sql_import/constants.py @@ -19,6 +19,11 @@ CACHE_SEMANTIC_SKIP_PAUSE_FILED = "{bk_biz_id}_{root_id}_semantic_check_skip_pause" SQL_SEMANTIC_CHECK_DATA_EXPIRE_TIME = 7 * 24 * 60 * 60 +# 最大预览SQL文件大小200MB +MAX_PREVIEW_SQL_FILE_SIZE = 200 * 1024 * 1024 +# 最大上传SQL文件大小1G +MAX_UPLOAD_SQL_FILE_SIZE = 1024 * 1024 * 1024 + class SQLCharset(str, StructuredEnum): """sql语句的字符集类型""" diff --git a/dbm-ui/backend/db_services/mysql/sql_import/handlers.py b/dbm-ui/backend/db_services/mysql/sql_import/handlers.py index 1b3219c34c..c89a7defa1 100644 --- a/dbm-ui/backend/db_services/mysql/sql_import/handlers.py +++ b/dbm-ui/backend/db_services/mysql/sql_import/handlers.py @@ -25,6 +25,7 @@ CACHE_SEMANTIC_AUTO_COMMIT_FIELD, CACHE_SEMANTIC_SKIP_PAUSE_FILED, CACHE_SEMANTIC_TASK_FIELD, + MAX_PREVIEW_SQL_FILE_SIZE, SQL_SEMANTIC_CHECK_DATA_EXPIRE_TIME, SQLImportMode, ) @@ -70,7 +71,9 @@ def _upload_sql_file( # 如果上传的是sql内容, 则创建一个sql文件 if sql_content: sql_file = tempfile.NamedTemporaryFile(suffix=".sql") - sql_file.write(str.encode(sql_content, encoding="utf-8")) + content_byte = str.encode(sql_content, encoding="utf-8") + sql_file.write(content_byte) + sql_file.size = len(content_byte) sql_file.seek(0) sql_file_list = [sql_file] @@ -83,9 +86,13 @@ def _upload_sql_file( # 恢复文件指针为文件头，否则会无法读取内容 file.seek(0) - sql_file_info.update( - sql_path=sql_path, sql_content=file.read().decode("utf-8"), raw_file_name=file.name - ) + # 超过最大预览限制，则不支持预览 + if file.size > MAX_PREVIEW_SQL_FILE_SIZE: + sql_content = _("当前SQL文件过大，暂不提供内容预览...") + else: + sql_content = file.read().decode("utf-8") + + sql_file_info.update(sql_path=sql_path, sql_content=sql_content, raw_file_name=file.name) sql_file_info_list.append(sql_file_info) @@ -113,7 +120,7 @@ def grammar_check( params={"path": dir_name, "files": file_name_list, "cluster_type": self.cluster_type} ) - # 填充sql内容。TODO：如果sql内容过大需要进行压缩吗？ + # 填充sql内容。 for sql_file_info in sql_file_info_list: sql_path = sql_file_info["sql_path"] file_name = os.path.split(sql_path)[1] diff --git a/dbm-ui/backend/db_services/mysql/sql_import/serializers.py b/dbm-ui/backend/db_services/mysql/sql_import/serializers.py index c665631f1b..1ef656b892 100644 --- a/dbm-ui/backend/db_services/mysql/sql_import/serializers.py +++ b/dbm-ui/backend/db_services/mysql/sql_import/serializers.py @@ -15,6 +15,7 @@ from backend.db_services.mysql.sql_import import mock_data from backend.db_services.mysql.sql_import.constants import ( BKREPO_SQLFILE_PATH, + MAX_UPLOAD_SQL_FILE_SIZE, SQLCharset, SQLExecuteTicketMode, SQLImportMode, @@ -43,6 +44,8 @@ def validate(self, attrs): raise ValidationError(_("不允许语法检查的sql的内容为空！")) for file in attrs.get("sql_files", []): + if file.size > MAX_UPLOAD_SQL_FILE_SIZE: + raise ValidationError(_("请保证单个文件{}不超过1G").fromat(file.name)) if file.name.rsplit(".")[-1] != "sql": raise ValidationError(_("请保证sql文件[{}]的后缀为.sql").format(file.name)) diff --git a/helm-charts/bk-dbm/Chart.lock b/helm-charts/bk-dbm/Chart.lock index e40a0134f9..489164f357 100644 --- a/helm-charts/bk-dbm/Chart.lock +++ b/helm-charts/bk-dbm/Chart.lock @@ -4,25 +4,25 @@ dependencies: version: 1.13.0 - name: mysql repository: https://charts.bitnami.com/bitnami - version: 9.15.0 + version: 9.23.0 - name: redis repository: https://charts.bitnami.com/bitnami version: 16.13.2 - name: etcd repository: https://charts.bitnami.com/bitnami - version: 9.7.5 + version: 9.15.2 - name: reloader repository: https://stakater.github.io/stakater-charts - version: 1.0.54 + version: 1.0.69 - name: grafana repository: file://./charts/grafana version: 7.9.8 - name: dbm repository: file://charts/dbm - version: 0.1.31 + version: 0.1.32 - name: dbconfig repository: file://charts/dbconfig - version: 0.1.12 + version: 0.1.13 - name: dbpriv repository: file://charts/dbpriv version: 0.1.33 @@ -56,5 +56,5 @@ dependencies: - name: backup-consumer repository: file://charts/backup-consumer version: 0.0.3 -digest: sha256:fb23872583e43da18a82a66979e2c0be8bc737d32fa04fc5e567741392484e25 -generated: "2023-12-16T16:58:39.846364+08:00" +digest: sha256:cdce7dfdf738437c0905f61ac8d058a90a1740ca22ed2da5c9ab6c2e863f91d1 +generated: "2024-03-07T20:39:45.062582+08:00" diff --git a/helm-charts/bk-dbm/Chart.yaml b/helm-charts/bk-dbm/Chart.yaml index 8edb4b1794..605f0d6662 100644 --- a/helm-charts/bk-dbm/Chart.yaml +++ b/helm-charts/bk-dbm/Chart.yaml @@ -27,11 +27,11 @@ dependencies: - condition: dbm.enabled name: dbm repository: file://charts/dbm - version: 0.1.31 + version: 0.1.32 - condition: dbconfig.enabled name: dbconfig repository: file://charts/dbconfig - version: 0.1.12 + version: 0.1.13 - condition: dbpriv.enabled name: dbpriv repository: file://charts/dbpriv @@ -79,5 +79,5 @@ dependencies: description: A Helm chart for bk-dbm name: bk-dbm type: application -version: 1.3.0-alpha.39 -appVersion: 1.3.0-alpha.39 \ No newline at end of file +version: 1.3.0-alpha.40 +appVersion: 1.3.0-alpha.40 \ No newline at end of file diff --git a/helm-charts/bk-dbm/charts/dbconfig/Chart.yaml b/helm-charts/bk-dbm/charts/dbconfig/Chart.yaml index 367ec525c0..1734df1393 100644 --- a/helm-charts/bk-dbm/charts/dbconfig/Chart.yaml +++ b/helm-charts/bk-dbm/charts/dbconfig/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 0.0.1-alpha.78 +appVersion: 0.0.1-alpha.80 description: A Helm chart for dbconfig name: dbconfig type: application -version: 0.1.12 +version: 0.1.13 diff --git a/helm-charts/bk-dbm/charts/dbm/Chart.yaml b/helm-charts/bk-dbm/charts/dbm/Chart.yaml index ada3861d75..4559ab20d9 100644 --- a/helm-charts/bk-dbm/charts/dbm/Chart.yaml +++ b/helm-charts/bk-dbm/charts/dbm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 1.3.0-alpha.431 +appVersion: 1.3.0-alpha.489 description: A Helm chart for dbm name: dbm type: application -version: 0.1.31 +version: 0.1.32