Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: support general http plugin query params #1363

Merged
merged 1 commit into from
Nov 3, 2023
Merged

feat: support general http plugin query params #1363

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 26 additions & 11 deletions src/bk-user/bkuser/plugins/general/http.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,15 @@
PageSize,
)
from bkuser.plugins.general.exceptions import RequestApiError, RespDataFormatError
from bkuser.plugins.general.models import AuthConfig
from bkuser.plugins.general.models import AuthConfig, QueryParam

logger = logging.getLogger(__name__)


def gen_query_params(params: List[QueryParam]) -> Dict[str, str]:
return {p.key: p.value for p in params}


def gen_headers(cfg: AuthConfig) -> Dict[str, str]:
headers = {"Content-Type": "application/json"}

Expand All @@ -44,14 +48,20 @@ def gen_headers(cfg: AuthConfig) -> Dict[str, str]:
return headers


def stringify_params(params: Dict[str, Any]) -> str:
"""字符串化查询参数，仅用于错误信息提示"""
return "&".join([f"{k}={v}" for k, v in params.items()])


def fetch_all_data(
url: str, headers: Dict[str, str], page_size: PageSize, timeout: int, retries: int
url: str, headers: Dict[str, str], params: Dict[str, Any], page_size: PageSize, timeout: int, retries: int
) -> List[Dict[str, Any]]:
"""
根据指定配置，请求数据源 API 以获取用户 / 部门数据

:param url: 数据源 URL，如 https://bk.example.com/apis/v1/users
:param headers: 请求头，包含认证信息等
:param params: 查询参数，即 url 中 ?scope=company 部分
:param timeout: 单次请求超时时间
:param retries: 请求失败重试次数
:returns: API 返回结果，应符合通用 HTTP 数据源 API 协议
Expand All @@ -73,21 +83,23 @@ def fetch_all_data(
cur_page, max_page = DEFAULT_PAGE, MAX_TOTAL_COUNT / page_size
total_cnt, items = 0, []
while True:
params = {"page": cur_page, "page_size": page_size}
params.update({"page": cur_page, "page_size": page_size})
resp = session.get(url, headers=headers, params=params, timeout=timeout)
if not resp.ok:
raise RequestApiError(
_("请求数据源 API {} 参数 {} 异常，状态码 {} 响应内容 {}").format(
url, params, resp.status_code, resp.content
url, stringify_params(params), resp.status_code, resp.content
) # noqa: E501
)

try:
resp_data = resp.json()
except JSONDecodeError: # noqa: PERF203
raise RespDataFormatError(
_("数据源 API {} 参数 {} 返回非 Json 格式，响应内容 {}").format(url, params, resp.content)
) # noqa: E501
_("数据源 API {} 参数 {} 返回非 Json 格式，响应内容 {}").format(
url, stringify_params(params), resp.content
) # noqa: E501
)

total_cnt = resp_data.get("count", 0)
cur_req_results = resp_data.get("results", [])
Expand All @@ -114,28 +126,31 @@ def fetch_all_data(
return items


def fetch_first_item(url: str, headers: Dict[str, str], timeout: int) -> Dict[str, Any] | None:
def fetch_first_item(url: str, headers: Dict[str, str], params: Dict[str, Any], timeout: int) -> Dict[str, Any] | None:
"""
根据指定配置，请求数据源 API 以获取用户 / 部门第一条数据（测试连通性用）

:param url: 数据源 URL，如 https://bk.example.com/apis/v1/users
:param headers: 请求头，包含认证信息等
:param params: 查询参数，即 url 中 ?scope=company 部分
:param timeout: 单次请求超时时间
:returns: API 返回结果，应符合通用 HTTP 数据源 API 协议
"""
params = {"page": DEFAULT_PAGE, "page_size": PAGE_SIZE_FOR_FETCH_FIRST}
params.update({"page": DEFAULT_PAGE, "page_size": PAGE_SIZE_FOR_FETCH_FIRST})
resp = requests.get(url, headers=headers, params=params, timeout=timeout)
if not resp.ok:
raise RequestApiError(
_("请求数据源 API {} 参数 {} 异常，状态码 {} 响应内容 {}").format(url, params, resp.status_code, resp.content) # noqa: E501
_("请求数据源 API {} 参数 {} 异常，状态码 {} 响应内容 {}").format(
url, stringify_params(params), resp.status_code, resp.content
) # noqa: E501
)

try:
resp_data = resp.json()
except JSONDecodeError: # noqa: PERF203
raise RespDataFormatError(
_("数据源 API {} 参数 {} 返回非 Json 格式，响应内容 {}").format(url, params, resp.content)
) # noqa: E501
_("数据源 API {} 参数 {} 返回非 Json 格式，响应内容 {}").format(url, stringify_params(params), resp.content) # noqa: E501
)

results = resp_data.get("results", [])
if not results:
Expand Down
11 changes: 11 additions & 0 deletions src/bk-user/bkuser/plugins/general/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,26 @@
)


class QueryParam(BaseModel):
"""查询参数"""

key: str
value: str


class ServerConfig(BaseModel):
"""数据服务相关配置"""

# 服务地址
server_base_url: str = Field(pattern=BASE_URL_REGEX)
# 用户数据 API 路径
user_api_path: str = Field(pattern=API_URL_PATH_REGEX)
# 用户数据 API 请求参数
user_api_query_params: list[QueryParam] = []
# 部门数据 API 路径
department_api_path: str = Field(pattern=API_URL_PATH_REGEX)
# 部门数据 API 请求参数
department_api_query_params: list[QueryParam] = []
# 单次分页请求数量
page_size: PageSize = PageSize.CNT_100
# 单次请求超时时间
Expand Down
6 changes: 5 additions & 1 deletion src/bk-user/bkuser/plugins/general/plugin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from bkuser.plugins.base import BaseDataSourcePlugin
from bkuser.plugins.constants import DataSourcePluginEnum
from bkuser.plugins.general.exceptions import RequestApiError, RespDataFormatError
from bkuser.plugins.general.http import fetch_all_data, fetch_first_item, gen_headers
from bkuser.plugins.general.http import fetch_all_data, fetch_first_item, gen_headers, gen_query_params
from bkuser.plugins.general.models import GeneralDataSourcePluginConfig
from bkuser.plugins.models import (
RawDataSourceDepartment,
Expand All @@ -42,6 +42,7 @@ def fetch_departments(self) -> List[RawDataSourceDepartment]:
depts = fetch_all_data(
cfg.server_base_url + cfg.department_api_path,
gen_headers(self.plugin_config.auth_config),
gen_query_params(cfg.department_api_query_params),
cfg.page_size,
cfg.request_timeout,
cfg.retries,
Expand All @@ -54,6 +55,7 @@ def fetch_users(self) -> List[RawDataSourceUser]:
users = fetch_all_data(
cfg.server_base_url + cfg.user_api_path,
gen_headers(self.plugin_config.auth_config),
gen_query_params(cfg.user_api_query_params),
cfg.page_size,
cfg.request_timeout,
cfg.retries,
Expand All @@ -69,12 +71,14 @@ def test_connection(self) -> TestConnectionResult:
user_data = fetch_first_item(
cfg.server_base_url + cfg.user_api_path,
gen_headers(self.plugin_config.auth_config),
gen_query_params(cfg.user_api_query_params),
cfg.request_timeout,
)

dept_data = fetch_first_item(
cfg.server_base_url + cfg.department_api_path,
gen_headers(self.plugin_config.auth_config),
gen_query_params(cfg.department_api_query_params),
cfg.request_timeout,
)
except (RequestApiError, RespDataFormatError) as e:
Expand Down
2 changes: 2 additions & 0 deletions src/bk-user/tests/fixtures/data_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,9 @@ def general_ds_plugin_cfg() -> Dict[str, Any]:
"server_config": {
"server_base_url": "http://bk.example.com:8090",
"user_api_path": "/api/v1/users",
"user_api_query_params": [{"key": "scope", "value": "company"}],
"department_api_path": "/api/v1/departments",
"department_api_query_params": [],
"request_timeout": 5,
"retries": 3,
},
Expand Down
Loading