Skip to content

Commit

Permalink
feat: support general http plugin query params
Browse files Browse the repository at this point in the history
  • Loading branch information
narasux committed Nov 3, 2023
1 parent cf2ed7f commit 534f506
Show file tree
Hide file tree
Showing 4 changed files with 44 additions and 12 deletions.
37 changes: 26 additions & 11 deletions src/bk-user/bkuser/plugins/general/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,15 @@
PageSize,
)
from bkuser.plugins.general.exceptions import RequestApiError, RespDataFormatError
from bkuser.plugins.general.models import AuthConfig
from bkuser.plugins.general.models import AuthConfig, QueryParam

logger = logging.getLogger(__name__)


def gen_query_params(params: List[QueryParam]) -> Dict[str, str]:
return {p.key: p.value for p in params}


def gen_headers(cfg: AuthConfig) -> Dict[str, str]:
headers = {"Content-Type": "application/json"}

Expand All @@ -44,14 +48,20 @@ def gen_headers(cfg: AuthConfig) -> Dict[str, str]:
return headers


def stringify_params(params: Dict[str, Any]) -> str:
"""字符串化查询参数,仅用于错误信息提示"""
return "&".join([f"{k}={v}" for k, v in params.items()])


def fetch_all_data(
url: str, headers: Dict[str, str], page_size: PageSize, timeout: int, retries: int
url: str, headers: Dict[str, str], params: Dict[str, Any], page_size: PageSize, timeout: int, retries: int
) -> List[Dict[str, Any]]:
"""
根据指定配置,请求数据源 API 以获取用户 / 部门数据
:param url: 数据源 URL,如 https://bk.example.com/apis/v1/users
:param headers: 请求头,包含认证信息等
:param params: 查询参数,即 url 中 ?scope=company 部分
:param timeout: 单次请求超时时间
:param retries: 请求失败重试次数
:returns: API 返回结果,应符合通用 HTTP 数据源 API 协议
Expand All @@ -73,21 +83,23 @@ def fetch_all_data(
cur_page, max_page = DEFAULT_PAGE, MAX_TOTAL_COUNT / page_size
total_cnt, items = 0, []
while True:
params = {"page": cur_page, "page_size": page_size}
params.update({"page": cur_page, "page_size": page_size})
resp = session.get(url, headers=headers, params=params, timeout=timeout)
if not resp.ok:
raise RequestApiError(
_("请求数据源 API {} 参数 {} 异常,状态码 {} 响应内容 {}").format(
url, params, resp.status_code, resp.content
url, stringify_params(params), resp.status_code, resp.content
) # noqa: E501
)

try:
resp_data = resp.json()
except JSONDecodeError: # noqa: PERF203
raise RespDataFormatError(
_("数据源 API {} 参数 {} 返回非 Json 格式,响应内容 {}").format(url, params, resp.content)
) # noqa: E501
_("数据源 API {} 参数 {} 返回非 Json 格式,响应内容 {}").format(
url, stringify_params(params), resp.content
) # noqa: E501
)

total_cnt = resp_data.get("count", 0)
cur_req_results = resp_data.get("results", [])
Expand All @@ -114,28 +126,31 @@ def fetch_all_data(
return items


def fetch_first_item(url: str, headers: Dict[str, str], timeout: int) -> Dict[str, Any] | None:
def fetch_first_item(url: str, headers: Dict[str, str], params: Dict[str, Any], timeout: int) -> Dict[str, Any] | None:
"""
根据指定配置,请求数据源 API 以获取用户 / 部门第一条数据(测试连通性用)
:param url: 数据源 URL,如 https://bk.example.com/apis/v1/users
:param headers: 请求头,包含认证信息等
:param params: 查询参数,即 url 中 ?scope=company 部分
:param timeout: 单次请求超时时间
:returns: API 返回结果,应符合通用 HTTP 数据源 API 协议
"""
params = {"page": DEFAULT_PAGE, "page_size": PAGE_SIZE_FOR_FETCH_FIRST}
params.update({"page": DEFAULT_PAGE, "page_size": PAGE_SIZE_FOR_FETCH_FIRST})
resp = requests.get(url, headers=headers, params=params, timeout=timeout)
if not resp.ok:
raise RequestApiError(
_("请求数据源 API {} 参数 {} 异常,状态码 {} 响应内容 {}").format(url, params, resp.status_code, resp.content) # noqa: E501
_("请求数据源 API {} 参数 {} 异常,状态码 {} 响应内容 {}").format(
url, stringify_params(params), resp.status_code, resp.content
) # noqa: E501
)

try:
resp_data = resp.json()
except JSONDecodeError: # noqa: PERF203
raise RespDataFormatError(
_("数据源 API {} 参数 {} 返回非 Json 格式,响应内容 {}").format(url, params, resp.content)
) # noqa: E501
_("数据源 API {} 参数 {} 返回非 Json 格式,响应内容 {}").format(url, stringify_params(params), resp.content) # noqa: E501
)

results = resp_data.get("results", [])
if not results:
Expand Down
11 changes: 11 additions & 0 deletions src/bk-user/bkuser/plugins/general/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,26 @@
)


class QueryParam(BaseModel):
"""查询参数"""

key: str
value: str


class ServerConfig(BaseModel):
"""数据服务相关配置"""

# 服务地址
server_base_url: str = Field(pattern=BASE_URL_REGEX)
# 用户数据 API 路径
user_api_path: str = Field(pattern=API_URL_PATH_REGEX)
# 用户数据 API 请求参数
user_api_query_params: list[QueryParam] = []
# 部门数据 API 路径
department_api_path: str = Field(pattern=API_URL_PATH_REGEX)
# 部门数据 API 请求参数
department_api_query_params: list[QueryParam] = []
# 单次分页请求数量
page_size: PageSize = PageSize.CNT_100
# 单次请求超时时间
Expand Down
6 changes: 5 additions & 1 deletion src/bk-user/bkuser/plugins/general/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from bkuser.plugins.base import BaseDataSourcePlugin
from bkuser.plugins.constants import DataSourcePluginEnum
from bkuser.plugins.general.exceptions import RequestApiError, RespDataFormatError
from bkuser.plugins.general.http import fetch_all_data, fetch_first_item, gen_headers
from bkuser.plugins.general.http import fetch_all_data, fetch_first_item, gen_headers, gen_query_params
from bkuser.plugins.general.models import GeneralDataSourcePluginConfig
from bkuser.plugins.models import (
RawDataSourceDepartment,
Expand All @@ -42,6 +42,7 @@ def fetch_departments(self) -> List[RawDataSourceDepartment]:
depts = fetch_all_data(
cfg.server_base_url + cfg.department_api_path,
gen_headers(self.plugin_config.auth_config),
gen_query_params(cfg.department_api_query_params),
cfg.page_size,
cfg.request_timeout,
cfg.retries,
Expand All @@ -54,6 +55,7 @@ def fetch_users(self) -> List[RawDataSourceUser]:
users = fetch_all_data(
cfg.server_base_url + cfg.user_api_path,
gen_headers(self.plugin_config.auth_config),
gen_query_params(cfg.user_api_query_params),
cfg.page_size,
cfg.request_timeout,
cfg.retries,
Expand All @@ -69,12 +71,14 @@ def test_connection(self) -> TestConnectionResult:
user_data = fetch_first_item(
cfg.server_base_url + cfg.user_api_path,
gen_headers(self.plugin_config.auth_config),
gen_query_params(cfg.user_api_query_params),
cfg.request_timeout,
)

dept_data = fetch_first_item(
cfg.server_base_url + cfg.department_api_path,
gen_headers(self.plugin_config.auth_config),
gen_query_params(cfg.department_api_query_params),
cfg.request_timeout,
)
except (RequestApiError, RespDataFormatError) as e:
Expand Down
2 changes: 2 additions & 0 deletions src/bk-user/tests/fixtures/data_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,9 @@ def general_ds_plugin_cfg() -> Dict[str, Any]:
"server_config": {
"server_base_url": "http://bk.example.com:8090",
"user_api_path": "/api/v1/users",
"user_api_query_params": [{"key": "scope", "value": "company"}],
"department_api_path": "/api/v1/departments",
"department_api_query_params": [],
"request_timeout": 5,
"retries": 3,
},
Expand Down

0 comments on commit 534f506

Please sign in to comment.