diff --git a/src/backend/booster/server/pkg/api/v2/handler.go b/src/backend/booster/server/pkg/api/v2/handler.go index 5215b33e..b6f0e779 100644 --- a/src/backend/booster/server/pkg/api/v2/handler.go +++ b/src/backend/booster/server/pkg/api/v2/handler.go @@ -16,10 +16,11 @@ import ( "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/blog" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/codec" - "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/types" + commonTypes "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/types" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/server/pkg/api" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/server/pkg/engine" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/server/pkg/manager" + "github.com/TencentBlueKing/bk-turbo/src/backend/booster/server/pkg/types" "github.com/emicklei/go-restful" ) @@ -167,7 +168,7 @@ func ReleaseResource(req *restful.Request, resp *restful.Response) { blog.Errorf("release resource: release task(%s) failed, url(%s): %v", param.TaskID, req.Request.URL.String(), err) - if err == engine.ErrorUnterminatedTaskNoFound { + if err == engine.ErrorUnterminatedTaskNoFound || err == types.ErrorTaskAlreadyTerminated { api.ReturnRest(&api.RestResponse{Resp: resp, Message: err.Error()}) return } @@ -210,7 +211,7 @@ func getApplyParam(req *restful.Request) (*manager.TaskCreateParam, error) { } param := &manager.TaskCreateParam{ - ProjectID: types.GetProjectIDWithScene(protocol.ProjectID, protocol.Scene), + ProjectID: commonTypes.GetProjectIDWithScene(protocol.ProjectID, protocol.Scene), BuildID: protocol.BuildID, ClientVersion: protocol.ClientVersion, ClientCPU: protocol.ClientCPU, @@ -240,7 +241,7 @@ func getMessageParam(req *restful.Request) (*ParamMessage, error) { if protocol.Type == "" { protocol.Type = MessageTask } - protocol.ProjectID = types.GetProjectIDWithScene(protocol.ProjectID, protocol.Scene) + protocol.ProjectID = commonTypes.GetProjectIDWithScene(protocol.ProjectID, protocol.Scene) blog.Debugf("get message param: get message: %s", string(body)) return &protocol, nil diff --git a/src/backend/booster/server/pkg/resource/crm/operator/k8s/operator.go b/src/backend/booster/server/pkg/resource/crm/operator/k8s/operator.go index 1be8de11..1b184487 100644 --- a/src/backend/booster/server/pkg/resource/crm/operator/k8s/operator.go +++ b/src/backend/booster/server/pkg/resource/crm/operator/k8s/operator.go @@ -408,6 +408,26 @@ func (o *operator) getFederationTotalNum(url string, ist config.InstanceType) (* return result, nil } +func (o *operator) getPodList(clusterID string) (*coreV1.PodList, error) { + blog.Debugf("k8s-operator: begin to get federation podlist %s, %s", clusterID, o.conf.BcsNamespace) + + if o.conf.BcsNamespace == "" { + return nil, fmt.Errorf("k8s-operator: get podlist failed clusterID(%s): namespace is nil", clusterID) + } + client, err := o.getClientSet(clusterID) + if err != nil { + blog.Errorf("k8s-operator: try to get podlist clusterID(%s) namespace(%s)"+ + "and get client set failed: %v", clusterID, o.conf.BcsNamespace, err) + return nil, err + } + podList, err := client.clientSet.CoreV1().Pods(o.conf.BcsNamespace).List(context.TODO(), metaV1.ListOptions{}) + if err != nil { + blog.Errorf("k8s-operator: get pod list from k8s failed clusterID(%s): %v", clusterID, err) + return nil, err + } + return podList, nil +} + func (o *operator) getFederationResource(clusterID string) ([]*op.NodeInfo, error) { blog.Debugf("k8s-operator: begin to get federation resource %s, %s", clusterID, o.conf.BcsNamespace) nodeInfoList := make([]*op.NodeInfo, 0, 1000) @@ -415,6 +435,11 @@ func (o *operator) getFederationResource(clusterID string) ([]*op.NodeInfo, erro return nil, fmt.Errorf("crm: get federation resource request failed clusterID(%s): namespace is nil", clusterID) } url := fmt.Sprintf(bcsAPIFederatedURI, o.conf.BcsAPIPool.GetAddress(), clusterID, o.conf.BcsNamespace) + podList, err := o.getPodList(clusterID) + if err != nil { + return nodeInfoList, fmt.Errorf("crm: get federation resource request failed clusterID(%s): %s", clusterID, err) + } + for _, ist := range o.conf.InstanceType { result, err := o.getFederationTotalNum(url, ist) if err != nil { //接口请求失败,直接返回错误 @@ -433,12 +458,37 @@ func (o *operator) getFederationResource(clusterID string) ([]*op.NodeInfo, erro return nodeInfoList, err } totalIst := float64(result.Data.Total) + resourceUsed := make(coreV1.ResourceList) + for _, pod := range podList.Items { + if pod.Status.Phase == coreV1.PodSucceeded || pod.Status.Phase == coreV1.PodFailed { + continue + } + if pod.Spec.NodeSelector != nil { + if pod.Spec.NodeSelector[o.platformLabelKey] != ist.Platform || + pod.Spec.NodeSelector[o.cityLabelKey] != ist.Group { + continue + } + } + for podName, podLimitValue := range podLimits(&pod) { + if value, ok := resourceUsed[podName]; !ok { + resourceUsed[podName] = podLimitValue.DeepCopy() + } else { + value.Add(podLimitValue) + resourceUsed[podName] = value + } + } + } nodeInfoList = append(nodeInfoList, &op.NodeInfo{ IP: clusterID + "-" + o.conf.BcsNamespace + "-" + ist.Platform + "-" + ist.Group, Hostname: clusterID + "-" + o.conf.BcsNamespace + "-" + ist.Platform + "-" + ist.Group, DiskLeft: totalIst, - CPULeft: totalIst * ist.CPUPerInstance, - MemLeft: totalIst * ist.MemPerInstance, + //CPULeft: totalIst * ist.CPUPerInstance, + //MemLeft: totalIst * ist.MemPerInstance, + MemUsed: float64(resourceUsed.Memory().Value()) / 1024 / 1024, + CPUUsed: float64(resourceUsed.Cpu().Value()), + DiskUsed: float64(resourceUsed.StorageEphemeral().Value()), + CPUTotal: float64(resourceUsed.Cpu().Value()) + totalIst*ist.CPUPerInstance, + MemTotal: float64(resourceUsed.Memory().Value())/1024/1024 + totalIst*ist.MemPerInstance, Attributes: map[string]string{ op.AttributeKeyPlatform: ist.Platform, op.AttributeKeyCity: ist.Group, @@ -984,6 +1034,19 @@ func podRequests(pod *coreV1.Pod) coreV1.ResourceList { return requests } +func podLimits(pod *coreV1.Pod) coreV1.ResourceList { + limits := coreV1.ResourceList{} + for _, container := range pod.Spec.Containers { + addResourceList(limits, container.Resources.Limits) + } + // init containers define the minimum of any resource + for _, container := range pod.Spec.InitContainers { + maxResourceList(limits, container.Resources.Limits) + } + + return limits +} + // addResourceList adds the resources in newList to list func addResourceList(list, new coreV1.ResourceList) { for name, quantity := range new {