perf: 获取agent状态做分页获取优化 #7440

gcloud/utils/data_handler.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+
+def _default_func(data, *args, **kwargs):
+    return data
+
+
+def chunk_data(data, chunk_size, func=None, *args, **kwargs):
+    return [(func or _default_func)(data[i : i + chunk_size], *args, **kwargs) for i in range(0, len(data), chunk_size)]

pipeline_plugins/cmdb_ip_picker/query.py

@@ -23,8 +23,10 @@
 from api.utils.request import batch_request
 from gcloud.conf import settings
 from gcloud.utils import cmdb
+from gcloud.utils.data_handler import chunk_data
 from gcloud.utils.handlers import handle_api_error
 from gcloud.utils.ip import format_sundry_ip
+from pipeline_plugins.components.utils.common import batch_execute_func
 
 from .constants import ERROR_CODES, NO_ERROR
 from .utils import (
@@ -40,6 +42,17 @@
 get_client_by_user = settings.ESB_GET_CLIENT_BY_USER
 
 
+def format_agent_ip(data, *args, **kwargs):
+    bk_biz_id = kwargs["bk_biz_id"]
+    return [
+        {
+            "host_id": host["bk_host_id"],
+            "meta": {"bk_biz_id": bk_biz_id, "scope_type": "biz", "scope_id": bk_biz_id},
+        }
+        for host in data
+    ]
+
+
 def cmdb_search_topo_tree(request, bk_biz_id, bk_supplier_account=""):
     """
     @summary: 获取 CMDB 上业务的拓扑树，包含空闲机和故障机模块，根节点是业务
@@ -162,24 +175,21 @@ def cmdb_search_host(request, bk_biz_id, bk_supplier_account="", bk_supplier_id=
                     host["agent"] = agent_id_status_map.get(bk_agent_id, -1)
             else:
                 client = BKNodeManClient(username=request.user.username)
-                agent_kwargs = {
-                    "all_scope": True,
-                    "host_list": [
-                        {
-                            "host_id": host["bk_host_id"],
-                            "meta": {"bk_biz_id": bk_biz_id, "scope_type": "biz", "scope_id": bk_biz_id},
-                        }
-                        for host in data
-                    ],
-                }
-                agent_result = client.get_ipchooser_host_details(**agent_kwargs)
-                if not agent_result["result"]:
-                    message = handle_api_error(
-                        _("节点管理(nodeman)"), "nodeman.get_ipchooser_host_details", agent_kwargs, agent_result
-                    )
-                    result = {"result": False, "code": ERROR_CODES.API_GSE_ERROR, "message": message}
-                    return JsonResponse(result)
-                agent_data = format_agent_data(agent_result["data"])
+                host_list = chunk_data(data, 1000, format_agent_ip, bk_biz_id=bk_biz_id)
+                agent_kwargs = [{"all_scope": True, "host_list": host} for host in host_list]
+                results = batch_execute_func(client.get_ipchooser_host_details, agent_kwargs, interval_enabled=True)
+
+                agent_data = []
+                for result in results:
+                    agent_result = result["result"]
+                    if not agent_result["result"]:
+                        message = handle_api_error(
+                            _("节点管理(nodeman)"), "nodeman.get_ipchooser_host_details", agent_kwargs, agent_result
+                        )
+                        result = {"result": False, "code": ERROR_CODES.API_GSE_ERROR, "message": message}
+                        return JsonResponse(result)
+                    agent_data.extend(agent_result["data"])
+                agent_data = format_agent_data(agent_data)
                 for host in data:
                     # agent在线状态，0为不在线，1为在线，-1为未知
                     agent_info = agent_data.get(

pipeline_plugins/variables/collections/sites/open/ip_filter_base.py

@@ -22,10 +22,11 @@
 from gcloud.core.models import Project
 from gcloud.exceptions import ApiRequestError
 from gcloud.utils import cmdb
+from gcloud.utils.data_handler import chunk_data
 from gcloud.utils.handlers import handle_api_error
 from gcloud.utils.ip import IpRegexType, extract_ip_from_ip_str, get_ip_by_regex_type
 from pipeline_plugins.base.utils.inject import supplier_account_for_business, supplier_id_for_project
-from pipeline_plugins.cmdb_ip_picker.utils import agent_params_pagination, format_agent_data, get_gse_agent_status_ipv6
+from pipeline_plugins.cmdb_ip_picker.utils import format_agent_data, get_gse_agent_status_ipv6
 from pipeline_plugins.components.collections.sites.open.cc.base import cc_get_host_by_innerip_with_ipv6
 from pipeline_plugins.components.utils.common import batch_execute_func
 
@@ -95,11 +96,25 @@ def match_ges_v2(self, gse_agent_status, username, bk_biz_id, bk_supplier_id, or
 
         return match_ip
 
+    @staticmethod
+    def format_origin_ip(data, *args, **kwargs):
+        bk_biz_id = kwargs["bk_biz_id"]
+        return [
+            {
+                "cloud_id": host["bk_cloud_id"],
+                "ip": host["ip"],
+                "meta": {"bk_biz_id": bk_biz_id, "scope_type": "biz", "scope_id": bk_biz_id},
+            }
+            for host in data
+        ]
+
     def match_gse_v1(self, gse_agent_status, username, bk_biz_id, bk_supplier_id, origin_ip_list):
         match_ip = origin_ip_list
         if gse_agent_status in [GseAgentStatus.ONlINE.value, GseAgentStatus.OFFLINE.value]:
+
             client = get_nodeman_client_by_user(username=username)
-            agent_kwargs = agent_params_pagination(origin_ip_list, bk_biz_id)
+            host_list = chunk_data(origin_ip_list, 1000, self.format_origin_ip, bk_biz_id=bk_biz_id)
+            agent_kwargs = [{"all_scope": True, "host_list": host} for host in host_list]
             results = batch_execute_func(client.get_ipchooser_host_details, agent_kwargs, interval_enabled=True)
             agent_data = []
             for result in results: