feature: Agent 安装 DHCP 场景适配 (closed #787)

apps/backend/agent/manager.py

@@ -44,7 +44,15 @@ def register_host(cls):
         )
         return act
 
-    def query_tjj_password(self):
+    @classmethod
+    def add_or_update_hosts(cls):
+        """新增或更新主机"""
+        act = AgentServiceActivity(
+            component_code=components.AddOrUpdateHostsComponent.code, name=components.AddOrUpdateHostsComponent.name
+        )
+        return act
+
+    def query_password(self):
         """查询铁将军密码"""
         act = AgentServiceActivity(
             component_code=components.QueryPasswordComponent.code, name=components.QueryPasswordComponent.name
@@ -138,14 +146,6 @@ def get_agent_status(cls, expect_status: str, name=components.GetAgentStatusComp
         act.component.inputs.expect_status = Var(type=Var.PLAIN, value=expect_status)
         return act
 
-    @classmethod
-    def get_agent_id(cls, name=components.GetAgentIDComponent.name):
-        """查询 Agent ID"""
-        if settings.GSE_VERSION == "V1":
-            return None
-        act = AgentServiceActivity(component_code=components.GetAgentIDComponent.code, name=name)
-        return act
-
     @classmethod
     def check_agent_status(cls, name=components.CheckAgentStatusComponent.name):
         """查询Agent状态是否正常"""

apps/backend/components/collections/agent_new/components.py

@@ -14,12 +14,12 @@
 from pipeline.component_framework.component import Component
 
 from . import base
+from .add_or_update_hosts import AddOrUpdateHostsService
 from .bind_host_agent import BindHostAgentService
 from .check_agent_status import CheckAgentStatusService
 from .check_policy_gse_to_proxy import CheckPolicyGseToProxyService
 from .choose_access_point import ChooseAccessPointService
 from .configure_policy import ConfigurePolicyService
-from .get_agent_id import GetAgentIDService
 from .get_agent_status import GetAgentStatusService
 from .install import InstallService
 from .install_plugins import InstallPluginsService
@@ -79,12 +79,6 @@ class GetAgentStatusComponent(Component):
     bound_service = GetAgentStatusService
 
 
-class GetAgentIDComponent(Component):
-    name = _("查询AgentID")
-    code = "get_agent_id"
-    bound_service = GetAgentIDService
-
-
 class ReloadAgentConfigComponent(Component):
     name = _("重载Agent配置")
     code = "reload_agent_config"
@@ -179,3 +173,9 @@ class UnBindHostAgentComponent(Component):
     name = _("主机解绑 Agent 信息")
     code = "unbind_host_agent"
     bound_service = UnBindHostAgentService
+
+
+class AddOrUpdateHostsComponent(Component):
+    name = _("新增或更新主机信息")
+    code = "add_or_update_hosts"
+    bound_service = AddOrUpdateHostsService

apps/backend/components/collections/agent_new/query_password.py

@@ -9,8 +9,10 @@
 specific language governing permissions and limitations under the License.
 """
 
-from typing import Any, Dict, List, Union
+import base64
+from typing import Any, Dict, List, Optional, Union
 
+from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 
 from apps.core.concurrent import controller
@@ -59,15 +61,111 @@ def query_password(
 
         return {"is_ok": is_ok, "success_ips": success_ips, "failed_ips": failed_ips, "err_msg": err_msg}
 
+    def check_and_update_identity_data(
+        self, sub_insts: List[models.SubscriptionInstanceRecord]
+    ) -> List[models.SubscriptionInstanceRecord]:
+
+        bk_host_ids: List[int] = [sub_inst.instance_info["host"]["bk_host_id"] for sub_inst in sub_insts]
+        exist_identity_data_objs: List[models.IdentityData] = models.IdentityData.objects.filter(
+            bk_host_id__in=bk_host_ids
+        )
+        host_id__identity_data_obj_map: Dict[int, models.IdentityData] = {
+            identity_data_obj.bk_host_id: identity_data_obj for identity_data_obj in exist_identity_data_objs
+        }
+
+        def _is_auth_info_empty(_sub_inst: models.SubscriptionInstanceRecord, _host_info: Dict[str, Any]) -> bool:
+            # 「第三方拉取密码」或者「手动安装」场景下无需校验是否存在认证信息
+            if not (_host_info.get("auth_type") == constants.AuthType.TJJ_PASSWORD or is_manual):
+                # 记录不存在认证信息的订阅实例ID，跳过记录
+                if not (_host_info.get("password") or _host_info.get("key")):
+                    return True
+            return False
+
+        empty_auth_info_sub_inst_ids: List[int] = []
+        identity_data_objs_to_be_created: List[models.IdentityData] = []
+        identity_data_objs_to_be_updated: List[models.IdentityData] = []
+        sub_insts_with_auth_info: List[models.SubscriptionInstanceRecord] = []
+        for sub_inst in sub_insts:
+            host_info: Dict[str, Any] = sub_inst.instance_info["host"]
+            bk_host_id: int = host_info["bk_host_id"]
+            is_manual: bool = host_info.get("is_manual", False)
+            sub_insts_with_auth_info.append(sub_inst)
+
+            # 先校验再更新，防止存量历史任务重试（认证信息已失效）的情况下，重置最新的认证信息快照
+            identity_data_obj: Optional[models.IdentityData] = host_id__identity_data_obj_map.get(bk_host_id)
+
+            if not identity_data_obj:
+                if _is_auth_info_empty(sub_inst, host_info):
+                    empty_auth_info_sub_inst_ids.append(sub_inst.id)
+                    continue
+
+                # 新建认证信息对象
+                identity_data_obj: models.IdentityData = models.IdentityData(
+                    bk_host_id=bk_host_id,
+                    auth_type=host_info.get("auth_type"),
+                    account=host_info.get("account"),
+                    password=base64.b64decode(host_info.get("password", "")).decode(),
+                    port=host_info.get("port"),
+                    key=base64.b64decode(host_info.get("key", "")).decode(),
+                    retention=host_info.get("retention", 1),
+                    extra_data=host_info.get("extra_data", {}),
+                    updated_at=timezone.now(),
+                )
+                identity_data_objs_to_be_created.append(identity_data_obj)
+                continue
+
+            # 更新场景下，传入 `auth_type` 需要提供完整认证信息用于更新，否则使用快照
+            auth_type: Optional[str] = host_info.get("auth_type")
+            if not auth_type:
+                identity_data_objs_to_be_updated.append(identity_data_obj)
+                continue
+
+            # 处理认证信息不完整的情况
+            if _is_auth_info_empty(sub_inst, host_info):
+                empty_auth_info_sub_inst_ids.append(sub_inst.id)
+                continue
+
+            # 允许缺省时使用快照
+            identity_data_obj.port = host_info.get("port") or identity_data_obj.port
+            identity_data_obj.account = host_info.get("account") or identity_data_obj.account
+
+            # 更新认证信息
+            identity_data_obj.auth_type = auth_type
+            identity_data_obj.password = base64.b64decode(host_info.get("password", "")).decode()
+            identity_data_obj.key = base64.b64decode(host_info.get("key", "")).decode()
+            identity_data_obj.retention = host_info.get("retention", 1)
+            identity_data_obj.extra_data = host_info.get("extra_data", {})
+            identity_data_obj.updated_at = timezone.now()
+
+            identity_data_objs_to_be_updated.append(identity_data_obj)
+
+        models.IdentityData.objects.bulk_create(identity_data_objs_to_be_created, batch_size=self.batch_size)
+        models.IdentityData.objects.bulk_update(
+            identity_data_objs_to_be_updated,
+            fields=["auth_type", "account", "password", "port", "key", "retention", "extra_data", "updated_at"],
+            batch_size=self.batch_size,
+        )
+
+        # 移除不存在认证信息的实例
+        self.move_insts_to_failed(
+            sub_inst_ids=empty_auth_info_sub_inst_ids, log_content=_("登录认证信息已被清空\n" "- 若为重试操作，请新建任务重新发起")
+        )
+
+        return sub_insts_with_auth_info
+
     def _execute(self, data, parent_data, common_data: AgentCommonData):
         creator = data.get_one_of_inputs("creator")
         host_id_obj_map = common_data.host_id_obj_map
 
+        subscription_instances: List[models.SubscriptionInstanceRecord] = self.check_and_update_identity_data(
+            common_data.subscription_instances
+        )
+
         no_need_query_inst_ids = []
         # 这里暂不支持多
         cloud_ip_map = {}
         oa_ticket = ""
-        for sub_inst in common_data.subscription_instances:
+        for sub_inst in subscription_instances:
             bk_host_id = sub_inst.instance_info["host"]["bk_host_id"]
             host = host_id_obj_map[bk_host_id]
 

apps/backend/components/collections/agent_new/register_host.py

@@ -8,7 +8,6 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-import base64
 import json
 from collections import ChainMap, defaultdict
 from typing import Any, Callable, Dict, Iterable, List, Optional, Set
@@ -263,35 +262,6 @@ def add_hosts_to_biz_resource(
             for success_host_key in success_host_keys
         }
 
-    def handle_empty_auth_info_case(
-        self, subscription_instances: List[models.SubscriptionInstanceRecord]
-    ) -> Dict[str, models.SubscriptionInstanceRecord]:
-        """
-        处理不存在认证信息的情况
-        :param subscription_instances: 订阅实例对象列表
-        :return: host_key__sub_inst_map 校验通过的「host_key - 订阅实例映射」
-        """
-        empty_auth_info_sub_inst_ids: List[int] = []
-        host_key__sub_inst_map: Dict[str, models.SubscriptionInstanceRecord] = {}
-        for subscription_instance in subscription_instances:
-            host_info = subscription_instance.instance_info["host"]
-            is_manual = host_info.get("is_manual", False)
-
-            # 「第三方拉取密码」或者「手动安装」的场景下无需校验是否存在认证信息
-            if not (host_info.get("auth_type") == constants.AuthType.TJJ_PASSWORD or is_manual):
-                # 记录不存在认证信息的订阅实例ID，跳过记录
-                if not (host_info.get("password") or host_info.get("key")):
-                    empty_auth_info_sub_inst_ids.append(subscription_instance.id)
-                    continue
-            # 通过校验的订阅实例构建host_key - 订阅实例 映射
-            host_key__sub_inst_map[f"{host_info['bk_cloud_id']}-{host_info['bk_host_innerip']}"] = subscription_instance
-
-        # 移除不存在认证信息的实例
-        self.move_insts_to_failed(
-            sub_inst_ids=empty_auth_info_sub_inst_ids, log_content=_("该主机的登录认证信息已被清空，无法重试，请重新发起安装任务")
-        )
-        return host_key__sub_inst_map
-
     def handle_add_hosts_to_cmdb_case(
         self, host_key__sub_inst_map: Dict[str, models.SubscriptionInstanceRecord]
     ) -> Dict[str, int]:
@@ -340,9 +310,6 @@ def handle_update_db(
         host_ids_in_exist_hosts: Set[int] = set(
             models.Host.objects.filter(bk_host_id__in=bk_host_ids).values_list("bk_host_id", flat=True)
         )
-        host_ids_in_exist_identity_data: Set[int] = set(
-            models.IdentityData.objects.filter(bk_host_id__in=bk_host_ids).values_list("bk_host_id", flat=True)
-        )
         host_ids_in_exist_proc_statuses: Set[int] = set(
             models.ProcessStatus.objects.filter(
                 bk_host_id__in=bk_host_ids,
@@ -353,8 +320,6 @@ def handle_update_db(
         host_objs_to_be_created: List[models.Host] = []
         host_objs_to_be_updated: List[models.Host] = []
         proc_status_objs_to_be_created: List[models.ProcessStatus] = []
-        identity_data_objs_to_be_created: List[models.IdentityData] = []
-        identity_data_objs_to_be_updated: List[models.IdentityData] = []
         sub_inst_objs_to_be_updated: List[models.SubscriptionInstanceRecord] = []
         for host_key, bk_host_id in host_key__bk_host_id_map.items():
 
@@ -402,22 +367,6 @@ def handle_update_db(
                 ]
                 host_objs_to_be_created.append(host_obj)
 
-            identity_data_obj = models.IdentityData(
-                bk_host_id=bk_host_id,
-                auth_type=host_info.get("auth_type"),
-                account=host_info.get("account"),
-                password=base64.b64decode(host_info.get("password", "")).decode(),
-                port=host_info.get("port"),
-                key=base64.b64decode(host_info.get("key", "")).decode(),
-                retention=host_info.get("retention", 1),
-                extra_data=host_info.get("extra_data", {}),
-                updated_at=timezone.now(),
-            )
-            if bk_host_id in host_ids_in_exist_identity_data:
-                identity_data_objs_to_be_updated.append(identity_data_obj)
-            else:
-                identity_data_objs_to_be_created.append(identity_data_obj)
-
             if bk_host_id not in host_ids_in_exist_proc_statuses:
                 proc_status_obj = models.ProcessStatus(
                     bk_host_id=bk_host_id,
@@ -439,12 +388,6 @@ def handle_update_db(
             batch_size=self.batch_size,
         )
         models.ProcessStatus.objects.bulk_create(proc_status_objs_to_be_created, batch_size=self.batch_size)
-        models.IdentityData.objects.bulk_create(identity_data_objs_to_be_created, batch_size=self.batch_size)
-        models.IdentityData.objects.bulk_update(
-            identity_data_objs_to_be_updated,
-            fields=["auth_type", "account", "password", "port", "key", "retention", "extra_data", "updated_at"],
-            batch_size=self.batch_size,
-        )
         models.SubscriptionInstanceRecord.objects.bulk_update(
             sub_inst_objs_to_be_updated, fields=["instance_info"], batch_size=self.batch_size
         )
@@ -458,8 +401,11 @@ def _execute(self, data, parent_data, common_data: AgentCommonData):
 
         subscription_instances: List[models.SubscriptionInstanceRecord] = common_data.subscription_instances
 
-        # key / password expired, failed
-        host_key__sub_inst_map = self.handle_empty_auth_info_case(subscription_instances)
+        host_key__sub_inst_map: Dict[str, models.SubscriptionInstanceRecord] = {}
+        for subscription_instance in subscription_instances:
+            host_info = subscription_instance.instance_info["host"]
+            # 通过校验的订阅实例构建host_key - 订阅实例 映射
+            host_key__sub_inst_map[f"{host_info['bk_cloud_id']}-{host_info['bk_host_innerip']}"] = subscription_instance
 
         # host infos group by bk_biz_id and register add_host_to_resource
         host_key__bk_host_id_map = self.handle_add_hosts_to_cmdb_case(host_key__sub_inst_map)