From bb5857be4f9482d9f1a00ad7c8e5b369321d322d Mon Sep 17 00:00:00 2001 From: crayon <873217631@qq.com> Date: Tue, 12 Jul 2022 10:49:50 +0800 Subject: [PATCH] =?UTF-8?q?feature:=20SaaS=20=E9=80=82=E9=85=8D=E5=8A=A8?= =?UTF-8?q?=E6=80=81=20IP=20=E5=AE=89=E8=A3=85=E6=A0=A1=E9=AA=8C=EF=BC=8C?= =?UTF-8?q?=E7=BB=9F=E4=B8=80=E4=B8=9A=E5=8A=A1=E9=80=BB=E8=BE=91=20IPv6?= =?UTF-8?q?=20=E8=A1=A8=E7=A4=BA=E6=B3=95=20(closed=20#926=20closed=20#927?= =?UTF-8?q?=20#787)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../collections/agent_new/query_password.py | 27 +-- apps/backend/subscription/serializers.py | 10 + apps/backend/subscription/steps/agent.py | 1 + apps/backend/subscription/tools.py | 37 +++- apps/node_man/constants.py | 11 + apps/node_man/handlers/host.py | 98 ++++---- apps/node_man/handlers/job.py | 99 ++++----- apps/node_man/handlers/meta.py | 5 + apps/node_man/handlers/validator.py | 209 ++++++++++++------ apps/node_man/serializers/host.py | 13 +- apps/node_man/serializers/job.py | 22 +- .../node_man/tests/test_handlers/test_host.py | 2 +- .../tests/test_handlers/test_validator.py | 29 +-- apps/node_man/tests/utils.py | 9 +- apps/utils/basic.py | 14 +- common/context_processors.py | 1 + frontend/index.html | 2 + static/index.html | 4 +- 18 files changed, 363 insertions(+), 230 deletions(-) diff --git a/apps/backend/components/collections/agent_new/query_password.py b/apps/backend/components/collections/agent_new/query_password.py index cd2709854..334787cce 100644 --- a/apps/backend/components/collections/agent_new/query_password.py +++ b/apps/backend/components/collections/agent_new/query_password.py @@ -114,27 +114,16 @@ def _is_auth_info_empty(_sub_inst: models.SubscriptionInstanceRecord, _host_info identity_data_objs_to_be_created.append(identity_data_obj) continue - # 更新场景下，传入 `auth_type` 需要提供完整认证信息用于更新，否则使用快照 - auth_type: Optional[str] = host_info.get("auth_type") - if not auth_type: - identity_data_objs_to_be_updated.append(identity_data_obj) - continue - - # 处理认证信息不完整的情况 - if _is_auth_info_empty(sub_inst, host_info): - empty_auth_info_sub_inst_ids.append(sub_inst.id) - continue - - # 允许缺省时使用快照 + # 更新策略：优先使用传入数据，否则使用历史快照 identity_data_obj.port = host_info.get("port") or identity_data_obj.port identity_data_obj.account = host_info.get("account") or identity_data_obj.account - - # 更新认证信息 - identity_data_obj.auth_type = auth_type - identity_data_obj.password = base64.b64decode(host_info.get("password", "")).decode() - identity_data_obj.key = base64.b64decode(host_info.get("key", "")).decode() - identity_data_obj.retention = host_info.get("retention", 1) - identity_data_obj.extra_data = host_info.get("extra_data", {}) + identity_data_obj.auth_type = host_info.get("auth_type") or identity_data_obj.auth_type + identity_data_obj.password = ( + base64.b64decode(host_info.get("password", "")).decode() or identity_data_obj.password + ) + identity_data_obj.key = base64.b64decode(host_info.get("key", "")).decode() or identity_data_obj.key + identity_data_obj.retention = host_info.get("retention", 1) or identity_data_obj.retention + identity_data_obj.extra_data = host_info.get("extra_data", {}) or identity_data_obj.extra_data identity_data_obj.updated_at = timezone.now() identity_data_objs_to_be_updated.append(identity_data_obj) diff --git a/apps/backend/subscription/serializers.py b/apps/backend/subscription/serializers.py index 8ecddb4d3..b9a848bda 100644 --- a/apps/backend/subscription/serializers.py +++ b/apps/backend/subscription/serializers.py @@ -17,6 +17,7 @@ from apps.node_man import constants, models, tools from apps.node_man.models import ProcessStatus from apps.node_man.serializers import policy +from apps.utils import basic class GatewaySerializer(serializers.Serializer): @@ -32,6 +33,15 @@ class ScopeSerializer(serializers.Serializer): need_register = serializers.BooleanField(required=False, default=False) nodes = serializers.ListField() + def validate(self, attrs): + for node in attrs["nodes"]: + basic.ipv6_formatter(data=node, ipv6_field_names=["ip"]) + basic.ipv6_formatter( + data=node.get("instance_info", {}), + ipv6_field_names=["bk_host_innerip_v6", "bk_host_outerip_v6", "login_ip", "data_ip"], + ) + return attrs + class TargetHostSerializer(serializers.Serializer): bk_host_id = serializers.CharField(required=False, label="目标机器主机ID") diff --git a/apps/backend/subscription/steps/agent.py b/apps/backend/subscription/steps/agent.py index 7175e8b00..ca174510b 100644 --- a/apps/backend/subscription/steps/agent.py +++ b/apps/backend/subscription/steps/agent.py @@ -176,6 +176,7 @@ class ReinstallAgent(AgentAction): def _generate_activities(self, agent_manager: AgentManager): activities = [ + agent_manager.add_or_update_hosts() if settings.BKAPP_ENABLE_DHCP else None, agent_manager.query_password(), agent_manager.choose_ap(), agent_manager.install(), diff --git a/apps/backend/subscription/tools.py b/apps/backend/subscription/tools.py index 9058fb1b2..32bc13719 100644 --- a/apps/backend/subscription/tools.py +++ b/apps/backend/subscription/tools.py @@ -10,6 +10,7 @@ """ import copy import hashlib +import ipaddress import logging import math import os @@ -476,16 +477,32 @@ def get_host_detail(host_info_list: list, bk_biz_id: int = None): host_infos_gby_bk_cloud_id[host_info["bk_cloud_id"]].append(host_info) rules = [] for bk_cloud_id, host_infos in host_infos_gby_bk_cloud_id.items(): - ips = [host_info["ip"] for host_info in host_infos] - rules.append( - { - "condition": "AND", - "rules": [ - {"field": "bk_host_innerip", "operator": "in", "value": ips}, - {"field": "bk_cloud_id", "operator": "equal", "value": bk_cloud_id}, - ], - } - ) + ipv4s = set() + ipv6s = set() + for host_info in host_infos: + if ipaddress.ip_address(host_info["ip"]).version == constants.CmdbIpVersion.V6.value: + ipv6s.add(host_info["ip"]) + else: + ipv4s.add(host_info["ip"]) + for ip_field_name, ips in [("bk_host_innerip", ipv4s), ("bk_host_innerip_v6", ipv6s)]: + # 如果为空， + if not ips: + continue + rules.append( + { + "condition": "AND", + "rules": [ + # 仅允许静态 IP 通过 ip + 云区域 方式下发订阅 + { + "field": "bk_addressing", + "operator": "equal", + "value": constants.CmdbAddressingType.STATIC.value, + }, + {"field": ip_field_name, "operator": "in", "value": ips}, + {"field": "bk_cloud_id", "operator": "equal", "value": bk_cloud_id}, + ], + } + ) cond = {"host_property_filter": {"condition": "OR", "rules": rules}} else: # 如果不满足 bk_host_id / ip & bk_cloud_id 的传入格式，此时直接返回空列表，表示查询不到任何主机 diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py index 97ef01546..e9ef6c189 100644 --- a/apps/node_man/constants.py +++ b/apps/node_man/constants.py @@ -871,6 +871,17 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]: return {cls.STATIC: _("静态"), cls.DYNAMIC: _("动态")} +class CmdbIpVersion(EnhanceEnum): + """IP 版本""" + + V4 = 4 + V6 = 6 + + @classmethod + def _get_member__alias_map(cls) -> Dict[Enum, str]: + return {cls.V4: _("IPv4"), cls.V6: _("IPv6")} + + class PolicyRollBackType: SUPPRESSED = "SUPPRESSED" LOSE_CONTROL = "LOSE_CONTROL" diff --git a/apps/node_man/handlers/host.py b/apps/node_man/handlers/host.py index 7ff098245..e6924f552 100644 --- a/apps/node_man/handlers/host.py +++ b/apps/node_man/handlers/host.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from typing import List +from typing import Iterable, List, Optional from django.db import transaction from django.db.models import Count @@ -364,9 +364,12 @@ def list(self, params: dict, username: str): "bk_cloud_id", "bk_biz_id", "bk_host_id", + "bk_addressing", "os_type", "inner_ip", + "inner_ipv6", "outer_ip", + "outer_ipv6", "ap_id", "install_channel_id", "login_ip", @@ -478,7 +481,9 @@ def proxies(bk_cloud_id: int): "bk_cloud_id", "bk_host_id", "inner_ip", + "inner_ipv6", "outer_ip", + "outer_ipv6", "login_ip", "data_ip", "bk_biz_id", @@ -576,7 +581,15 @@ def biz_proxies(bk_biz_id: int): # 获得proxy相应数据 proxies = list( Host.objects.filter(bk_cloud_id__in=bk_cloud_ids, node_type=const.NodeType.PROXY).values( - "bk_cloud_id", "bk_host_id", "inner_ip", "outer_ip", "login_ip", "data_ip", "bk_biz_id" + "bk_cloud_id", + "bk_addressing", + "inner_ip", + "inner_ipv6", + "outer_ip", + "outer_ipv6", + "login_ip", + "data_ip", + "bk_biz_id", ) ) @@ -751,10 +764,13 @@ def remove_host(self, params: dict): return {} - def ip_list(self, ips): + @staticmethod + def ip_list(ips: Iterable[str], ip_version: int, bk_addressing: Optional[str] = None): """ - 返回存在ips的所有云区域-IP的列表 - :param ips: IP列表 + 返回存在 + :param ips: + :param ip_version: + :param bk_addressing: 寻址方式 :return: { bk_cloud_id+ip: { @@ -765,47 +781,51 @@ def ip_list(self, ips): } }, """ + ips = set(ips) + + if bk_addressing is None: + bk_addressing = const.CmdbAddressingType.STATIC.value + + login_ip_field_name = "login_ip" + login_ip_filter_k = "login_ip__in" + + # 根据 IP 版本筛选不同的 IP 字段 + if ip_version == const.CmdbIpVersion.V6.value: + inner_ip_field_name = "inner_ipv6" + outer_ip_field_name = "outer_ipv6" + inner_ip_filter_k = "inner_ipv6__in" + outer_ip_filter_k = "outer_ipv6__in" + else: + inner_ip_field_name = "inner_ip" + outer_ip_field_name = "outer_ip" + inner_ip_filter_k = "inner_ip__in" + outer_ip_filter_k = "outer_ip__in" + + fields: List[str] = [ + "inner_ip", + "inner_ipv6", + "outer_ip", + "outer_ipv6", + "login_ip", + "bk_cloud_id", + "bk_biz_id", + "node_type", + "bk_host_id", + ] + inner_ip_info = { - f"{host['bk_cloud_id']}-{host['inner_ip']}": { - "inner_ip": host["inner_ip"], - "outer_ip": host["outer_ip"], - "login_ip": host["login_ip"], - "bk_cloud_id": host["bk_cloud_id"], - "bk_biz_id": host["bk_biz_id"], - "node_type": host["node_type"], - "bk_host_id": host["bk_host_id"], - } - for host in Host.objects.filter(inner_ip__in=ips).values( - "inner_ip", "outer_ip", "login_ip", "bk_cloud_id", "bk_biz_id", "node_type", "bk_host_id" - ) + f"{host['bk_cloud_id']}-{host[inner_ip_field_name]}": host + for host in Host.objects.filter(bk_addressing=bk_addressing, **{inner_ip_filter_k: ips}).values(*fields) } outer_ip_info = { - f"{host['bk_cloud_id']}-{host['outer_ip']}": { - "inner_ip": host["inner_ip"], - "login_ip": host["login_ip"], - "bk_cloud_id": host["bk_cloud_id"], - "bk_biz_id": host["bk_biz_id"], - "node_type": host["node_type"], - "bk_host_id": host["bk_host_id"], - } - for host in Host.objects.filter(outer_ip__in=ips).values( - "inner_ip", "outer_ip", "login_ip", "bk_cloud_id", "bk_biz_id", "node_type", "bk_host_id" - ) + f"{host['bk_cloud_id']}-{host[outer_ip_field_name]}": host + for host in Host.objects.filter(bk_addressing=bk_addressing, **{outer_ip_filter_k: ips}).values(*fields) } login_ip_info = { - f"{host['bk_cloud_id']}-{host['login_ip']}": { - "inner_ip": host["inner_ip"], - "outer_ip": host["outer_ip"], - "bk_cloud_id": host["bk_cloud_id"], - "bk_biz_id": host["bk_biz_id"], - "node_type": host["node_type"], - "bk_host_id": host["bk_host_id"], - } - for host in Host.objects.filter(login_ip__in=ips).values( - "inner_ip", "outer_ip", "login_ip", "bk_cloud_id", "bk_biz_id", "node_type", "bk_host_id" - ) + f"{host['bk_cloud_id']}-{host[login_ip_field_name]}": host + for host in Host.objects.filter(bk_addressing=bk_addressing, **{login_ip_filter_k: ips}).values(*fields) } exists_ip_info = {} diff --git a/apps/node_man/handlers/job.py b/apps/node_man/handlers/job.py index c6c0b6a6b..8f5cc85cd 100644 --- a/apps/node_man/handlers/job.py +++ b/apps/node_man/handlers/job.py @@ -11,7 +11,7 @@ import copy import logging import re -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Set from django.conf import settings from django.core.paginator import Paginator @@ -59,23 +59,6 @@ def ugettext_to_unicode(self, ip_filter_list: list): filter_info["msg"] = str(filter_info["msg"]) return ip_filter_list - def task_status_list(self): - """ - 返回任务执行的状态 - :return: 以Host ID为键，返回任务执行状态 - { - bk_host_id: { - 'job_id': job_id, - 'status': status - } - } - """ - task_info = { - task["bk_host_id"]: {"status": task["status"]} - for task in models.JobTask.objects.values("bk_host_id", "instance_id", "job_id", "status") - } - return task_info - def check_ap_and_biz_scope(self, node_type: str, host: dict, cloud_info: dict): """ 返回主机的接入点、业务范围、节点类型。 @@ -361,7 +344,13 @@ def list(self, params: dict, username: str): return {"total": len(job_list), "list": job_page} def install( - self, hosts: List, op_type: str, node_type: str, job_type: str, ticket: str, extra_params: Dict[str, Any] + self, + hosts: List[Dict[str, Any]], + op_type: str, + node_type: str, + job_type: str, + ticket: str, + extra_params: Dict[str, Any], ): """ Job 任务处理器 @@ -374,31 +363,36 @@ def install( """ # 获取Hosts中的cloud_id列表、ap_id列表、内网、外网、登录IP列表、bk_biz_scope列表 - bk_cloud_ids = set() ap_ids = set() - bk_biz_scope = set() - inner_ips = set() is_manual = set() + bk_biz_scope = set() + bk_cloud_ids = set() + static_inner_ips: Dict[str, Set[str]] = {"inner_ips": set(), "inner_ipv6s": set()} for host in hosts: bk_cloud_ids.add(host["bk_cloud_id"]) bk_biz_scope.add(host["bk_biz_id"]) - inner_ips.add(host["inner_ip"]) is_manual.add(host["is_manual"]) + + if host["bk_addressing"] == constants.CmdbAddressingType.STATIC.value: + # 遍历需要支持的 IP 字段，汇总安装信息中存在该 IP 字段的值 + if host.get("inner_ip"): + static_inner_ips["inner_ips"].add(host["inner_ip"]) + if host.get("inner_ipv6"): + static_inner_ips["inner_ipv6s"].add(host["inner_ipv6"]) + # 用户ticket，用于后台异步执行时调用第三方接口使用 host["ticket"] = ticket if host.get("ap_id"): ap_ids.add(host["ap_id"]) # 如果混合了【手动安装】，【自动安装】则不允许通过 - # 此处暂不和入job validator. + # 此处暂不合入 job validator. if len(is_manual) > 1: raise exceptions.MixedOperationError else: is_manual = list(is_manual)[0] - bk_biz_scope = list(bk_biz_scope) - # 获得所有的业务列表 # 格式 { bk_biz_id: bk_biz_name , ...} biz_info = CmdbHandler().biz_id_name_without_permission() @@ -412,23 +406,22 @@ def install( # 获得用户输入的ip是否存在于数据库中 # 格式 { bk_cloud_id+ip: { 'bk_host_id': ..., 'bk_biz_id': ..., 'node_type': ...}} - inner_ip_info = HostHandler().ip_list(inner_ips) - - # 获得正在执行的任务状态 - task_info = self.task_status_list() + inner_ip_info = HostHandler().ip_list( + ips=static_inner_ips["inner_ips"], + ip_version=constants.CmdbIpVersion.V4.value, + bk_addressing=constants.CmdbAddressingType.STATIC.value, + ) + inner_ipv6_info = HostHandler().ip_list( + ips=static_inner_ips["inner_ipv6s"], + ip_version=constants.CmdbIpVersion.V6.value, + bk_addressing=constants.CmdbAddressingType.STATIC.value, + ) + inner_ip_info.update(inner_ipv6_info) # 对数据进行校验 # 重装则校验IP是否存在，存在才可重装 ip_filter_list, accept_list, proxy_not_alive = validator.install_validate( - hosts, - op_type, - node_type, - job_type, - biz_info, - cloud_info, - ap_id_name, - inner_ip_info, - task_info, + hosts, op_type, node_type, job_type, biz_info, cloud_info, ap_id_name, inner_ip_info ) if proxy_not_alive: @@ -440,7 +433,13 @@ def install( # 如果都被过滤了 raise exceptions.AllIpFiltered(data={"job_id": "", "ip_filter": self.ugettext_to_unicode(ip_filter_list)}) - if op_type in [constants.OpType.INSTALL, constants.OpType.REPLACE, constants.OpType.RELOAD]: + if any( + [ + op_type in [constants.OpType.INSTALL, constants.OpType.REPLACE, constants.OpType.RELOAD], + # 开启动态主机配置协议适配时，通过基础信息进行重装 + settings.BKAPP_ENABLE_DHCP and op_type in [constants.OpType.REINSTALL], + ] + ): # 安装、替换Proxy操作 subscription_nodes = self.subscription_install(accept_list, node_type, cloud_info, biz_info) subscription = self.create_subscription(job_type, subscription_nodes, extra_params=extra_params) @@ -488,10 +487,6 @@ def subscription_install(self, accept_list: list, node_type: str, cloud_info: di subscription_nodes = [] rsa_util = tools.HostTools.get_rsa_util() for host in accept_list: - inner_ip = host["inner_ip"] - outer_ip = host.get("outer_ip", "") - login_ip = host.get("login_ip", "") - host_ap_id, host_node_type = self.check_ap_and_biz_scope(node_type, host, cloud_info) instance_info = copy.deepcopy(host) instance_info.update( @@ -500,19 +495,17 @@ def subscription_install(self, accept_list: list, node_type: str, cloud_info: di "ap_id": host_ap_id, "install_channel_id": host.get("install_channel_id"), "bk_os_type": constants.BK_OS_TYPE[host["os_type"]], - "bk_host_innerip": inner_ip, - "bk_host_outerip": outer_ip, - "login_ip": login_ip, + "bk_host_innerip": host.get("inner_ip", ""), + "bk_host_innerip_v6": host.get("inner_ipv6", ""), + "bk_host_outerip": host.get("outer_ip", ""), + "bk_host_outerip_v6": host.get("outer_ipv6", ""), + "login_ip": host.get("login_ip", ""), "username": get_request_username(), "bk_biz_id": host["bk_biz_id"], "bk_biz_name": biz_info.get(host["bk_biz_id"]), "bk_cloud_id": host["bk_cloud_id"], "bk_cloud_name": str(cloud_info.get(host["bk_cloud_id"], {}).get("bk_cloud_name")), - # 开启动态主机配置协议适配后，增量主机走动态IP方案 - "bk_addressing": ( - constants.CmdbAddressingType.STATIC.value, - constants.CmdbAddressingType.DYNAMIC.value, - )[settings.BKAPP_ENABLE_DHCP], + "bk_addressing": host["bk_addressing"], "bk_supplier_account": settings.DEFAULT_SUPPLIER_ACCOUNT, "host_node_type": host_node_type, "os_type": host["os_type"], @@ -543,7 +536,7 @@ def subscription_install(self, accept_list: list, node_type: str, cloud_info: di { "bk_supplier_account": settings.DEFAULT_SUPPLIER_ACCOUNT, "bk_cloud_id": host["bk_cloud_id"], - "ip": inner_ip, + "ip": host.get("inner_ip", "") or host.get("inner_ipv6", ""), "instance_info": instance_info, } ) diff --git a/apps/node_man/handlers/meta.py b/apps/node_man/handlers/meta.py index b4945c8c2..a69eb98c1 100644 --- a/apps/node_man/handlers/meta.py +++ b/apps/node_man/handlers/meta.py @@ -162,6 +162,10 @@ def fetch_host_condition(self): {"name": install_channel["name"], "id": install_channel["id"]} for install_channel in InstallChannelHandler.list() ] + bk_addressing_children = [ + {"name": alias, "id": val} + for val, alias in constants.CmdbAddressingType.get_member_value__alias_map().items() + ] return self.filter_empty_children( [ {"name": _("操作系统"), "id": "os_type", "children": os_types_children + [{"name": _("其它"), "id": "none"}]}, @@ -169,6 +173,7 @@ def fetch_host_condition(self): {"name": _("安装方式"), "id": "is_manual", "children": is_manual_children}, {"name": _("Agent版本"), "id": "version", "children": versions_children}, {"name": _("云区域"), "id": "bk_cloud_id", "children": bk_cloud_ids_children}, + {"name": _("寻址方式"), "id": "bk_addressing", "children": bk_addressing_children}, {"name": _("安装通道"), "id": "install_channel_id", "children": install_channel_children}, {"name": _("IP"), "id": "inner_ip"}, ] diff --git a/apps/node_man/handlers/validator.py b/apps/node_man/handlers/validator.py index 47a690558..61d389e4e 100644 --- a/apps/node_man/handlers/validator.py +++ b/apps/node_man/handlers/validator.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from typing import List +import typing from django.db.models.aggregates import Count from django.utils.translation import ugettext_lazy as _ @@ -258,8 +258,92 @@ def update_pwd_validate(accept_list: list, identity_info: dict, ip_filter_list: return not_modified_host, modified_host, ip_filter_list +def new_install_ip_checker( + inner_ip_info: typing.Dict[str, typing.Dict[str, typing.Any]], + error_host: typing.Dict[str, typing.Dict], + ip: str, + biz_info: typing.Dict[str, typing.Any], + bk_cloud_name: str, + bk_cloud_id: int, + bk_biz_id: int, + ip_version: int, +) -> bool: + cloud_inner_ip: str = f"{bk_cloud_id}-{ip}" + ip_field_name: str = ("inner_ip", "inner_ipv6")[ip_version == const.CmdbIpVersion.V6.value] + + # ipv4 & ipv6 支持仅填一个，这种场景下 ip 为空视为校验成功，由非空 ip 决定校验最终结果 + if not ip: + return True + + # 内网主机信息不存在，符合新装场景 + if cloud_inner_ip not in inner_ip_info: + return True + + # 业务 / 内网IP / 云区域 一致时，视为重装，而不是占用 + if all( + [ + inner_ip_info[cloud_inner_ip]["bk_biz_id"] == bk_biz_id, + inner_ip_info[cloud_inner_ip].get(ip_field_name) == ip, + ] + ): + return True + + # 已被占用则跳过并记录 + error_host["msg"] = _( + """ + 该主机内网IP已存在于所选云区域：{cloud_name} 下, + 业务：{bk_biz_id}, + 节点类型：{node_type} + """ + ).format( + cloud_name=bk_cloud_name, + bk_biz_id=biz_info.get(inner_ip_info[cloud_inner_ip]["bk_biz_id"], inner_ip_info[cloud_inner_ip]["bk_biz_id"]), + node_type=inner_ip_info[cloud_inner_ip]["node_type"], + ) + + return False + + +def operate_ip_checker( + inner_ip_info: typing.Dict[str, typing.Dict[str, typing.Any]], + host: typing.Dict[str, typing.Dict], + error_host: typing.Dict[str, typing.Dict], + ip: str, + op_type: str, + node_type: str, + bk_cloud_id: int, +): + cloud_inner_ip: str = f"{bk_cloud_id}-{ip}" + + # ipv4 & ipv6 支持仅填一个，这种场景下 ip 为空视为校验成功，由非空 ip 决定校验最终结果 + if not ip: + return True + + # 检查：除安装操作外，其他操作时内网IP是否存在 + if cloud_inner_ip not in inner_ip_info: + error_host["msg"] = _("尚未被安装，无法执行 {op_type} 操作").format(op_type=const.JOB_TYPE_DICT[op_type + "_" + node_type]) + return False + + # 检查：检查 bk_host_id 是否准确 + if host["bk_host_id"] != inner_ip_info[cloud_inner_ip]["bk_host_id"]: + error_host["msg"] = _("Host ID 不正确，无法执行 {op_type} 操作").format( + op_type=const.JOB_TYPE_DICT[op_type + "_" + node_type] + ) + return False + + # 检查：节点类型是否与操作类型一致, 如本身为PROXY，重装却为AGENT + # 此处不区分P-AGENT和AGENT + if node_type not in inner_ip_info[cloud_inner_ip]["node_type"]: + error_host["msg"] = _("节点类型不正确，该主机是 {host_node_type}, 而请求的操作类型是 {node_type}").format( + host_node_type=inner_ip_info[cloud_inner_ip]["node_type"], node_type=node_type + ) + return False + + return True + + def install_validate( - hosts: List, + hosts: typing.List[typing.Dict[str, typing.Any]], op_type: str, node_type: str, job_type: str, @@ -267,7 +351,6 @@ def install_validate( cloud_info: dict, ap_id_name: dict, inner_ip_info: dict, - task_info: dict, ): """ 用于job任务的校验 @@ -279,7 +362,6 @@ def install_validate( :param cloud_info: 获得相应云区域 id, name, ap_id, bk_biz_scope :param ap_id_name: 获得接入点列表 :param inner_ip_info: DB中内网IP信息 - :param task_info: 任务执行信息 :return: 列表，ip被占用及其原因 """ ip_filter_list = [] @@ -290,7 +372,8 @@ def install_validate( available_clouds, proxies_count = check_available_proxy() for host in hosts: - ip = host["inner_ip"] + ip = host.get("inner_ip") + ipv6 = host.get("inner_ipv6") bk_cloud_id = host["bk_cloud_id"] bk_biz_id = host["bk_biz_id"] ap_id = host.get("ap_id") @@ -343,75 +426,69 @@ def install_validate( if ap_id != const.DEFAULT_AP_ID and ap_id not in ap_id_name: raise ApIDNotExistsError(_("接入点(id:{ap_id})不存在").format(ap_id=ap_id)) - # 检查：判断内网ip是否已被占用 - cloud_inner_ip = str(bk_cloud_id) + "-" + ip - if ( - op_type in [const.OpType.INSTALL, const.OpType.REPLACE] - and cloud_inner_ip in inner_ip_info - and job_type != const.JobType.INSTALL_PROXY # 这里允许把Agent安装为proxy，因此不做IP冲突检查 - ): - # 业务 / 内网IP / 云区域 一致时，视为重装，而不是占用 - if all( - [ - inner_ip_info[cloud_inner_ip]["bk_biz_id"] == bk_biz_id, - inner_ip_info[cloud_inner_ip]["inner_ip"] == ip, - ] + # # 允许 Agent 重装为 Proxy，无需进行校验o + if op_type in [const.OpType.INSTALL, const.OpType.REPLACE] and job_type != const.JobType.INSTALL_PROXY: + if new_install_ip_checker( + inner_ip_info=inner_ip_info, + error_host=error_host, + ip=ip, + biz_info=biz_info, + bk_cloud_name=cloud_name, + bk_cloud_id=bk_cloud_id, + bk_biz_id=bk_biz_id, + ip_version=const.CmdbIpVersion.V4.value, + ) and new_install_ip_checker( + inner_ip_info=inner_ip_info, + error_host=error_host, + ip=ipv6, + biz_info=biz_info, + bk_cloud_name=cloud_name, + bk_cloud_id=bk_cloud_id, + bk_biz_id=bk_biz_id, + ip_version=const.CmdbIpVersion.V6.value, ): + inner_ipv4_info: typing.Dict[str, typing.Any] = inner_ip_info.get(f"{bk_cloud_id}-{ip}") + # 如果 ipv4 / ipv6 同时存在，也必须是绑定关系 + if inner_ipv4_info and ipv6 and inner_ipv4_info["inner_ipv6"] and inner_ipv4_info["inner_ipv6"] != ipv6: + error_host["msg"] = _( + "该主机(bk_host_id:{bk_host_id}) 已存在且 IP 信息为：IPv4({ipv4}), IPv6({ipv6})，" + "不允许修改为 IPv4({ipv4}), IPv6({to_be_add_ipv6})" + ).format( + bk_host_id=inner_ipv4_info["bk_host_id"], + ipv4=ip, + ipv6=inner_ipv4_info["inner_ipv6"], + to_be_add_ipv6=ipv6, + ) + ip_filter_list.append(error_host) + continue accept_list.append(dict(host)) continue - # 已被占用则跳过并记录 - error_host["msg"] = _( - """ - 该主机内网IP已存在于所选云区域：{cloud_name} 下, - 业务：{bk_biz_id}, - 节点类型：{node_type} - """ - ).format( - cloud_name=cloud_name, - bk_biz_id=biz_info.get( - inner_ip_info[cloud_inner_ip]["bk_biz_id"], inner_ip_info[cloud_inner_ip]["bk_biz_id"] - ), - node_type=inner_ip_info[cloud_inner_ip]["node_type"], - ) - ip_filter_list.append(error_host) - continue - - # 非新装任务校验 - if op_type not in [const.OpType.INSTALL, const.OpType.REPLACE]: - - # 检查：除安装操作外，其他操作时内网IP是否存在 - if cloud_inner_ip not in inner_ip_info: - error_host["msg"] = _("尚未被安装，无法执行 {op_type} 操作").format( - op_type=const.JOB_TYPE_DICT[op_type + "_" + node_type] - ) + else: ip_filter_list.append(error_host) continue - # 检查：除安装操作外，其他操作时检测Host ID是否正确 - if host["bk_host_id"] != inner_ip_info[cloud_inner_ip]["bk_host_id"]: - error_host["msg"] = _("Host ID 不正确，无法执行 {op_type} 操作").format( - op_type=const.JOB_TYPE_DICT[op_type + "_" + node_type] + # 非新装任务校验 + if op_type not in [const.OpType.INSTALL, const.OpType.REPLACE]: + if not ( + operate_ip_checker( + inner_ip_info=inner_ip_info, + host=host, + error_host=error_host, + ip=ip, + op_type=op_type, + node_type=node_type, + bk_cloud_id=bk_cloud_id, ) - ip_filter_list.append(error_host) - continue - - # 检查：除安装操作外，是否该Host正在执行任务 - if task_info.get(host["bk_host_id"], {}).get("status") in [ - "RUNNING", - "PENDING", - ]: - error_host["msg"] = _("正在执行其他任务，无法执行新任务") - error_host["exception"] = "is_running" - error_host["job_id"] = task_info.get(host["bk_host_id"], {}).get("job_id") - ip_filter_list.append(error_host) - continue - - # 检查：节点类型是否与操作类型一致, 如本身为PROXY，重装却为AGENT - # 此处不区分P-AGENT和AGENT - if node_type not in inner_ip_info[cloud_inner_ip]["node_type"]: - error_host["msg"] = _("节点类型不正确，该主机是 {host_node_type}, 而请求的操作类型是 {node_type}").format( - host_node_type=inner_ip_info[cloud_inner_ip]["node_type"], node_type=node_type + and operate_ip_checker( + inner_ip_info=inner_ip_info, + host=host, + error_host=error_host, + ip=ipv6, + op_type=op_type, + node_type=node_type, + bk_cloud_id=bk_cloud_id, ) + ): ip_filter_list.append(error_host) continue diff --git a/apps/node_man/serializers/host.py b/apps/node_man/serializers/host.py index d31c50c26..163cf9214 100644 --- a/apps/node_man/serializers/host.py +++ b/apps/node_man/serializers/host.py @@ -13,6 +13,7 @@ from apps.exceptions import ValidationError from apps.node_man import constants, tools +from apps.utils import basic class HostSerializer(serializers.Serializer): @@ -40,10 +41,13 @@ class BizProxySerializer(serializers.Serializer): class HostUpdateSerializer(serializers.Serializer): bk_host_id = serializers.IntegerField(label=_("主机ID")) bk_cloud_id = serializers.IntegerField(label=_("云区域ID"), required=False) - inner_ip = serializers.IPAddressField(label=_("内网IP"), required=False) - outer_ip = serializers.IPAddressField(label=_("外网IP"), required=False) - login_ip = serializers.IPAddressField(label=_("登录IP"), required=False) - data_ip = serializers.IPAddressField(label=_("数据IP"), required=False) + inner_ip = serializers.IPAddressField(label=_("内网IP"), required=False, protocol="ipv4") + inner_ipv6 = serializers.IPAddressField(label=_("内网IPv6"), required=False, protocol="ipv6") + outer_ip = serializers.IPAddressField(label=_("外网IP"), required=False, protocol="ipv4") + outer_ipv6 = serializers.IPAddressField(label=_("外网IPv6"), required=False, protocol="ipv6") + # 登录 IP & 数据 IP 支持多 IP 协议 + login_ip = serializers.IPAddressField(label=_("登录IP"), required=False, protocol="both") + data_ip = serializers.IPAddressField(label=_("数据IP"), required=False, protocol="both") account = serializers.CharField(label=_("账号"), required=False) port = serializers.IntegerField(label=_("端口号"), required=False) ap_id = serializers.IntegerField(label=_("接入点ID"), required=False) @@ -63,6 +67,7 @@ def validate(self, attrs): attrs[field_need_encrypt] = tools.HostTools.decrypt_with_friendly_exc_handle( rsa_util=rsa_util, encrypt_message=attrs[field_need_encrypt], raise_exec=ValidationError ) + basic.ipv6_formatter(data=attrs, ipv6_field_names=["inner_ipv6", "outer_ipv6", "login_ip", "data_ip"]) return attrs diff --git a/apps/node_man/serializers/job.py b/apps/node_man/serializers/job.py index 9f15ac549..5f8134414 100644 --- a/apps/node_man/serializers/job.py +++ b/apps/node_man/serializers/job.py @@ -20,6 +20,7 @@ from apps.node_man.handlers.cmdb import CmdbHandler from apps.node_man.handlers.host import HostHandler from apps.node_man.periodic_tasks.sync_cmdb_host import bulk_differential_sync_biz_hosts +from apps.utils import basic class SortSerializer(serializers.Serializer): @@ -50,14 +51,21 @@ class HostSerializer(serializers.Serializer): bk_biz_id = serializers.IntegerField(label=_("业务ID")) bk_cloud_id = serializers.IntegerField(label=_("云区域ID")) bk_host_id = serializers.IntegerField(label=_("主机ID"), required=False) + bk_addressing = serializers.ChoiceField( + label=_("寻址方式"), + choices=constants.CmdbAddressingType.list_member_values(), + required=False, + default=constants.CmdbAddressingType.STATIC.value, + ) ap_id = serializers.IntegerField(label=_("接入点ID"), required=False) install_channel_id = serializers.IntegerField(label=_("安装通道ID"), required=False, allow_null=True) - inner_ip = serializers.IPAddressField(label=_("内网IP"), required=False, allow_blank=True) - outer_ip = serializers.IPAddressField(label=_("外网IP"), required=False, allow_blank=True) - login_ip = serializers.IPAddressField(label=_("登录IP"), required=False, allow_blank=True) - data_ip = serializers.IPAddressField(label=_("数据IP"), required=False, allow_blank=True) - inner_ipv6 = serializers.IPAddressField(label=_("内网IPv6"), required=False, allow_blank=True) - outer_ipv6 = serializers.IPAddressField(label=_("外网IPv6"), required=False, allow_blank=True) + inner_ip = serializers.IPAddressField(label=_("内网IP"), required=False, allow_blank=True, protocol="ipv4") + outer_ip = serializers.IPAddressField(label=_("外网IP"), required=False, allow_blank=True, protocol="ipv4") + login_ip = serializers.IPAddressField(label=_("登录IP"), required=False, allow_blank=True, protocol="both") + data_ip = serializers.IPAddressField(label=_("数据IP"), required=False, allow_blank=True, protocol="both") + inner_ipv6 = serializers.IPAddressField(label=_("内网IPv6"), required=False, allow_blank=True, protocol="ipv6") + outer_ipv6 = serializers.IPAddressField(label=_("外网IPv6"), required=False, allow_blank=True, protocol="ipv6") + os_type = serializers.ChoiceField(label=_("操作系统"), choices=list(constants.OS_TUPLE)) auth_type = serializers.ChoiceField(label=_("认证类型"), choices=list(constants.AUTH_TUPLE), required=False) account = serializers.CharField(label=_("账户"), required=False, allow_blank=True) @@ -81,6 +89,8 @@ def validate(self, attrs): if node_type == constants.NodeType.PROXY and not (attrs.get("outer_ip") or attrs.get("outer_ipv6")): raise ValidationError(_("Proxy 操作的请求参数 outer_ip 和 outer_ipv6 不能同时为空")) + basic.ipv6_formatter(data=attrs, ipv6_field_names=["inner_ipv6", "outer_ipv6", "login_ip", "data_ip"]) + op_not_need_identity = [ constants.OpType.REINSTALL, constants.OpType.RESTART, diff --git a/apps/node_man/tests/test_handlers/test_host.py b/apps/node_man/tests/test_handlers/test_host.py index d2200f30d..b14d149aa 100644 --- a/apps/node_man/tests/test_handlers/test_host.py +++ b/apps/node_man/tests/test_handlers/test_host.py @@ -352,6 +352,6 @@ def test_ip_list(self): host_to_create, identity_to_create, _ = create_host(number) # 测试 - result = HostHandler().ip_list([host.inner_ip for host in host_to_create]) + result = HostHandler().ip_list([host.inner_ip for host in host_to_create], const.CmdbIpVersion.V4.value) self.assertEqual(len(result), number) diff --git a/apps/node_man/tests/test_handlers/test_validator.py b/apps/node_man/tests/test_handlers/test_validator.py index 5d5ad4d10..b50869a4c 100644 --- a/apps/node_man/tests/test_handlers/test_validator.py +++ b/apps/node_man/tests/test_handlers/test_validator.py @@ -39,9 +39,7 @@ class TestValidator(TestCase): @staticmethod - def wrap_job_validate(data, task_info=None): - if task_info is None: - task_info = {} + def wrap_job_validate(data): # 封装数据生成和job_validate，减少重复代码 ( biz_info, @@ -63,18 +61,16 @@ def wrap_job_validate(data, task_info=None): cloud_info, ap_id_name, all_inner_ip_info, - task_info, ) return ip_filter_list, accept_list - def wrap_job_validate_raise(self, data, username, is_superuser, error, task_info=None): + def wrap_job_validate_raise(self, data, username, is_superuser, error): """ 封装job_validate测试异常抛出通用逻辑 :param data: :param username: :param is_superuser: :param error: - :param task_info: :return: data: { @@ -100,8 +96,6 @@ def wrap_job_validate_raise(self, data, username, is_superuser, error, task_info ] } """ - if task_info is None: - task_info = {} ( biz_info, data, @@ -124,7 +118,6 @@ def wrap_job_validate_raise(self, data, username, is_superuser, error, task_info cloud_info, ap_id_name, all_inner_ip_info, - task_info, ) @staticmethod @@ -216,7 +209,6 @@ def _test_job_validate_cloud_not_exists(self): cloud_info, ap_id_name, all_inner_ip_info, - {}, ) def _test_job_validate_proxy_not_available(self): @@ -249,7 +241,6 @@ def _test_job_validate_proxy_not_alive(self): cloud_info, ap_id_name, all_inner_ip_info, - {}, ) self.assertEqual(len(proxy_not_alive), 1) @@ -328,22 +319,6 @@ def _test_job_validate_inner_ip_had_exists_other_op(self): ip_filter_list, accept_list = self.wrap_job_validate(data) self.assertEquals(number, len(ip_filter_list)) - # 除安装操作外，该Host是否正在执行任务 - number = 1 - data = gen_job_data( - job_type=const.JobType.RESTART_AGENT, - count=number, - bk_cloud_id=1, - ap_id=const.DEFAULT_AP_ID, - ip=inner_ip, - host_to_create=host_to_create, - identity_to_create=identity_to_create, - ) - - ip_filter_list, accept_list = self.wrap_job_validate(data, task_info={bk_host_id: {"status": "RUNNING"}}) - - self.assertEquals(number, len(ip_filter_list)) - def _test_job_validate_install_type_not_consistent(self): number = 1 host_to_create, _, identity_to_create = create_host( diff --git a/apps/node_man/tests/utils.py b/apps/node_man/tests/utils.py index dc0c76e21..c89cd1326 100644 --- a/apps/node_man/tests/utils.py +++ b/apps/node_man/tests/utils.py @@ -767,6 +767,7 @@ def gen_install_accept_list(count, nodetype, ip=None, bk_cloud_id=None, ticket=N "bk_cloud_id": bk_cloud_id or 97 if nodetype == "PROXY" or nodetype == "PAGENT" else 0, "ap_id": [-1, 1][random.randint(0, 1)], "bk_biz_id": random.randint(1, 10), + "bk_addressing": constants.CmdbAddressingType.STATIC.value, "os_type": "LINUX", "inner_ip": ip or f"{random.randint(1, 255)}.{random.randint(1, 255)}." @@ -841,6 +842,7 @@ def gen_job_data( "bk_cloud_id": bk_cloud_id, "ap_id": ap_id or [-1, 1][random.randint(0, 1)], "bk_biz_id": bk_biz_id or random.randint(27, 39), + "bk_addressing": constants.CmdbAddressingType.STATIC.value, "os_type": "LINUX", "inner_ip": ip or f"{random.randint(1, 255)}.{random.randint(1, 255)}." f"{random.randint(1, 254)}.1", @@ -865,6 +867,7 @@ def gen_job_data( for i in range(count): accept_list["hosts"].append( { + "bk_addressing": constants.CmdbAddressingType.STATIC.value, "bk_host_id": bk_host_id or host_to_create[i].bk_host_id, "bk_cloud_id": bk_cloud_id, "ap_id": ap_id or [-1, 1][random.randint(0, 1)], @@ -1181,8 +1184,8 @@ def ret_to_validate_data(data): # 获得请求里所有在数据库中的IP的相关信息 # 格式 { inner_ip: {'bk_biz_id': bk_biz_id, 'node_type': node_type, 'bk_cloud_id': bk_cloud_id}, ...} - inner_ip_info = HostHandler().ip_list(inner_ips) - outer_ip_info = HostHandler().ip_list(outer_ips) - login_ip_info = HostHandler().ip_list(login_ips) + inner_ip_info = HostHandler().ip_list(inner_ips, constants.CmdbIpVersion.V4.value) + outer_ip_info = HostHandler().ip_list(outer_ips, constants.CmdbIpVersion.V4.value) + login_ip_info = HostHandler().ip_list(login_ips, constants.CmdbIpVersion.V4.value) return biz_info, data, cloud_info, ap_id_name, inner_ip_info, outer_ip_info, login_ip_info, bk_biz_scope diff --git a/apps/utils/basic.py b/apps/utils/basic.py index 1cd0c06e7..6f53f0484 100644 --- a/apps/utils/basic.py +++ b/apps/utils/basic.py @@ -8,9 +8,10 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import ipaddress from collections import Counter, namedtuple from copy import deepcopy -from typing import Any, Dict, Iterable, List, Set, Union +from typing import Any, Dict, Iterable, List, Optional, Set, Union def tuple_choices(tupl): @@ -178,3 +179,14 @@ def get_chr_seq(begin_chr: str, end_chr: str) -> List[str]: :return: """ return [chr(ascii_int) for ascii_int in range(ord(begin_chr), ord(end_chr) + 1)] + + +def ipv6_formatter(data: Dict[str, Any], ipv6_field_names: List[str]): + for ipv6_field_name in ipv6_field_names: + ipv6_val: Optional[str] = data.get(ipv6_field_name) + if not ipv6_val: + continue + ip_address = ipaddress.ip_address(ipv6_val) + # 将 v6 转为标准格式 + if ip_address.version == 6: + data[ipv6_field_name] = ip_address.exploded diff --git a/common/context_processors.py b/common/context_processors.py index 74c2f9479..73f74a43d 100644 --- a/common/context_processors.py +++ b/common/context_processors.py @@ -89,6 +89,7 @@ def mysetting(request): "USE_TJJ": getattr(settings, "USE_TJJ", False), "BK_DOCS_CENTER_URL": get_docs_center_url(), "BKAPP_RUN_ENV": settings.BKAPP_RUN_ENV, + "BKAPP_ENABLE_DHCP": settings.BKAPP_ENABLE_DHCP, # TAM前端监控 "TAM_ID": os.getenv("BKAPP_TAM_ID"), "TAM_URL": os.getenv("BKAPP_TAM_URL"), diff --git a/frontend/index.html b/frontend/index.html index 048ea0ea1..c440eda77 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -35,6 +35,7 @@ const USE_TJJ = '{{ USE_TJJ }}' const BK_DOCS_CENTER_URL = '{{ BK_DOCS_CENTER_URL }}' const BKAPP_RUN_ENV = '{{ BKAPP_RUN_ENV }}' + const BKAPP_ENABLE_DHCP = '{{ BKAPP_ENABLE_DHCP }}' const TAM_ID = '{{ TAM_ID }}' const LOGIN_URL = '{{ LOGIN_URL }}' const VERSION = '{{ VERSION }}' @@ -64,6 +65,7 @@ USE_TJJ, BK_DOCS_CENTER_URL, BKAPP_RUN_ENV, + BKAPP_ENABLE_DHCP, TAM_ID, LOGIN_URL, VERSION diff --git a/static/index.html b/static/index.html index 07a09dd87..a05cf928f 100644 --- a/static/index.html +++ b/static/index.html @@ -23,6 +23,7 @@ const USE_TJJ = '{{ USE_TJJ }}' const BK_DOCS_CENTER_URL = '{{ BK_DOCS_CENTER_URL }}' const BKAPP_RUN_ENV = '{{ BKAPP_RUN_ENV }}' + const BKAPP_ENABLE_DHCP = '{{ BKAPP_ENABLE_DHCP }}' const TAM_ID = '{{ TAM_ID }}' const LOGIN_URL = '{{ LOGIN_URL }}' const VERSION = '{{ VERSION }}' @@ -52,6 +53,7 @@ USE_TJJ, BK_DOCS_CENTER_URL, BKAPP_RUN_ENV, + BKAPP_ENABLE_DHCP, TAM_ID, LOGIN_URL, VERSION @@ -70,4 +72,4 @@ restfulApiList: [], // 当开启了接口测速，且项目中有些接口采用了 restful 规范，需要在该配置中列出，帮助 Aegis 识别哪些接口是同一条接 spa: true }) - }
\ No newline at end of file + }
\ No newline at end of file