From 8599e69b56fcc4c6f8ef63ec72e956fb4c0cf29d Mon Sep 17 00:00:00 2001 From: OrangeBao Date: Thu, 7 Dec 2023 19:19:39 +0800 Subject: [PATCH] feat: auto detect ip Signed-off-by: OrangeBao --- deploy/clusterlink-agent.yaml | 1 + deploy/crds/kosmos.io_clusters.yaml | 2 + hack/local-up-clusterlink.sh | 6 +- pkg/apis/kosmos/v1alpha1/cluster_types.go | 3 + .../agent-manager/auto_detect_controller.go | 103 ++++++++++++++++-- .../agent-manager/autodetection/reachaddr.go | 69 ++++++++++++ .../controllers/node/node_controller.go | 15 +-- pkg/operator/clusterlink/agent/manifests.go | 1 + 8 files changed, 173 insertions(+), 27 deletions(-) create mode 100644 pkg/clusterlink/agent-manager/autodetection/reachaddr.go diff --git a/deploy/clusterlink-agent.yaml b/deploy/clusterlink-agent.yaml index cde3bb24e..6ab502061 100644 --- a/deploy/clusterlink-agent.yaml +++ b/deploy/clusterlink-agent.yaml @@ -28,6 +28,7 @@ spec: command: - clusterlink-agent - -kubeconfig=/etc/clusterlink/kubeconfig + - --v=4 env: - name: CLUSTER_NAME value: "" diff --git a/deploy/crds/kosmos.io_clusters.yaml b/deploy/crds/kosmos.io_clusters.yaml index d96895c13..cbf674914 100644 --- a/deploy/crds/kosmos.io_clusters.yaml +++ b/deploy/crds/kosmos.io_clusters.yaml @@ -43,6 +43,8 @@ spec: properties: clusterLinkOptions: properties: + autodetectionMethod: + type: string bridgeCIDRs: default: ip: 220.0.0.0/8 diff --git a/hack/local-up-clusterlink.sh b/hack/local-up-clusterlink.sh index 1f1a6db79..990636255 100755 --- a/hack/local-up-clusterlink.sh +++ b/hack/local-up-clusterlink.sh @@ -23,9 +23,9 @@ source "$(dirname "${BASH_SOURCE[0]}")/cluster.sh" make images GOOS="linux" --directory="${ROOT}" #cluster cluster -create_cluster $HOST_CLUSTER_NAME $HOST_CLUSTER_POD_CIDR $HOST_CLUSTER_SERVICE_CIDR true -create_cluster $MEMBER1_CLUSTER_NAME $MEMBER1_CLUSTER_POD_CIDR $MEMBER1_CLUSTER_SERVICE_CIDR true -create_cluster $MEMBER2_CLUSTER_NAME $MEMBER2_CLUSTER_POD_CIDR $MEMBER2_CLUSTER_SERVICE_CIDR true +create_cluster $HOST_CLUSTER_NAME $HOST_CLUSTER_POD_CIDR $HOST_CLUSTER_SERVICE_CIDR false +create_cluster $MEMBER1_CLUSTER_NAME $MEMBER1_CLUSTER_POD_CIDR $MEMBER1_CLUSTER_SERVICE_CIDR false +create_cluster $MEMBER2_CLUSTER_NAME $MEMBER2_CLUSTER_POD_CIDR $MEMBER2_CLUSTER_SERVICE_CIDR false #deploy cluster deploy_cluster $HOST_CLUSTER_NAME diff --git a/pkg/apis/kosmos/v1alpha1/cluster_types.go b/pkg/apis/kosmos/v1alpha1/cluster_types.go index 9c1fe5a25..8ba7bf20c 100644 --- a/pkg/apis/kosmos/v1alpha1/cluster_types.go +++ b/pkg/apis/kosmos/v1alpha1/cluster_types.go @@ -91,6 +91,9 @@ type ClusterLinkOptions struct { // +optional GlobalCIDRsMap map[string]string `json:"globalCIDRsMap,omitempty"` + + // +optional + AutodetectionMethod string `json:"autodetectionMethod,omitempty"` } type ClusterTreeOptions struct { diff --git a/pkg/clusterlink/agent-manager/auto_detect_controller.go b/pkg/clusterlink/agent-manager/auto_detect_controller.go index 27124b08a..bc326dbd0 100644 --- a/pkg/clusterlink/agent-manager/auto_detect_controller.go +++ b/pkg/clusterlink/agent-manager/auto_detect_controller.go @@ -3,23 +3,29 @@ package agent import ( "context" "fmt" + "strings" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" kosmosv1alpha1 "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/clusterlink/agent-manager/autodetection" "github.com/kosmos.io/kosmos/pkg/clusterlink/controllers/node" "github.com/kosmos.io/kosmos/pkg/clusterlink/network" + "github.com/kosmos.io/kosmos/pkg/utils" + interfacepolicy "github.com/kosmos.io/kosmos/pkg/utils/interface-policy" ) const ( @@ -27,6 +33,15 @@ const ( AutoDetectRequeueTime = 10 * time.Second ) +const ( + // AUTODETECTION_METHOD_FIRST = "first-found" + AUTODETECTION_METHOD_CAN_REACH = "can-reach=" + // AUTODETECTION_METHOD_INTERFACE = "interface=" + // AUTODETECTION_METHOD_SKIP_INTERFACE = "skip-interface=" + // AUTODETECTION_METHOD_CIDR = "cidr=" + // K8S_INTERNAL_IP = "kubernetes-internal-ip" +) + type AutoDetectReconciler struct { client.Client ClusterName string @@ -49,10 +64,6 @@ func (r *AutoDetectReconciler) SetupWithManager(mgr manager.Manager) error { return false } - if eventObj.Spec.InterfaceName == network.AutoSelectInterfaceFlag || len(eventObj.Spec.InterfaceName) == 0 { - return false - } - return true } @@ -73,9 +84,65 @@ func (r *AutoDetectReconciler) SetupWithManager(mgr manager.Manager) error { return skipEvent(genericEvent.Object) }, })). + Watches(&source.Kind{Type: &kosmosv1alpha1.Cluster{}}, handler.EnqueueRequestsFromMapFunc(r.newClusterMapFunc())). Complete(r) } +func (r *AutoDetectReconciler) newClusterMapFunc() handler.MapFunc { + return func(a client.Object) []reconcile.Request { + var requests []reconcile.Request + cluster := a.(*kosmosv1alpha1.Cluster) + klog.V(4).Infof("auto detect cluster change: %s, currentNode cluster name: %s", cluster.Name, r.ClusterName) + if cluster.Name == r.ClusterName { + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Name: node.ClusterNodeName(r.ClusterName, r.NodeName), + }}) + } + return requests + } +} + +func (r *AutoDetectReconciler) detectInterfaceName(ctx context.Context) (string, error) { + var Cluster kosmosv1alpha1.Cluster + + if err := r.Get(ctx, types.NamespacedName{ + Name: r.ClusterName, + Namespace: "", + }, &Cluster); err != nil { + return "", err + } + + if Cluster.Spec.ClusterLinkOptions != nil { + defaultNICName := interfacepolicy.GetInterfaceName(Cluster.Spec.ClusterLinkOptions.NICNodeNames, r.NodeName, Cluster.Spec.ClusterLinkOptions.DefaultNICName) + + if defaultNICName != network.AutoSelectInterfaceFlag { + return defaultNICName, nil + } + + method := Cluster.Spec.ClusterLinkOptions.AutodetectionMethod + // TODO: set default reachable ip when defaultNICName == * and meth == "" + if method == "" { + method = fmt.Sprintf("%s%s", AUTODETECTION_METHOD_CAN_REACH, "8.8.8.8") + } + if strings.HasPrefix(method, AUTODETECTION_METHOD_CAN_REACH) { + // Autodetect the IP by connecting a UDP socket to a supplied address. + destStr := strings.TrimPrefix(method, AUTODETECTION_METHOD_CAN_REACH) + + version := 4 + if utils.IsIPv6(destStr) { + version = 6 + } + + if i, _, err := autodetection.ReachDestination(destStr, version); err != nil { + return "", err + } else { + return i.Name, nil + } + } + } + return "", fmt.Errorf("can not detect nic") +} + func detectIP(interfaceName string) (string, string) { detectFunc := func(version int) (string, error) { _, n, err := autodetection.FilteredEnumeration([]string{interfaceName}, nil, nil, version) @@ -101,7 +168,9 @@ func detectIP(interfaceName string) (string, string) { } func shouldUpdate(old, new kosmosv1alpha1.ClusterNode) bool { - return old.Spec.IP != new.Spec.IP || old.Spec.IP6 != new.Spec.IP6 + return old.Spec.IP != new.Spec.IP || + old.Spec.IP6 != new.Spec.IP6 || + old.Spec.InterfaceName != new.Spec.InterfaceName } func (r *AutoDetectReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { @@ -112,6 +181,7 @@ func (r *AutoDetectReconciler) Reconcile(ctx context.Context, request reconcile. var clusterNode kosmosv1alpha1.ClusterNode if err := r.Get(ctx, request.NamespacedName, &clusterNode); err != nil { if apierrors.IsNotFound(err) { + klog.V(4).Infof("auto_detect_controller cluster node not found %s", request.NamespacedName) return reconcile.Result{}, nil } klog.Errorf("get clusternode %s error: %v", request.NamespacedName, err) @@ -123,17 +193,28 @@ func (r *AutoDetectReconciler) Reconcile(ctx context.Context, request reconcile. return reconcile.Result{}, nil } - // only do autodetect when clusterNode.Spec.InterfaceName != * and not nil - if clusterNode.Spec.InterfaceName == network.AutoSelectInterfaceFlag || len(clusterNode.Spec.InterfaceName) == 0 { - return reconcile.Result{}, nil + // update clusterNode + newClusterNode := clusterNode.DeepCopy() + + currentInterfaceName, err := r.detectInterfaceName(ctx) + if err != nil { + if apierrors.IsNotFound(err) { + klog.V(4).Infof("cluster is not found, %s", request.NamespacedName) + return reconcile.Result{}, nil + } + klog.Errorf("get cluster %s error: %v", request.NamespacedName, err) + return reconcile.Result{RequeueAfter: AutoDetectRequeueTime}, nil } + // update interface + newClusterNode.Spec.InterfaceName = currentInterfaceName + + klog.V(4).Infof("auto detect interface name: %s", currentInterfaceName) + // detect IP by Name - ipv4, ipv6 := detectIP(clusterNode.Spec.InterfaceName) + ipv4, ipv6 := detectIP(currentInterfaceName) klog.V(4).Infof("auto detect ipv4: %s, ipv6: %s", ipv4, ipv6) - // update clusterNode - newClusterNode := clusterNode.DeepCopy() if ipv4 != "" { newClusterNode.Spec.IP = ipv4 } diff --git a/pkg/clusterlink/agent-manager/autodetection/reachaddr.go b/pkg/clusterlink/agent-manager/autodetection/reachaddr.go new file mode 100644 index 000000000..2936a29cc --- /dev/null +++ b/pkg/clusterlink/agent-manager/autodetection/reachaddr.go @@ -0,0 +1,69 @@ +// For reference: +// https://github.com/projectcalico/calico/blob/master/node/pkg/lifecycle/startup/autodetection/reachaddr.go + +// Copyright (c) 2017 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package autodetection + +import ( + "fmt" + gonet "net" + + "github.com/projectcalico/calico/libcalico-go/lib/net" + log "github.com/sirupsen/logrus" +) + +// ReachDestination auto-detects the interface Network by setting up a UDP +// connection to a "reach" destination. +func ReachDestination(dest string, version int) (*Interface, *net.IPNet, error) { + log.Debugf("Auto-detecting IPv%d CIDR by reaching destination %s", version, dest) + + // Open a UDP connection to determine which external IP address is + // used to access the supplied destination. + protocol := fmt.Sprintf("udp%d", version) + address := fmt.Sprintf("[%s]:80", dest) + conn, err := gonet.Dial(protocol, address) + if err != nil { + return nil, nil, err + } + defer conn.Close() // nolint: errcheck + + // Get the local address as a golang IP and use that to find the matching + // interface CIDR. + addr := conn.LocalAddr() + if addr == nil { + return nil, nil, fmt.Errorf("no address detected by connecting to %s", dest) + } + udpAddr := addr.(*gonet.UDPAddr) + log.WithFields(log.Fields{"IP": udpAddr.IP, "Destination": dest}).Info("Auto-detected address by connecting to remote") + + // Get a full list of interface and IPs and find the CIDR matching the + // found IP. + ifaces, err := GetInterfaces(gonet.Interfaces, nil, nil, version) + if err != nil { + return nil, nil, err + } + for _, iface := range ifaces { + log.WithField("Name", iface.Name).Debug("Checking interface CIDRs") + for _, cidr := range iface.Cidrs { + log.WithField("CIDR", cidr.String()).Debug("Checking CIDR") + if cidr.IP.Equal(udpAddr.IP) { + log.WithField("CIDR", cidr.String()).Debug("Found matching interface CIDR") + return &iface, &cidr, nil + } + } + } + + return nil, nil, fmt.Errorf("autodetected IPv%d address does not match any addresses found on local interfaces: %s", version, udpAddr.IP.String()) +} diff --git a/pkg/clusterlink/controllers/node/node_controller.go b/pkg/clusterlink/controllers/node/node_controller.go index f0a5f8c4f..68a230aed 100644 --- a/pkg/clusterlink/controllers/node/node_controller.go +++ b/pkg/clusterlink/controllers/node/node_controller.go @@ -22,10 +22,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" clusterlinkv1alpha1 "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" - "github.com/kosmos.io/kosmos/pkg/clusterlink/network" "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" "github.com/kosmos.io/kosmos/pkg/utils" - interfacepolicy "github.com/kosmos.io/kosmos/pkg/utils/interface-policy" ) const ( @@ -109,20 +107,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( }, }, } - cluster, err := r.ClusterLinkClient.KosmosV1alpha1().Clusters().Get(ctx, r.ClusterName, metav1.GetOptions{}) - if err != nil { - klog.Errorf("get cluster %s err: %v", r.ClusterName, err) - return reconcile.Result{Requeue: true}, nil - } - err = CreateOrUpdateClusterNode(r.ClusterLinkClient, clusterNode, func(n *clusterlinkv1alpha1.ClusterNode) error { + err := CreateOrUpdateClusterNode(r.ClusterLinkClient, clusterNode, func(n *clusterlinkv1alpha1.ClusterNode) error { n.Spec.NodeName = node.Name n.Spec.ClusterName = r.ClusterName - n.Spec.InterfaceName = interfacepolicy.GetInterfaceName(cluster.Spec.ClusterLinkOptions.NICNodeNames, node.Name, cluster.Spec.ClusterLinkOptions.DefaultNICName) - if n.Spec.InterfaceName == network.AutoSelectInterfaceFlag { - n.Spec.IP = internalIP - n.Spec.IP6 = internalIP6 - } + // n.Spec.InterfaceName while set by clusterlink-agent return nil }) if err != nil { diff --git a/pkg/operator/clusterlink/agent/manifests.go b/pkg/operator/clusterlink/agent/manifests.go index fdac753e0..ff3e33f7d 100644 --- a/pkg/operator/clusterlink/agent/manifests.go +++ b/pkg/operator/clusterlink/agent/manifests.go @@ -39,6 +39,7 @@ spec: command: - clusterlink-agent - --kubeconfig=/etc/clusterlink/kubeconfig + - --v=4 env: - name: CLUSTER_NAME value: "{{ .ClusterName }}"