title: Add ipsec tunnel mode to support cross clusters and elastic ip

Signed-off-by: GreatLazyMan <[email protected]>

cluster/images/Dockerfile

@@ -4,6 +4,6 @@ ARG BINARY
 
 RUN apk add --no-cache ca-certificates
 RUN apk update && apk upgrade
-RUN apk add ip6tables iptables curl tcpdump busybox-extras
+RUN apk add ip6tables iptables ipset curl tcpdump busybox-extras
 
 COPY ${BINARY} /bin/${BINARY}

cmd/clusterlink/elector/app/elector.go

@@ -126,10 +126,11 @@ func run(ctx context.Context, opts *options.Options) error {
 						err := elector.EnsureGateWayRole()
 						if err != nil {
 							klog.Errorf("set gateway role failure: %v, retry after 10 sec.", err)
-							time.Sleep(10 * time.Second)
+							time.Sleep(3 * time.Second)
 						} else {
-							klog.V(4).Info("ensure gateway role success, recheck after 60 sec.")
-							time.Sleep(60 * time.Second)
+							timeToRecheck := 3 * time.Second
+							klog.V(4).Infof("ensure gateway role success, recheck after %d sec.", int(timeToRecheck))
+							time.Sleep(timeToRecheck)
 						}
 					}
 				}

deploy/crds/kosmos.io_clusternodes.yaml

@@ -45,6 +45,8 @@ spec:
             properties:
               clusterName:
                 type: string
+              elasticip:
+                type: string
               interfaceName:
                 type: string
               ip:
@@ -63,11 +65,13 @@ spec:
                 type: array
             type: object
           status:
+            properties:
+              nodeStatus:
+                type: string
             type: object
         required:
         - spec
         type: object
     served: true
     storage: true
-    subresources:
-      status: {}
+    subresources: {}

deploy/crds/kosmos.io_clusters.yaml

@@ -58,6 +58,10 @@ spec:
                     - ip
                     - ip6
                     type: object
+                  clusterpodCIDRs:
+                    items:
+                      type: string
+                    type: array
                   cni:
                     default: calico
                     type: string
@@ -107,9 +111,17 @@ spec:
                       - nodeName
                       type: object
                     type: array
+                  nodeElasticIPMap:
+                    additionalProperties:
+                      type: string
+                    description: NodeElasticIPMap presents mapping between nodename
+                      in kubernetes and elasticIP
+                    type: object
                   useIPPool:
                     default: false
                     type: boolean
+                  useexternalapiserver:
+                    type: boolean
                 type: object
               clusterTreeOptions:
                 properties:

deploy/crds/kosmos.io_nodeconfigs.yaml

@@ -92,6 +92,18 @@ spec:
                   - mac
                   type: object
                 type: array
+              ipsetsavoidmasq:
+                items:
+                  properties:
+                    cidr:
+                      type: string
+                    name:
+                      type: string
+                  required:
+                  - cidr
+                  - name
+                  type: object
+                type: array
               iptables:
                 items:
                   properties:
@@ -122,6 +134,52 @@ spec:
                   - gw
                   type: object
                 type: array
+              xfrmpolicies:
+                items:
+                  properties:
+                    dir:
+                      type: integer
+                    leftip:
+                      type: string
+                    leftnet:
+                      type: string
+                    reqid:
+                      type: integer
+                    rightip:
+                      type: string
+                    rightnet:
+                      type: string
+                  required:
+                  - dir
+                  - leftip
+                  - leftnet
+                  - reqid
+                  - rightip
+                  - rightnet
+                  type: object
+                type: array
+              xfrmstates:
+                items:
+                  properties:
+                    PSK:
+                      type: string
+                    leftip:
+                      type: string
+                    reqid:
+                      type: integer
+                    rightip:
+                      type: string
+                    spi:
+                      format: int32
+                      type: integer
+                  required:
+                  - PSK
+                  - leftip
+                  - reqid
+                  - rightip
+                  - spi
+                  type: object
+                type: array
             type: object
           status:
             properties:

pkg/apis/kosmos/v1alpha1/cluster_types.go

@@ -94,6 +94,14 @@ type ClusterLinkOptions struct {
 
 	// +optional
 	AutodetectionMethod string `json:"autodetectionMethod,omitempty"`
+
+	// NodeElasticIPMap presents mapping between nodename in kubernetes and elasticIP
+	// +optional
+	NodeElasticIPMap map[string]string `json:"nodeElasticIPMap,omitempty"`
+	// +optional
+	ClusterPodCIDRs []string `json:"clusterpodCIDRs,omitempty"`
+	// +optional
+	UseExternalApiserver bool `json:"useexternalapiserver,omitempty"`
 }
 
 type ClusterTreeOptions struct {

pkg/apis/kosmos/v1alpha1/clusternode_types.go

@@ -8,7 +8,6 @@ import (
 
 // +genclient
 // +genclient:nonNamespaced
-// +kubebuilder:subresource:status
 // +kubebuilder:resource:scope="Cluster"
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +kubebuilder:printcolumn:name="ROLES",type=string,JSONPath=`.spec.roles`
@@ -33,6 +32,8 @@ type ClusterNodeSpec struct {
 	// +optional
 	IP string `json:"ip,omitempty"`
 	// +optional
+	ElasticIP string `json:"elasticip,omitempty"`
+	// +optional
 	IP6 string `json:"ip6,omitempty"`
 	// +optional
 	Roles []Role `json:"roles,omitempty"`
@@ -41,6 +42,8 @@ type ClusterNodeSpec struct {
 }
 
 type ClusterNodeStatus struct {
+	// +optional
+	NodeStatus string `json:"nodeStatus,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/kosmos/v1alpha1/constants.go

@@ -28,3 +28,17 @@ type DeviceType string
 const (
 	VxlanDevice DeviceType = "vxlan"
 )
+
+const (
+	DefaultPSK       string = "bfd6224354977084568832b811226b3d6cff6685"
+	DefaultPSKPreStr        = "WelcometoKosmos"
+	DefaultReqID     int    = 336
+)
+
+type IPSECDirection int
+
+const (
+	IPSECIn  IPSECDirection = 0
+	IPSECOut IPSECDirection = 1
+	IPSECFwd IPSECDirection = 2
+)

pkg/apis/kosmos/v1alpha1/nodeconfig_types.go

@@ -21,11 +21,14 @@ type NodeConfig struct {
 }
 
 type NodeConfigSpec struct {
-	Devices  []Device   `json:"devices,omitempty"`
-	Routes   []Route    `json:"routes,omitempty"`
-	Iptables []Iptables `json:"iptables,omitempty"`
-	Fdbs     []Fdb      `json:"fdbs,omitempty"`
-	Arps     []Arp      `json:"arps,omitempty"`
+	Devices          []Device     `json:"devices,omitempty"`
+	Routes           []Route      `json:"routes,omitempty"`
+	Iptables         []Iptables   `json:"iptables,omitempty"`
+	Fdbs             []Fdb        `json:"fdbs,omitempty"`
+	Arps             []Arp        `json:"arps,omitempty"`
+	XfrmPolicies     []XfrmPolicy `json:"xfrmpolicies,omitempty"`
+	XfrmStates       []XfrmState  `json:"xfrmstates,omitempty"`
+	IPsetsAvoidMasqs []IPset      `json:"ipsetsavoidmasq,omitempty"`
 }
 
 type NodeConfigStatus struct {
@@ -101,6 +104,50 @@ func (a *Arp) Compare(v Arp) bool {
 		a.Dev == v.Dev
 }
 
+type XfrmPolicy struct {
+	LeftIP   string `json:"leftip"`
+	LeftNet  string `json:"leftnet"`
+	RightIP  string `json:"rightip"`
+	RightNet string `json:"rightnet"`
+	ReqID    int    `json:"reqid"`
+	Dir      int    `json:"dir"`
+}
+
+func (a *XfrmPolicy) Compare(v XfrmPolicy) bool {
+	return a.LeftIP == v.LeftIP &&
+		a.LeftNet == v.LeftNet &&
+		a.RightNet == v.RightNet &&
+		a.RightIP == v.RightIP &&
+		a.ReqID == v.ReqID &&
+		a.Dir == v.Dir
+}
+
+type XfrmState struct {
+	LeftIP  string `json:"leftip"`
+	RightIP string `json:"rightip"`
+	ReqID   int    `json:"reqid"`
+	SPI     uint32 `json:"spi"`
+	PSK     string `json:"PSK"`
+}
+
+func (a *XfrmState) Compare(v XfrmState) bool {
+	return a.LeftIP == v.LeftIP &&
+		a.RightIP == v.RightIP &&
+		a.ReqID == v.ReqID &&
+		a.PSK == v.PSK &&
+		a.SPI == v.SPI
+}
+
+type IPset struct {
+	CIDR string `json:"cidr"`
+	Name string `json:"name"`
+}
+
+func (a *IPset) Compare(v IPset) bool {
+	return a.CIDR == v.CIDR &&
+		a.Name == v.Name
+}
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 type NodeConfigList struct {

pkg/clusterlink/agent-manager/network-manager/network_manager.go

@@ -112,6 +112,29 @@ func (e *NetworkManager) Diff(oldConfig, newConfig *clusterlinkv1alpha1.NodeConf
 		createConfig.Routes = createRecord
 		isSame = false
 	}
+	// ipsec:
+	if flag, deleteRecord, createRecord := compareFunc(oldConfig.XfrmPolicies, newConfig.XfrmPolicies, func(a, b clusterlinkv1alpha1.XfrmPolicy) bool {
+		return a.Compare(b)
+	}); !flag {
+		deleteConfig.XfrmPolicies = deleteRecord
+		createConfig.XfrmPolicies = createRecord
+		isSame = false
+	}
+	if flag, deleteRecord, createRecord := compareFunc(oldConfig.XfrmStates, newConfig.XfrmStates, func(a, b clusterlinkv1alpha1.XfrmState) bool {
+		return a.Compare(b)
+	}); !flag {
+		deleteConfig.XfrmStates = deleteRecord
+		createConfig.XfrmStates = createRecord
+		isSame = false
+	}
+	//ipset
+	if flag, deleteRecord, createRecord := compareFunc(oldConfig.IPsetsAvoidMasqs, newConfig.IPsetsAvoidMasqs, func(a, b clusterlinkv1alpha1.IPset) bool {
+		return a.Compare(b)
+	}); !flag {
+		deleteConfig.IPsetsAvoidMasqs = deleteRecord
+		createConfig.IPsetsAvoidMasqs = createRecord
+		isSame = false
+	}
 	// iptables:
 	if flag, deleteRecord, createRecord := compareFunc(oldConfig.Iptables, newConfig.Iptables, func(a, b clusterlinkv1alpha1.Iptables) bool {
 		return a.Compare(b)
@@ -188,6 +211,24 @@ func (e *NetworkManager) WriteSys(configDiff *ConfigDiff) error {
 				errs = errors.Wrap(err, fmt.Sprint(errs))
 			}
 		}
+		if config.XfrmPolicies != nil {
+			if err := e.NetworkInterface.DeleteXfrmPolicies(config.XfrmPolicies); err != nil {
+				klog.Warning(err)
+				errs = errors.Wrap(err, fmt.Sprint(errs))
+			}
+		}
+		if config.XfrmStates != nil {
+			if err := e.NetworkInterface.DeleteXfrmStates(config.XfrmStates); err != nil {
+				klog.Warning(err)
+				errs = errors.Wrap(err, fmt.Sprint(errs))
+			}
+		}
+		if config.IPsetsAvoidMasqs != nil {
+			if err := e.NetworkInterface.DeleteIPsetsAvoidMasq(config.IPsetsAvoidMasqs); err != nil {
+				klog.Warning(err)
+				errs = errors.Wrap(err, fmt.Sprint(errs))
+			}
+		}
 	}
 
 	if configDiff.createConfig != nil {
@@ -223,6 +264,24 @@ func (e *NetworkManager) WriteSys(configDiff *ConfigDiff) error {
 				errs = errors.Wrap(err, fmt.Sprint(errs))
 			}
 		}
+		if config.XfrmPolicies != nil {
+			if err := e.NetworkInterface.AddXfrmPolicies(config.XfrmPolicies); err != nil {
+				klog.Warning(err)
+				errs = errors.Wrap(err, fmt.Sprint(errs))
+			}
+		}
+		if config.XfrmStates != nil {
+			if err := e.NetworkInterface.AddXfrmStates(config.XfrmStates); err != nil {
+				klog.Warning(err)
+				errs = errors.Wrap(err, fmt.Sprint(errs))
+			}
+		}
+		if config.IPsetsAvoidMasqs != nil {
+			if err := e.NetworkInterface.AddIPsetsAvoidMasq(config.IPsetsAvoidMasqs); err != nil {
+				klog.Warning(err)
+				errs = errors.Wrap(err, fmt.Sprint(errs))
+			}
+		}
 	}
 
 	return errs
@@ -254,11 +313,14 @@ func (e *NetworkManager) UpdateFromChecker() NodeConfigSyncStatus {
 }
 
 func printNodeConfig(data *clusterlinkv1alpha1.NodeConfigSpec) {
-	klog.Infof("device: ", data.Devices)
-	klog.Infof("Arps: ", data.Arps)
-	klog.Infof("Fdbs: ", data.Fdbs)
-	klog.Infof("Iptables: ", data.Iptables)
-	klog.Infof("Routes: ", data.Routes)
+	klog.Infof("device: %v", data.Devices)
+	klog.Infof("Arps: %v", data.Arps)
+	klog.Infof("Fdbs: %v", data.Fdbs)
+	klog.Infof("Iptables: %v", data.Iptables)
+	klog.Infof("Routes: %v", data.Routes)
+	klog.Infof("XfrmPolicys: %v", data.XfrmPolicies)
+	klog.Infof("XfrmStates: %v", data.XfrmStates)
+	klog.Infof("IPsetsAvoidMasqs: %v", data.IPsetsAvoidMasqs)
 }
 
 func (e *NetworkManager) UpdateSync() NodeConfigSyncStatus {

pkg/clusterlink/constants.go

@@ -0,0 +1,16 @@
+package clusterlink
+
+const (
+	FLANNEL             = "flannel"
+	TECENT_GLOBALROUTER = "globalrouter"
+)
+
+var (
+	NonMasqCNI = map[string]struct{}{
+		FLANNEL:             {},
+		TECENT_GLOBALROUTER: {},
+	}
+	NonMasqCNISlice = []string{
+		FLANNEL, TECENT_GLOBALROUTER,
+	}
+)