Merge branch 'kosmos-io:main' into main

.github/workflows/ci.yml

@@ -65,4 +65,40 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: '1.20'
-      - run: make test
+      - run: make test
+  e2e:
+    name: E2e test
+    needs: build
+    runs-on: ubuntu-22.04
+    steps:
+      # Free up disk space on Ubuntu
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed, if set to "true" but frees about 6 GB
+          tool-cache: false
+          # all of these default to true, but feel free to set to "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          docker-images: false
+          swap-storage: false
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.20'
+      - name: Prepare e2e env
+        run: ./hack/prepare-e2e.sh
+      - name: Run e2e test
+        run: ./hack/rune2e.sh
+      - name: Upload logs
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: kosmos-e2e-logs-${{ github.run_id }}
+          path: ${{ github.workspace }}/e2e-test/logs-*

Makefile

@@ -7,6 +7,7 @@ REGISTRY?="ghcr.io/kosmos-io"
 REGISTRY_USER_NAME?=""
 REGISTRY_PASSWORD?=""
 REGISTRY_SERVER_ADDRESS?=""
+KIND_IMAGE_TAG?="v1.25.3"
 
 TARGETS :=  clusterlink-controller-manager  \
 			kosmos-operator \
@@ -133,4 +134,11 @@ ifeq (, $(shell which golangci-lint))
 GOLANGLINT_BIN=$(shell go env GOPATH)/bin/golangci-lint
 else
 GOLANGLINT_BIN=$(shell which golangci-lint)
-endif
+endif
+
+image-base-kind-builder:
+	docker buildx build \
+	    -t $(REGISTRY)/node:$(KIND_IMAGE_TAG) \
+        --platform=linux/amd64,linux/arm64 \
+        --push \
+        -f cluster/images/buildx.kind.Dockerfile .

cluster/images/buildx.kind.Dockerfile

@@ -0,0 +1,2 @@
+FROM kindest/node:v1.25.3
+RUN clean-install tcpdump && clean-install iputils-ping && clean-install iptables && clean-install net-tools

cmd/clusterlink/OWNERS

@@ -0,0 +1,10 @@
+approvers:
+  - wuyingjun-lucky
+  - hanweisen
+  - wangyizhi1
+  - OrangeBao
+reviewers:
+  - wuyingjun-lucky
+  - hanweisen
+  - wangyizhi1
+  - OrangeBao

cmd/clusterlink/agent/app/agent.go

@@ -15,7 +15,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/kosmos.io/kosmos/cmd/clusterlink/agent/app/options"
-	linkagent "github.com/kosmos.io/kosmos/pkg/clusterlink/agent"
+	linkagent "github.com/kosmos.io/kosmos/pkg/clusterlink/agent-manager"
 	"github.com/kosmos.io/kosmos/pkg/clusterlink/network"
 	kosmosclientset "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned"
 	kosmosinformer "github.com/kosmos.io/kosmos/pkg/generated/informers/externalversions"
@@ -116,6 +116,17 @@ func run(ctx context.Context, opts *options.Options) error {
 		return err
 	}
 
+	autoDetectController := linkagent.AutoDetectReconciler{
+		Client:      mgr.GetClient(),
+		NodeName:    os.Getenv(utils.EnvNodeName),
+		ClusterName: os.Getenv(utils.EnvClusterName),
+	}
+
+	if err = autoDetectController.SetupWithManager(mgr); err != nil {
+		klog.Fatalf("Unable to create auto detect controller: %v", err)
+		return err
+	}
+
 	factory.Start(ctx.Done())
 	factory.WaitForCacheSync(ctx.Done())
 

cmd/clusterlink/floater/app/floater.go

@@ -16,7 +16,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"github.com/kosmos.io/kosmos/cmd/clusterlink/floater/app/options"
-	networkmanager "github.com/kosmos.io/kosmos/pkg/clusterlink/agent/network-manager"
+	networkmanager "github.com/kosmos.io/kosmos/pkg/clusterlink/agent-manager/network-manager"
 	"github.com/kosmos.io/kosmos/pkg/clusterlink/network"
 	"github.com/kosmos.io/kosmos/pkg/sharedcli"
 	"github.com/kosmos.io/kosmos/pkg/sharedcli/klogflag"

cmd/clustertree/cluster-manager/app/manager.go

@@ -178,9 +178,10 @@ func run(ctx context.Context, opts *options.Options) error {
 	if opts.MultiClusterService {
 		// add serviceExport controller
 		ServiceExportController := mcs.ServiceExportController{
-			RootClient:    mgr.GetClient(),
-			EventRecorder: mgr.GetEventRecorderFor(mcs.ServiceExportControllerName),
-			Logger:        mgr.GetLogger(),
+			RootClient:         mgr.GetClient(),
+			EventRecorder:      mgr.GetEventRecorderFor(mcs.ServiceExportControllerName),
+			Logger:             mgr.GetLogger(),
+			ReservedNamespaces: opts.ReservedNamespaces,
 		}
 		if err = ServiceExportController.SetupWithManager(mgr); err != nil {
 			return fmt.Errorf("error starting %s: %v", mcs.ServiceExportControllerName, err)
@@ -194,6 +195,7 @@ func run(ctx context.Context, opts *options.Options) error {
 			AutoCreateMCSPrefix: opts.AutoCreateMCSPrefix,
 			RootKosmosClient:    rootKosmosClient,
 			GlobalLeafManager:   globalleafManager,
+			ReservedNamespaces:  opts.ReservedNamespaces,
 		}
 		if err = autoCreateMCSController.SetupWithManager(mgr); err != nil {
 			return fmt.Errorf("error starting %s: %v", mcs.AutoCreateMCSControllerName, err)

cmd/clustertree/cluster-manager/app/options/options.go

@@ -35,8 +35,11 @@ type Options struct {
 	// Enable oneway storage controllers
 	OnewayStorageControllers bool
 
-	// AutoCreateMCSPrefix is the prefix of the namespace for service to auto create in leaf cluster
+	// AutoCreateMCSPrefix are the prefix of the namespace for service to auto create in leaf cluster
 	AutoCreateMCSPrefix []string
+
+	// ReservedNamespaces are the protected namespaces to prevent Kosmos for deleting system resources
+	ReservedNamespaces []string
 }
 
 type KubernetesOptions struct {
@@ -78,5 +81,6 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&o.RootCoreDNSServiceName, "root-coredns-service-name", CoreDNSServiceName, "The name of the CoreDNS service in the root cluster, used to locate the CoreDNS service when MultiClusterService is disabled.")
 	flags.BoolVar(&o.OnewayStorageControllers, "oneway-storage-controllers", false, "Turn on or off oneway storage controllers.")
 	flags.StringSliceVar(&o.AutoCreateMCSPrefix, "auto-mcs-prefix", []string{}, "The prefix of namespace for service to auto create mcs resources")
+	flags.StringSliceVar(&o.ReservedNamespaces, "reserved-namespaces", []string{"kube-system"}, "The namespaces protected by Kosmos that the controller-manager will skip.")
 	options.BindLeaderElectionFlags(&o.LeaderElection, flags)
 }

deploy/clusterlink-agent.yaml

@@ -46,6 +46,12 @@ spec:
           - mountPath: /etc/clusterlink/kubeconfig
             name: proxy-config
             readOnly: true
+          - mountPath: /run/xtables.lock
+            name: iptableslock
+            readOnly: false
+          - mountPath: /lib/modules
+            name: lib-modules
+            readOnly: true
       terminationGracePeriodSeconds: 30
       securityContext:
         privileged: true
@@ -55,3 +61,12 @@ spec:
         configMap:
           defaultMode: 420
           name: proxy-config
+      - hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+        name: iptableslock
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+
+

deploy/clusterlink-elector.yml

@@ -21,6 +21,23 @@ spec:
       labels:
         app: elector
     spec:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: kosmos.io/exclude
+                  operator: DoesNotExist
+      podAntiAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                    - elector
+            namespaces:
+              - kosmos-system
+            topologyKey: kubernetes.io/hostname
       containers:
         - command:
             - clusterlink-elector

deploy/clustertree-cluster-manager.yml

@@ -79,6 +79,7 @@ spec:
           command:
             - clustertree-cluster-manager
             - --multi-cluster-service=true
+            - --auto-mcs-prefix=kosmos-e2e
             - --v=4
       volumes:
         - name: credentials