forked from kosmos-io/kosmos
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
draft: support serverless leaf cluster
Signed-off-by: OrangeBao <[email protected]>
- Loading branch information
Showing
37 changed files
with
3,833 additions
and
563 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
80 changes: 80 additions & 0 deletions
80
pkg/clustertree/cluster-manager/controllers/pod/leaf-pod/k8s/syncer.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| package k8s | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "time" | ||
|
|
||
| "k8s.io/apimachinery/pkg/api/errors" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/client-go/kubernetes" | ||
| "k8s.io/klog" | ||
|
|
||
| "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/runtime" | ||
| "github.com/kosmos.io/kosmos/pkg/utils/podutils" | ||
| ) | ||
|
|
||
| type leafPodK8sSyncer struct { | ||
| LeafClient *kubernetes.Clientset | ||
| RootClient kubernetes.Interface | ||
| } | ||
|
|
||
| const ( | ||
| LeafPodControllerName = "leaf-pod-controller" | ||
| LeafPodRequeueTime = 10 * time.Second | ||
| ) | ||
|
|
||
| func DeletePodInRootCluster(ctx context.Context, rootnamespacedname runtime.NamespacedName, rootClient kubernetes.Interface) error { | ||
| rPod, err := rootClient.CoreV1().Pods(rootnamespacedname.Namespace).Get(ctx, rootnamespacedname.Name, metav1.GetOptions{}) | ||
| if err != nil { | ||
| if errors.IsNotFound(err) { | ||
| return nil | ||
| } else { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| rPodCopy := rPod.DeepCopy() | ||
|
|
||
| if err := rootClient.CoreV1().Pods(rPodCopy.Namespace).Delete(ctx, rPodCopy.Name, metav1.DeleteOptions{ | ||
| GracePeriodSeconds: new(int64), | ||
| }); err != nil { | ||
| if !errors.IsNotFound(err) { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (s *leafPodK8sSyncer) Reconcile(ctx context.Context, key runtime.NamespacedName) (runtime.Result, error) { | ||
| pod, err := s.LeafClient.CoreV1().Pods(key.Namespace).Get(ctx, key.Name, metav1.GetOptions{}) | ||
| if err != nil { | ||
| if errors.IsNotFound(err) { | ||
| // delete pod in root | ||
| if err := DeletePodInRootCluster(ctx, key, s.RootClient); err != nil { | ||
| return runtime.Result{RequeueAfter: LeafPodRequeueTime}, nil | ||
| } | ||
| return runtime.Result{}, nil | ||
| } | ||
|
|
||
| klog.Errorf("get %s error: %v", key, err) | ||
| return runtime.Result{RequeueAfter: LeafPodRequeueTime}, nil | ||
| } | ||
|
|
||
| podCopy := pod.DeepCopy() | ||
|
|
||
| // if ShouldSkipStatusUpdate(podCopy) { | ||
| // return reconcile.Result{}, nil | ||
| // } | ||
|
|
||
| if podutils.IsKosmosPod(podCopy) { | ||
| podutils.FitObjectMeta(&podCopy.ObjectMeta) | ||
| podCopy.ResourceVersion = "0" | ||
| if _, err := s.RootClient.CoreV1().Pods(podCopy.Namespace).UpdateStatus(ctx, podCopy, metav1.UpdateOptions{}); err != nil && !errors.IsNotFound(err) { | ||
| klog.V(4).Info(fmt.Sprintf("error while updating pod status in kubernetes: %s", err)) | ||
| return runtime.Result{RequeueAfter: LeafPodRequeueTime}, nil | ||
| } | ||
| } | ||
| return runtime.Result{}, nil | ||
| } |
102 changes: 102 additions & 0 deletions
102
pkg/clustertree/cluster-manager/controllers/pod/leaf-pod/k8s/workerqueue.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| package k8s | ||
|
|
||
| import ( | ||
| "time" | ||
|
|
||
| "github.com/google/go-cmp/cmp" | ||
| corev1 "k8s.io/api/core/v1" | ||
| kubeinformers "k8s.io/client-go/informers" | ||
| "k8s.io/client-go/kubernetes" | ||
| "k8s.io/client-go/tools/cache" | ||
| "k8s.io/client-go/util/workqueue" | ||
| "k8s.io/klog" | ||
|
|
||
| leafpodsyncers "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers/pod/leaf-pod" | ||
| "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/runtime" | ||
| "github.com/kosmos.io/kosmos/pkg/utils" | ||
| "github.com/kosmos.io/kosmos/pkg/utils/podutils" | ||
| ) | ||
|
|
||
| func NewLeafPodK8wWorkerQueue(opts *leafpodsyncers.LeafPodWorkerQueueOption) runtime.Controller { | ||
| // create the workqueue | ||
| queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) | ||
|
|
||
| client, err := kubernetes.NewForConfig(opts.Config) | ||
| if err != nil { | ||
| klog.Fatal(err) | ||
| } | ||
|
|
||
| // Create a shared informer factory for Kubernetes pods in the current namespace (if specified) and scheduled to the current node. | ||
| podInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions( | ||
| client, | ||
| 5*time.Second, | ||
| ) | ||
|
|
||
| podInformer := podInformerFactory.Core().V1().Pods() | ||
|
|
||
| eventFilter := func(obj interface{}) (bool, *corev1.Pod) { | ||
| p, ok := obj.(*corev1.Pod) | ||
|
|
||
| if !ok { | ||
| klog.Fatal("convert pod error") | ||
| return false, p | ||
| } | ||
|
|
||
| if len(p.Spec.NodeName) == 0 { | ||
| return false, p | ||
| } | ||
|
|
||
| if p.GetNamespace() == utils.ReservedNS { | ||
| return false, p | ||
| } | ||
|
|
||
| return podutils.IsKosmosPod(p), p | ||
| } | ||
|
|
||
| _, err = podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ | ||
| AddFunc: func(obj interface{}) { | ||
| if flag, pod := eventFilter(obj); flag { | ||
| queue.Add(runtime.NamespacedName{ | ||
| Name: pod.Name, | ||
| Namespace: pod.Namespace, | ||
| }) | ||
| } | ||
| }, | ||
| UpdateFunc: func(old interface{}, new interface{}) { | ||
| if flag, pod := eventFilter(old); flag { | ||
| if !cmp.Equal(old.(*corev1.Pod).Status, new.(*corev1.Pod).Status) { | ||
| queue.Add(runtime.NamespacedName{ | ||
| Name: pod.Name, | ||
| Namespace: pod.Namespace, | ||
| }) | ||
| } | ||
| } | ||
| }, | ||
| DeleteFunc: func(obj interface{}) { | ||
| if flag, pod := eventFilter(obj); flag { | ||
| queue.Add(runtime.NamespacedName{ | ||
| Name: pod.Name, | ||
| Namespace: pod.Namespace, | ||
| }) | ||
| } | ||
| }, | ||
| }) | ||
|
|
||
| if err != nil { | ||
| klog.Fatalf("add event handler error: %s", err) | ||
| panic(err) | ||
| } | ||
|
|
||
| leafClient, err := kubernetes.NewForConfig(opts.Config) | ||
| if err != nil { | ||
| klog.Fatalf("could not build clientset for cluster %s", err) | ||
| panic(err) | ||
| } | ||
|
|
||
| leafK8sSyncer := &leafPodK8sSyncer{ | ||
| LeafClient: leafClient, | ||
| RootClient: opts.RootClient, | ||
| } | ||
|
|
||
| return runtime.NewK8sWorkerQueue(queue, podInformer.Informer(), leafK8sSyncer) | ||
| } |
Oops, something went wrong.