From a7d29d672f63735915af9a4d0d5446d121d74a9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=91=B8=E9=B1=BC=E5=96=B5?= Date: Wed, 30 Aug 2023 17:39:35 +0800 Subject: [PATCH 01/13] unserve v1alpha1 version of platformadmin crd (#1659) Signed-off-by: LavenderQAQ --- charts/yurt-manager/crds/iot.openyurt.io_platformadmins.yaml | 2 +- pkg/apis/iot/v1alpha1/platformadmin_types.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/yurt-manager/crds/iot.openyurt.io_platformadmins.yaml b/charts/yurt-manager/crds/iot.openyurt.io_platformadmins.yaml index edb4df7fd76..0b980281c1b 100644 --- a/charts/yurt-manager/crds/iot.openyurt.io_platformadmins.yaml +++ b/charts/yurt-manager/crds/iot.openyurt.io_platformadmins.yaml @@ -8380,7 +8380,7 @@ spec: type: integer type: object type: object - served: true + served: false storage: false subresources: status: {} diff --git a/pkg/apis/iot/v1alpha1/platformadmin_types.go b/pkg/apis/iot/v1alpha1/platformadmin_types.go index b32d110beff..d108599164a 100644 --- a/pkg/apis/iot/v1alpha1/platformadmin_types.go +++ b/pkg/apis/iot/v1alpha1/platformadmin_types.go @@ -110,6 +110,7 @@ type PlatformAdminCondition struct { // +kubebuilder:printcolumn:name="Deployment",type="integer",JSONPath=".status.deploymentReplicas",description="The Deployment Replica." // +kubebuilder:printcolumn:name="ReadyDeployment",type="integer",JSONPath=".status.deploymentReadyReplicas",description="The Ready Deployment Replica." // +kubebuilder:deprecatedversion:warning="iot.openyurt.io/v1alpha1 PlatformAdmin will be deprecated in future; use iot.openyurt.io/v1alpha2 PlatformAdmin; v1alpha1 PlatformAdmin.Spec.ServiceType only support ClusterIP" +// +kubebuilder:unservedversion // PlatformAdmin is the Schema for the samples API type PlatformAdmin struct { From 0dbc3f161ee300634a23e68e6de1b2304cf5eb82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=91=B8=E9=B1=BC=E5=96=B5?= Date: Wed, 30 Aug 2023 17:57:35 +0800 Subject: [PATCH 02/13] fix: yurt-iot-dock cannot be dynamically deployed in platformadmin (#1679) Signed-off-by: LavenderQAQ --- .../platformadmin/platformadmin_controller.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go b/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go index 46686d11dab..6d01a0ea155 100644 --- a/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go +++ b/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go @@ -777,12 +777,6 @@ func (r *ReconcilePlatformAdmin) initFramework(ctx context.Context, platformAdmi r.calculateDesiredComponents(platformAdmin, platformAdminFramework, nil) } - yurtIotDock, err := newYurtIoTDockComponent(platformAdmin, platformAdminFramework) - if err != nil { - return err - } - platformAdminFramework.Components = append(platformAdminFramework.Components, yurtIotDock) - // For better serialization, the serialization method of the Kubernetes runtime library is used data, err := runtime.Encode(r.yamlSerializer, platformAdminFramework) if err != nil { @@ -862,6 +856,16 @@ func (r *ReconcilePlatformAdmin) calculateDesiredComponents(platformAdmin *iotv1 } } + // The yurt-iot-dock is maintained by openyurt and is not obtained through an auto-collector. + // Therefore, it needs to be handled separately + if addedComponentSet.Has(util.IotDockName) { + yurtIotDock, err := newYurtIoTDockComponent(platformAdmin, platformAdminFramework) + if err != nil { + klog.Errorf(Format("newYurtIoTDockComponent error %v", err)) + } + desiredComponents = append(desiredComponents, yurtIotDock) + } + // TODO: In order to be compatible with v1alpha1, we need to add the component from annotation translation here if additionalComponents != nil { desiredComponents = append(desiredComponents, additionalComponents...) From 8b94570e826517def374354762ffa5af4fce4d56 Mon Sep 17 00:00:00 2001 From: wesleysu <59680532+River-sh@users.noreply.github.com> Date: Thu, 31 Aug 2023 10:01:36 +0800 Subject: [PATCH 03/13] add gateway internal service controller (#1677) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 珩轩 --- pkg/yurtmanager/controller/controller.go | 7 +- pkg/yurtmanager/controller/raven/common.go | 7 +- .../gateway_internal_service_controller.go | 405 ++++++++++++++++++ ...ateway_internal_service_controller_test.go | 212 +++++++++ ...teway_internal_service_enqueue_handlers.go | 147 +++++++ .../controller/raven/utils/options.go | 68 +++ .../controller/raven/utils/utils.go | 92 +++- 7 files changed, 921 insertions(+), 17 deletions(-) create mode 100644 pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller.go create mode 100644 pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller_test.go create mode 100644 pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_enqueue_handlers.go create mode 100644 pkg/yurtmanager/controller/raven/utils/options.go diff --git a/pkg/yurtmanager/controller/controller.go b/pkg/yurtmanager/controller/controller.go index 297c990dd99..2e8dd41dc39 100644 --- a/pkg/yurtmanager/controller/controller.go +++ b/pkg/yurtmanager/controller/controller.go @@ -28,6 +28,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/dns" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewayinternalservice" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewaypickup" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology" servicetopologyendpoints "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/endpoints" @@ -56,8 +57,6 @@ func init() { controllerAddFuncs[daemonpodupdater.ControllerName] = []AddControllerFn{daemonpodupdater.Add} controllerAddFuncs[delegatelease.ControllerName] = []AddControllerFn{delegatelease.Add} controllerAddFuncs[podbinding.ControllerName] = []AddControllerFn{podbinding.Add} - controllerAddFuncs[raven.GatewayPickupControllerName] = []AddControllerFn{gatewaypickup.Add} - controllerAddFuncs[raven.GatewayDNSControllerName] = []AddControllerFn{dns.Add} controllerAddFuncs[nodepool.ControllerName] = []AddControllerFn{nodepool.Add} controllerAddFuncs[yurtcoordinatorcert.ControllerName] = []AddControllerFn{yurtcoordinatorcert.Add} controllerAddFuncs[servicetopology.ControllerName] = []AddControllerFn{servicetopologyendpoints.Add, servicetopologyendpointslice.Add} @@ -65,6 +64,10 @@ func init() { controllerAddFuncs[yurtappset.ControllerName] = []AddControllerFn{yurtappset.Add} controllerAddFuncs[yurtappdaemon.ControllerName] = []AddControllerFn{yurtappdaemon.Add} controllerAddFuncs[platformadmin.ControllerName] = []AddControllerFn{platformadmin.Add} + + controllerAddFuncs[raven.GatewayPickupControllerName] = []AddControllerFn{gatewaypickup.Add} + controllerAddFuncs[raven.GatewayDNSControllerName] = []AddControllerFn{dns.Add} + controllerAddFuncs[raven.GatewayInternalServiceController] = []AddControllerFn{gatewayinternalservice.Add} } // If you want to add additional RBAC, enter it here !!! @kadisi diff --git a/pkg/yurtmanager/controller/raven/common.go b/pkg/yurtmanager/controller/raven/common.go index 583c3be51e0..7a3b1a59567 100644 --- a/pkg/yurtmanager/controller/raven/common.go +++ b/pkg/yurtmanager/controller/raven/common.go @@ -21,7 +21,8 @@ var ( ) const ( - ControllerName = "gateway" - GatewayPickupControllerName = "raven-gateway-pickup" - GatewayDNSControllerName = "raven-dns" + ControllerName = "gateway" + GatewayPickupControllerName = "raven-gateway-pickup" + GatewayInternalServiceController = "raven-gateway-internal-service" + GatewayDNSControllerName = "raven-dns" ) diff --git a/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller.go b/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller.go new file mode 100644 index 00000000000..0f1e7c8946c --- /dev/null +++ b/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller.go @@ -0,0 +1,405 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gatewayinternalservice + +import ( + "context" + "fmt" + "net" + "sort" + "strconv" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + common "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/utils" +) + +const ( + HTTPPorts = "http" + HTTPSPorts = "https" +) + +func Format(format string, args ...interface{}) string { + s := fmt.Sprintf(format, args...) + return fmt.Sprintf("%s: %s", common.GatewayInternalServiceController, s) +} + +// Add creates a new Service Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { + return add(mgr, newReconciler(c, mgr)) +} + +var _ reconcile.Reconciler = &ReconcileService{} + +// ReconcileService reconciles a Gateway object +type ReconcileService struct { + client.Client + scheme *runtime.Scheme + recorder record.EventRecorder + option utils.Option +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { + return &ReconcileService{ + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + recorder: mgr.GetEventRecorderFor(common.GatewayInternalServiceController), + option: utils.NewOption(), + } +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New(common.GatewayInternalServiceController, mgr, controller.Options{ + Reconciler: r, MaxConcurrentReconciles: common.ConcurrentReconciles, + }) + if err != nil { + return err + } + + // Watch for changes to Gateway + err = c.Watch(&source.Kind{Type: &ravenv1beta1.Gateway{}}, &EnqueueRequestForGatewayEvent{}) + if err != nil { + return err + } + + //Watch for changes to raven agent + err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{}}, &EnqueueRequestForConfigEvent{}, predicate.NewPredicateFuncs( + func(object client.Object) bool { + cm, ok := object.(*corev1.ConfigMap) + if !ok { + return false + } + if cm.GetNamespace() != utils.WorkingNamespace { + return false + } + if cm.GetName() != utils.RavenAgentConfig { + return false + } + return true + }, + )) + if err != nil { + return err + } + + return nil +} + +// Reconcile reads that state of the cluster for a Gateway object and makes changes based on the state read +// and what is in the Gateway.Spec +func (r *ReconcileService) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + klog.V(2).Info(Format("started reconciling Service %s/%s", req.Namespace, req.Name)) + defer func() { + klog.V(2).Info(Format("finished reconciling Service %s/%s", req.Namespace, req.Name)) + }() + + gwList, err := r.listExposedGateway(ctx) + if err != nil { + return reconcile.Result{Requeue: true}, err + } + + enableProxy, _ := utils.CheckServer(ctx, r.Client) + r.option.SetProxyOption(enableProxy) + if err := r.reconcileService(ctx, req, gwList); err != nil { + err = fmt.Errorf(Format("unable to reconcile service: %s", err)) + return reconcile.Result{}, err + } + + if err := r.reconcileEndpoint(ctx, req, gwList); err != nil { + err = fmt.Errorf(Format("unable to reconcile endpoint: %s", err)) + return reconcile.Result{}, err + } + return reconcile.Result{}, nil +} + +func (r *ReconcileService) listExposedGateway(ctx context.Context) ([]*ravenv1beta1.Gateway, error) { + var gatewayList ravenv1beta1.GatewayList + if err := r.List(ctx, &gatewayList); err != nil { + return nil, fmt.Errorf(Format("unable to list gateways: %s", err)) + } + exposedGateways := make([]*ravenv1beta1.Gateway, 0) + for _, gw := range gatewayList.Items { + switch gw.Spec.ExposeType { + case ravenv1beta1.ExposeTypePublicIP: + exposedGateways = append(exposedGateways, gw.DeepCopy()) + case ravenv1beta1.ExposeTypeLoadBalancer: + exposedGateways = append(exposedGateways, gw.DeepCopy()) + default: + continue + } + } + return exposedGateways, nil +} + +func (r *ReconcileService) reconcileService(ctx context.Context, req ctrl.Request, gatewayList []*ravenv1beta1.Gateway) error { + if len(gatewayList) == 0 || !r.option.GetProxyOption() { + return r.cleanService(ctx, req) + } + return r.updateService(ctx, req, gatewayList) +} + +func (r *ReconcileService) cleanService(ctx context.Context, req ctrl.Request) error { + if err := r.Delete(ctx, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: req.Name, + Namespace: req.Namespace, + }, + }); err != nil && !apierrs.IsNotFound(err) { + return err + } + return nil +} + +func generateService(req ctrl.Request) corev1.Service { + return corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: req.Name, + Namespace: req.Namespace, + Labels: map[string]string{ + "app": "raven", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + }, + } +} + +func (r *ReconcileService) updateService(ctx context.Context, req ctrl.Request, gatewayList []*ravenv1beta1.Gateway) error { + insecurePort, securePort := r.getTargetPort() + servicePorts := acquiredSpecPorts(gatewayList, insecurePort, securePort) + sort.Slice(servicePorts, func(i, j int) bool { + return servicePorts[i].Name < servicePorts[j].Name + }) + + var svc corev1.Service + err := r.Get(ctx, req.NamespacedName, &svc) + if err != nil && !apierrs.IsNotFound(err) { + return err + } + if apierrs.IsNotFound(err) { + klog.V(2).InfoS(Format("create service"), "name", req.Name, "namespace", req.Namespace) + svc = generateService(req) + svc.Spec.Ports = servicePorts + return r.Create(ctx, &svc) + } + svc.Spec.Ports = servicePorts + return r.Update(ctx, &svc) +} + +func (r *ReconcileService) getTargetPort() (insecurePort, securePort int32) { + insecurePort = ravenv1beta1.DefaultProxyServerInsecurePort + securePort = ravenv1beta1.DefaultProxyServerSecurePort + var cm corev1.ConfigMap + err := r.Get(context.TODO(), types.NamespacedName{Namespace: utils.WorkingNamespace, Name: utils.RavenAgentConfig}, &cm) + if err != nil { + return + } + if cm.Data == nil { + return + } + _, internalInsecurePort, err := net.SplitHostPort(cm.Data[utils.ProxyServerInsecurePortKey]) + if err == nil { + insecure, _ := strconv.Atoi(internalInsecurePort) + insecurePort = int32(insecure) + } + + _, internalSecurePort, err := net.SplitHostPort(cm.Data[utils.ProxyServerSecurePortKey]) + if err == nil { + secure, _ := strconv.Atoi(internalSecurePort) + securePort = int32(secure) + } + return +} + +func acquiredSpecPorts(gatewayList []*ravenv1beta1.Gateway, insecurePort, securePort int32) []corev1.ServicePort { + specPorts := make([]corev1.ServicePort, 0) + for _, gw := range gatewayList { + specPorts = append(specPorts, generateServicePorts(gw.Spec.ProxyConfig.ProxyHTTPPort, HTTPPorts, insecurePort)...) + specPorts = append(specPorts, generateServicePorts(gw.Spec.ProxyConfig.ProxyHTTPSPort, HTTPSPorts, securePort)...) + } + return specPorts +} + +func generateServicePorts(ports, namePrefix string, targetPort int32) []corev1.ServicePort { + svcPorts := make([]corev1.ServicePort, 0) + for _, port := range splitPorts(ports) { + p, err := strconv.Atoi(port) + if err != nil { + continue + } + svcPorts = append(svcPorts, corev1.ServicePort{ + Name: fmt.Sprintf("%s-%s", namePrefix, port), + Protocol: corev1.ProtocolTCP, + Port: int32(p), + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: targetPort}, + }) + } + return svcPorts +} + +func splitPorts(str string) []string { + ret := make([]string, 0) + for _, val := range strings.Split(str, ",") { + ret = append(ret, strings.TrimSpace(val)) + } + return ret +} + +func (r *ReconcileService) reconcileEndpoint(ctx context.Context, req ctrl.Request, gatewayList []*ravenv1beta1.Gateway) error { + var service corev1.Service + err := r.Get(ctx, req.NamespacedName, &service) + if err != nil && !apierrs.IsNotFound(err) { + return err + } + if apierrs.IsNotFound(err) || service.DeletionTimestamp != nil || len(gatewayList) == 0 || !r.option.GetProxyOption() { + return r.cleanEndpoint(ctx, req) + } + return r.updateEndpoint(ctx, req, &service, gatewayList) +} + +func (r *ReconcileService) cleanEndpoint(ctx context.Context, req ctrl.Request) error { + if err := r.Delete(ctx, &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: req.Name, + Namespace: req.Namespace, + }, + }); err != nil && !apierrs.IsNotFound(err) { + return err + } + return nil +} + +func generateEndpoint(req ctrl.Request) corev1.Endpoints { + klog.V(2).InfoS(Format("create endpoint"), "name", req.Name, "namespace", req.Namespace) + return corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: req.Name, + Namespace: req.Namespace, + }, + Subsets: []corev1.EndpointSubset{}, + } +} + +func (r *ReconcileService) updateEndpoint(ctx context.Context, req ctrl.Request, service *corev1.Service, gatewayList []*ravenv1beta1.Gateway) error { + + subsets := []corev1.EndpointSubset{ + { + Addresses: r.ensureSpecEndpoints(ctx, gatewayList), + Ports: ensureSpecPorts(service), + }, + } + if len(subsets[0].Addresses) < 1 || len(subsets[0].Ports) < 1 { + klog.Warning(Format("endpoints %s/%s miss available node address or port, get node %d and port %d", + req.Namespace, req.Name, len(subsets[0].Addresses), len(subsets[0].Ports))) + return nil + } + var eps corev1.Endpoints + err := r.Get(ctx, req.NamespacedName, &eps) + if err != nil && !apierrs.IsNotFound(err) { + return err + } + if apierrs.IsNotFound(err) { + eps = generateEndpoint(req) + eps.Subsets = subsets + return r.Create(ctx, &eps) + } + eps.Subsets = subsets + return r.Update(ctx, &eps) +} + +func (r *ReconcileService) ensureSpecEndpoints(ctx context.Context, gateways []*ravenv1beta1.Gateway) []corev1.EndpointAddress { + specAddresses := make([]corev1.EndpointAddress, 0) + for _, gw := range gateways { + + if len(gw.Status.ActiveEndpoints) < 1 { + newGw, err := r.waitElectEndpoints(ctx, gw.Name) + if err == nil { + gw = newGw + } + } + for _, aep := range gw.Status.ActiveEndpoints { + if aep.Type != ravenv1beta1.Proxy { + continue + } + var node corev1.Node + err := r.Get(ctx, types.NamespacedName{Name: aep.NodeName}, &node) + if err != nil { + continue + } + specAddresses = append(specAddresses, corev1.EndpointAddress{ + IP: utils.GetNodeInternalIP(node), + NodeName: func(n corev1.Node) *string { return &n.Name }(node), + }) + } + } + return specAddresses +} + +func (r *ReconcileService) waitElectEndpoints(ctx context.Context, gwName string) (*ravenv1beta1.Gateway, error) { + var gw ravenv1beta1.Gateway + err := wait.PollImmediate(time.Second*5, time.Minute, func() (done bool, err error) { + err = r.Get(ctx, types.NamespacedName{Name: gwName}, &gw) + if err != nil { + return false, err + } + if len(gw.Status.ActiveEndpoints) < 1 { + return false, nil + } + return true, nil + }) + if err != nil { + return nil, err + } + return gw.DeepCopy(), nil +} + +func ensureSpecPorts(svc *corev1.Service) []corev1.EndpointPort { + specPorts := make([]corev1.EndpointPort, 0) + for _, port := range svc.Spec.Ports { + specPorts = append(specPorts, corev1.EndpointPort{ + Name: port.Name, + Port: int32(port.TargetPort.IntValue()), + Protocol: port.Protocol, + }) + } + return specPorts +} diff --git a/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller_test.go b/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller_test.go new file mode 100644 index 00000000000..12166e8b267 --- /dev/null +++ b/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller_test.go @@ -0,0 +1,212 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gatewayinternalservice + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/openyurtio/openyurt/pkg/apis" + "github.com/openyurtio/openyurt/pkg/apis/raven" + ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/utils" +) + +const ( + Node1Name = "node-1" + Node2Name = "node-2" + Node3Name = "node-3" + Node4Name = "node-4" + Node1Address = "192.168.0.1" + Node2Address = "192.168.0.2" + Node3Address = "192.168.0.3" + Node4Address = "192.168.0.4" + MockGateway = "gw-mock" +) + +func MockReconcile() *ReconcileService { + nodeList := &corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: Node1Name, + Labels: map[string]string{ + raven.LabelCurrentGateway: MockGateway, + }, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: Node1Address, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: Node2Name, + Labels: map[string]string{ + raven.LabelCurrentGateway: MockGateway, + }, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: Node2Address, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: Node3Name, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: Node3Address, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: Node4Name, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: Node4Address, + }, + }, + }, + }, + }, + } + configmaps := &corev1.ConfigMapList{ + Items: []corev1.ConfigMap{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RavenGlobalConfig, + Namespace: utils.WorkingNamespace, + }, + Data: map[string]string{ + utils.RavenEnableProxy: "true", + utils.RavenEnableTunnel: "true", + }, + }, + }, + } + gateways := &ravenv1beta1.GatewayList{ + Items: []ravenv1beta1.Gateway{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: MockGateway, + }, + Spec: ravenv1beta1.GatewaySpec{ + NodeSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + raven.LabelCurrentGateway: MockGateway, + }, + }, + ProxyConfig: ravenv1beta1.ProxyConfiguration{ + Replicas: 2, + ProxyHTTPPort: "10266,10267,10255,9100", + ProxyHTTPSPort: "10250,9445", + }, + TunnelConfig: ravenv1beta1.TunnelConfiguration{ + Replicas: 1, + }, + Endpoints: []ravenv1beta1.Endpoint{ + { + NodeName: Node1Name, + Type: ravenv1beta1.Proxy, + UnderNAT: false, + }, + { + NodeName: Node2Name, + Type: ravenv1beta1.Proxy, + UnderNAT: false, + }, + }, + ExposeType: ravenv1beta1.ExposeTypeLoadBalancer, + }, + Status: ravenv1beta1.GatewayStatus{ + Nodes: []ravenv1beta1.NodeInfo{ + { + NodeName: Node1Name, + PrivateIP: Node1Address, + }, + { + NodeName: Node2Name, + PrivateIP: Node2Address, + }, + }, + ActiveEndpoints: []*ravenv1beta1.Endpoint{ + { + NodeName: Node1Name, + Type: ravenv1beta1.Proxy, + UnderNAT: false, + }, + { + NodeName: Node2Name, + Type: ravenv1beta1.Proxy, + UnderNAT: false, + }, + }, + }, + }, + }, + } + objs := []runtime.Object{nodeList, gateways, configmaps} + scheme := runtime.NewScheme() + err := clientgoscheme.AddToScheme(scheme) + if err != nil { + return nil + } + err = apis.AddToScheme(scheme) + if err != nil { + return nil + } + + return &ReconcileService{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objs...).Build(), + recorder: record.NewFakeRecorder(100), + option: utils.NewOption(), + } +} + +func TestReconcileService_Reconcile(t *testing.T) { + r := MockReconcile() + _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: types.NamespacedName{Name: utils.GatewayProxyInternalService, Namespace: utils.WorkingNamespace}}) + if err != nil { + t.Errorf("failed to reconcile service %s/%s", utils.WorkingNamespace, utils.GatewayProxyInternalService) + } +} diff --git a/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_enqueue_handlers.go b/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_enqueue_handlers.go new file mode 100644 index 00000000000..6efe3157eea --- /dev/null +++ b/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_enqueue_handlers.go @@ -0,0 +1,147 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gatewayinternalservice + +import ( + "net" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/event" + + ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/utils" +) + +type EnqueueRequestForGatewayEvent struct{} + +func (h *EnqueueRequestForGatewayEvent) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) { + gw, ok := e.Object.(*ravenv1beta1.Gateway) + if !ok { + klog.Error(Format("fail to assert runtime Object to v1alpha1.Gateway")) + return + } + if gw.Spec.ExposeType == "" { + return + } + klog.V(2).Infof(Format("enqueue service %s/%s due to gateway %s create event", utils.WorkingNamespace, utils.GatewayProxyInternalService, gw.GetName())) + utils.AddGatewayProxyInternalService(q) +} + +func (h *EnqueueRequestForGatewayEvent) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) { + newGw, ok := e.ObjectNew.(*ravenv1beta1.Gateway) + if !ok { + klog.Error(Format("fail to assert runtime Object to v1alpha1.Gateway")) + return + } + oldGw, ok := e.ObjectOld.(*ravenv1beta1.Gateway) + if !ok { + klog.Error(Format("fail to assert runtime Object to v1alpha1.Gateway")) + return + } + if oldGw.Spec.ExposeType == "" && newGw.Spec.ExposeType == "" { + return + } + klog.V(2).Infof(Format("enqueue service %s/%s due to gateway %s update event", utils.WorkingNamespace, utils.GatewayProxyInternalService, newGw.GetName())) + utils.AddGatewayProxyInternalService(q) +} + +func (h *EnqueueRequestForGatewayEvent) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) { + gw, ok := e.Object.(*ravenv1beta1.Gateway) + if !ok { + klog.Error(Format("fail to assert runtime Object to v1alpha1.Gateway")) + return + } + if gw.Spec.ExposeType == "" { + return + } + klog.V(2).Infof(Format("enqueue service %s/%s due to gateway %s delete event", utils.WorkingNamespace, utils.GatewayProxyInternalService, gw.GetName())) + utils.AddGatewayProxyInternalService(q) +} + +func (h *EnqueueRequestForGatewayEvent) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) { + return +} + +type EnqueueRequestForConfigEvent struct{} + +func (h *EnqueueRequestForConfigEvent) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) { + cm, ok := e.Object.(*corev1.ConfigMap) + if !ok { + klog.Error(Format("fail to assert runtime Object to corev1.Configmap")) + return + } + if cm.Data == nil { + return + } + _, _, err := net.SplitHostPort(cm.Data[utils.ProxyServerInsecurePortKey]) + if err == nil { + klog.V(2).Infof(Format("enqueue service %s/%s due to config %s/%s create event", + utils.WorkingNamespace, utils.GatewayProxyInternalService, utils.WorkingNamespace, utils.RavenAgentConfig)) + utils.AddGatewayProxyInternalService(q) + return + } + _, _, err = net.SplitHostPort(cm.Data[utils.ProxyServerSecurePortKey]) + if err == nil { + klog.V(2).Infof(Format("enqueue service %s/%s due to config %s/%s create event", + utils.WorkingNamespace, utils.GatewayProxyInternalService, utils.WorkingNamespace, utils.RavenAgentConfig)) + utils.AddGatewayProxyInternalService(q) + return + } +} + +func (h *EnqueueRequestForConfigEvent) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) { + newCm, ok := e.ObjectNew.(*corev1.ConfigMap) + if !ok { + klog.Error(Format("fail to assert runtime Object to corev1.Configmap")) + return + } + oldCm, ok := e.ObjectOld.(*corev1.ConfigMap) + if !ok { + klog.Error(Format("fail to assert runtime Object to corev1.Configmap")) + return + } + _, newInsecurePort, newErr := net.SplitHostPort(newCm.Data[utils.ProxyServerInsecurePortKey]) + _, oldInsecurePort, oldErr := net.SplitHostPort(oldCm.Data[utils.ProxyServerInsecurePortKey]) + if newErr == nil && oldErr == nil { + if newInsecurePort != oldInsecurePort { + klog.V(2).Infof(Format("enqueue service %s/%s due to config %s/%s update event", + utils.WorkingNamespace, utils.GatewayProxyInternalService, utils.WorkingNamespace, utils.RavenAgentConfig)) + utils.AddGatewayProxyInternalService(q) + return + } + } + _, newSecurePort, newErr := net.SplitHostPort(newCm.Data[utils.ProxyServerSecurePortKey]) + _, oldSecurePort, oldErr := net.SplitHostPort(oldCm.Data[utils.ProxyServerSecurePortKey]) + if newErr == nil && oldErr == nil { + if newSecurePort != oldSecurePort { + klog.V(2).Infof(Format("enqueue service %s/%s due to config %s/%s update event", + utils.WorkingNamespace, utils.GatewayProxyInternalService, utils.WorkingNamespace, utils.RavenAgentConfig)) + utils.AddGatewayProxyInternalService(q) + return + } + } +} + +func (h *EnqueueRequestForConfigEvent) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) { + return +} + +func (h *EnqueueRequestForConfigEvent) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) { + return +} diff --git a/pkg/yurtmanager/controller/raven/utils/options.go b/pkg/yurtmanager/controller/raven/utils/options.go new file mode 100644 index 00000000000..89d244704b3 --- /dev/null +++ b/pkg/yurtmanager/controller/raven/utils/options.go @@ -0,0 +1,68 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import "sync" + +type Option interface { + SetProxyOption(enable bool) + SetTunnelOption(enable bool) + GetProxyOption() bool + GetTunnelOption() bool + Reset() +} + +type ServerOption struct { + mu sync.RWMutex + enableProxy bool + enableTunnel bool +} + +func NewOption() Option { + return &ServerOption{enableTunnel: false, enableProxy: false} +} + +func (o *ServerOption) SetProxyOption(enable bool) { + o.mu.Lock() + defer o.mu.Unlock() + o.enableProxy = enable +} + +func (o *ServerOption) SetTunnelOption(enable bool) { + o.mu.Lock() + defer o.mu.Unlock() + o.enableTunnel = enable +} + +func (o *ServerOption) GetProxyOption() bool { + o.mu.Lock() + defer o.mu.Unlock() + return o.enableProxy +} + +func (o *ServerOption) GetTunnelOption() bool { + o.mu.Lock() + defer o.mu.Unlock() + return o.enableTunnel +} + +func (o *ServerOption) Reset() { + o.mu.Lock() + defer o.mu.Unlock() + o.enableTunnel = false + o.enableProxy = false +} diff --git a/pkg/yurtmanager/controller/raven/utils/utils.go b/pkg/yurtmanager/controller/raven/utils/utils.go index 7219ebde5d8..a5ecdfe9c89 100644 --- a/pkg/yurtmanager/controller/raven/utils/utils.go +++ b/pkg/yurtmanager/controller/raven/utils/utils.go @@ -18,29 +18,46 @@ package utils import ( "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math/rand" "net" + "strconv" "strings" + "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - - "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" ) const ( WorkingNamespace = "kube-system" RavenGlobalConfig = "raven-cfg" + LabelCurrentGatewayEndpoints = "raven.openyurt.io/endpoints-name" GatewayProxyInternalService = "x-raven-proxy-internal-svc" - GatewayProxyServiceNamePrefix = "x-raven-proxy-svc-" - GatewayTunnelServiceNamePrefix = "x-raven-tunnel-svc-" - RavenProxyNodesConfig = "edge-tunnel-nodes" - ProxyNodesKey = "tunnel-nodes" + GatewayProxyServiceNamePrefix = "x-raven-proxy-svc" + GatewayTunnelServiceNamePrefix = "x-raven-tunnel-svc" + + RavenProxyNodesConfig = "edge-tunnel-nodes" + ProxyNodesKey = "tunnel-nodes" + RavenAgentConfig = "raven-agent-config" + ProxyServerSecurePortKey = "proxy-internal-secure-addr" + ProxyServerInsecurePortKey = "proxy-internal-insecure-addr" + ProxyServerExposedPortKey = "proxy-external-addr" + VPNServerExposedPortKey = "tunnel-bind-addr" + RavenEnableProxy = "EnableL7Proxy" + RavenEnableTunnel = "EnableL3Tunnel" - RavenEnableProxy = "EnableL7Proxy" - RavenEnableTunnel = "EnableL3Tunnel" + KubeletSecurePort = 10250 + KubeletInsecurePort = 10255 + PrometheusSecurePort = 9100 + PrometheusInsecurePort = 9445 ) // GetNodeInternalIP returns internal ip of the given `node`. @@ -55,10 +72,6 @@ func GetNodeInternalIP(node corev1.Node) string { return ip } -func IsGatewayExposeByLB(gateway *v1alpha1.Gateway) bool { - return gateway.Spec.ExposeType == v1alpha1.ExposeTypeLoadBalancer -} - // AddGatewayToWorkQueue adds the Gateway the reconciler's workqueue func AddGatewayToWorkQueue(gwName string, q workqueue.RateLimitingInterface) { @@ -86,8 +99,63 @@ func CheckServer(ctx context.Context, client client.Client) (enableProxy, enable return enableProxy, enableTunnel } +func AddNodePoolToWorkQueue(npName string, q workqueue.RateLimitingInterface) { + if npName != "" { + q.Add(reconcile.Request{ + NamespacedName: types.NamespacedName{Name: npName}, + }) + } +} + func AddDNSConfigmapToWorkQueue(q workqueue.RateLimitingInterface) { q.Add(reconcile.Request{ NamespacedName: types.NamespacedName{Namespace: WorkingNamespace, Name: RavenProxyNodesConfig}, }) } + +func AddGatewayProxyInternalService(q workqueue.RateLimitingInterface) { + q.Add(reconcile.Request{ + NamespacedName: types.NamespacedName{Namespace: WorkingNamespace, Name: GatewayProxyInternalService}, + }) +} + +func IsValidPort(s string) bool { + if s == "" { + return false + } + port, err := strconv.Atoi(s) + if err != nil { + return false + } + if port < 0 || port > 65535 { + return false + } + return true +} + +func HashObject(o interface{}) string { + data, _ := json.Marshal(o) + var a interface{} + err := json.Unmarshal(data, &a) + if err != nil { + klog.Errorf("unmarshal: %s", err.Error()) + } + return computeHash(PrettyYaml(a)) +} + +func PrettyYaml(obj interface{}) string { + bs, err := yaml.Marshal(obj) + if err != nil { + klog.Errorf("failed to parse yaml, %v", err.Error()) + } + return string(bs) +} + +func computeHash(target string) string { + hash := sha256.Sum224([]byte(target)) + return strings.ToLower(hex.EncodeToString(hash[:])) +} + +func FormatName(name string) string { + return strings.Join([]string{name, fmt.Sprintf("%08x", rand.Uint32())}, "-") +} From 16e3a3455caf4dcbc6b5a2e9e1ef55a291a10363 Mon Sep 17 00:00:00 2001 From: Liang Deng <283304489@qq.com> Date: Thu, 31 Aug 2023 12:26:36 +0800 Subject: [PATCH 04/13] feat: add token format checking to yurtadm join process (#1681) Signed-off-by: Liang Deng <283304489@qq.com> --- pkg/yurtadm/cmd/join/join.go | 4 ++++ pkg/yurtadm/constants/constants.go | 2 ++ pkg/yurtadm/util/kubernetes/kubernetes.go | 10 ++++++++++ 3 files changed, 16 insertions(+) diff --git a/pkg/yurtadm/cmd/join/join.go b/pkg/yurtadm/cmd/join/join.go index 9e554abb945..89a855e8500 100644 --- a/pkg/yurtadm/cmd/join/join.go +++ b/pkg/yurtadm/cmd/join/join.go @@ -261,6 +261,10 @@ func newJoinData(args []string, opt *joinOptions) (*joinData, error) { return nil, errors.New("join token is empty, so unable to bootstrap worker node.") } + if !yurtadmutil.IsValidBootstrapToken(opt.token) { + return nil, errors.Errorf("the bootstrap token %s was not of the form %s", opt.token, yurtconstants.BootstrapTokenPattern) + } + if opt.nodeType != yurtconstants.EdgeNode && opt.nodeType != yurtconstants.CloudNode { return nil, errors.Errorf("node type(%s) is invalid, only \"edge and cloud\" are supported", opt.nodeType) } diff --git a/pkg/yurtadm/constants/constants.go b/pkg/yurtadm/constants/constants.go index 77f7e63485e..60ae0eb3511 100644 --- a/pkg/yurtadm/constants/constants.go +++ b/pkg/yurtadm/constants/constants.go @@ -57,6 +57,8 @@ const ( KubeletHostname = "--hostname-override=[^\"\\s]*" KubeletEnvironmentFile = "EnvironmentFile=.*" + BootstrapTokenPattern = `\A([a-z0-9]{6})\.([a-z0-9]{16})\z` + DaemonReload = "systemctl daemon-reload" RestartKubeletSvc = "systemctl restart kubelet" diff --git a/pkg/yurtadm/util/kubernetes/kubernetes.go b/pkg/yurtadm/util/kubernetes/kubernetes.go index 17a0c51250b..472c89b33d6 100644 --- a/pkg/yurtadm/util/kubernetes/kubernetes.go +++ b/pkg/yurtadm/util/kubernetes/kubernetes.go @@ -25,6 +25,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "runtime" "strings" "time" @@ -62,6 +63,9 @@ var ( PropagationPolicy = metav1.DeletePropagationBackground ErrClusterVersionEmpty = errors.New("cluster version should not be empty") + + // BootstrapTokenRegexp is a compiled regular expression of TokenRegexpString + BootstrapTokenRegexp = regexp.MustCompile(constants.BootstrapTokenPattern) ) // RunJobAndCleanup runs the job, wait for it to be complete, and delete it @@ -541,3 +545,9 @@ func GetDefaultClientSet() (*kubernetes.Clientset, error) { } return cliSet, nil } + +// IsValidBootstrapToken returns whether the given string is valid as a Bootstrap Token and +// in other words satisfies the BootstrapTokenRegexp +func IsValidBootstrapToken(token string) bool { + return BootstrapTokenRegexp.MatchString(token) +} From 1a5366bd7062b7d9a0e46ded4e3783bfa6679a14 Mon Sep 17 00:00:00 2001 From: wesleysu <59680532+River-sh@users.noreply.github.com> Date: Fri, 1 Sep 2023 09:48:36 +0800 Subject: [PATCH 05/13] add gateway public service controller (#1685) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 珩轩 --- pkg/yurtmanager/controller/controller.go | 2 + pkg/yurtmanager/controller/raven/common.go | 2 +- ...teway_internal_service_enqueue_handlers.go | 14 +- .../gateway_pickup_controller.go | 19 + .../gateway_public_service_controller.go | 695 ++++++++++++++++++ .../gateway_public_service_controller_test.go | 217 ++++++ ...gateway_public_service_enqueue_handlers.go | 163 ++++ .../controller/raven/utils/utils.go | 9 +- pkg/yurtmanager/webhook/server.go | 2 +- 9 files changed, 1107 insertions(+), 16 deletions(-) create mode 100644 pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller.go create mode 100644 pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller_test.go create mode 100644 pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_enqueue_handlers.go diff --git a/pkg/yurtmanager/controller/controller.go b/pkg/yurtmanager/controller/controller.go index 2e8dd41dc39..06f16c73b13 100644 --- a/pkg/yurtmanager/controller/controller.go +++ b/pkg/yurtmanager/controller/controller.go @@ -30,6 +30,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/dns" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewayinternalservice" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewaypickup" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewaypublicservice" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology" servicetopologyendpoints "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/endpoints" servicetopologyendpointslice "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/endpointslice" @@ -68,6 +69,7 @@ func init() { controllerAddFuncs[raven.GatewayPickupControllerName] = []AddControllerFn{gatewaypickup.Add} controllerAddFuncs[raven.GatewayDNSControllerName] = []AddControllerFn{dns.Add} controllerAddFuncs[raven.GatewayInternalServiceController] = []AddControllerFn{gatewayinternalservice.Add} + controllerAddFuncs[raven.GatewayPublicServiceController] = []AddControllerFn{gatewaypublicservice.Add} } // If you want to add additional RBAC, enter it here !!! @kadisi diff --git a/pkg/yurtmanager/controller/raven/common.go b/pkg/yurtmanager/controller/raven/common.go index 7a3b1a59567..9b94c11f0fc 100644 --- a/pkg/yurtmanager/controller/raven/common.go +++ b/pkg/yurtmanager/controller/raven/common.go @@ -21,8 +21,8 @@ var ( ) const ( - ControllerName = "gateway" GatewayPickupControllerName = "raven-gateway-pickup" GatewayInternalServiceController = "raven-gateway-internal-service" + GatewayPublicServiceController = "raven-gateway-public-service" GatewayDNSControllerName = "raven-dns" ) diff --git a/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_enqueue_handlers.go b/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_enqueue_handlers.go index 6efe3157eea..3c888edb4c4 100644 --- a/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_enqueue_handlers.go +++ b/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_enqueue_handlers.go @@ -33,7 +33,7 @@ type EnqueueRequestForGatewayEvent struct{} func (h *EnqueueRequestForGatewayEvent) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) { gw, ok := e.Object.(*ravenv1beta1.Gateway) if !ok { - klog.Error(Format("fail to assert runtime Object to v1alpha1.Gateway")) + klog.Error(Format("fail to assert runtime Object %s/%s to v1beta1.Gateway", e.Object.GetNamespace(), e.Object.GetName())) return } if gw.Spec.ExposeType == "" { @@ -46,12 +46,12 @@ func (h *EnqueueRequestForGatewayEvent) Create(e event.CreateEvent, q workqueue. func (h *EnqueueRequestForGatewayEvent) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) { newGw, ok := e.ObjectNew.(*ravenv1beta1.Gateway) if !ok { - klog.Error(Format("fail to assert runtime Object to v1alpha1.Gateway")) + klog.Error(Format("fail to assert runtime Object %s/%s to v1beta1.Gateway", e.ObjectNew.GetNamespace(), e.ObjectNew.GetName())) return } oldGw, ok := e.ObjectOld.(*ravenv1beta1.Gateway) if !ok { - klog.Error(Format("fail to assert runtime Object to v1alpha1.Gateway")) + klog.Error(Format("fail to assert runtime Object %s/%s to v1beta1.Gateway", e.ObjectOld.GetNamespace(), e.ObjectOld.GetName())) return } if oldGw.Spec.ExposeType == "" && newGw.Spec.ExposeType == "" { @@ -64,7 +64,7 @@ func (h *EnqueueRequestForGatewayEvent) Update(e event.UpdateEvent, q workqueue. func (h *EnqueueRequestForGatewayEvent) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) { gw, ok := e.Object.(*ravenv1beta1.Gateway) if !ok { - klog.Error(Format("fail to assert runtime Object to v1alpha1.Gateway")) + klog.Error(Format("fail to assert runtime Object %s/%s to v1beta1.Gateway", e.Object.GetNamespace(), e.Object.GetName())) return } if gw.Spec.ExposeType == "" { @@ -83,7 +83,7 @@ type EnqueueRequestForConfigEvent struct{} func (h *EnqueueRequestForConfigEvent) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) { cm, ok := e.Object.(*corev1.ConfigMap) if !ok { - klog.Error(Format("fail to assert runtime Object to corev1.Configmap")) + klog.Error(Format("fail to assert runtime Object %s/%s to v1.Configmap", e.Object.GetNamespace(), e.Object.GetName())) return } if cm.Data == nil { @@ -108,12 +108,12 @@ func (h *EnqueueRequestForConfigEvent) Create(e event.CreateEvent, q workqueue.R func (h *EnqueueRequestForConfigEvent) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) { newCm, ok := e.ObjectNew.(*corev1.ConfigMap) if !ok { - klog.Error(Format("fail to assert runtime Object to corev1.Configmap")) + klog.Error(Format("fail to assert runtime Object %s/%s to v1.Configmap", e.ObjectNew.GetNamespace(), e.ObjectNew.GetName())) return } oldCm, ok := e.ObjectOld.(*corev1.ConfigMap) if !ok { - klog.Error(Format("fail to assert runtime Object to corev1.Configmap")) + klog.Error(Format("fail to assert runtime Object %s/%s to v1.Configmap", e.ObjectOld.GetNamespace(), e.ObjectOld.GetName())) return } _, newInsecurePort, newErr := net.SplitHostPort(newCm.Data[utils.ProxyServerInsecurePortKey]) diff --git a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go index 2449cd775a3..3f56bb8ea0f 100644 --- a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go +++ b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go @@ -21,6 +21,7 @@ import ( "fmt" "reflect" "sort" + "strconv" "strings" "time" @@ -180,6 +181,7 @@ func (r *ReconcileGateway) Reconcile(ctx context.Context, req reconcile.Request) activeEp := r.electActiveEndpoint(nodeList, &gw) r.recordEndpointEvent(&gw, gw.Status.ActiveEndpoints, activeEp) gw.Status.ActiveEndpoints = activeEp + r.configEndpoints(ctx, &gw) // 2. get nodeInfo list of nodes managed by the Gateway var nodes []ravenv1beta1.NodeInfo for _, v := range nodeList.Items { @@ -363,3 +365,20 @@ func getActiveEndpointsInfo(eps []*ravenv1beta1.Endpoint) (map[string][]string, } return infos, len(infos[ActiveEndpointsName]) } + +func (r *ReconcileGateway) configEndpoints(ctx context.Context, gw *ravenv1beta1.Gateway) { + enableProxy, enableTunnel := utils.CheckServer(ctx, r.Client) + for idx, val := range gw.Status.ActiveEndpoints { + if gw.Status.ActiveEndpoints[idx].Config == nil { + gw.Status.ActiveEndpoints[idx].Config = make(map[string]string) + } + switch val.Type { + case ravenv1beta1.Proxy: + gw.Status.ActiveEndpoints[idx].Config[utils.RavenEnableProxy] = strconv.FormatBool(enableProxy) + case ravenv1beta1.Tunnel: + gw.Status.ActiveEndpoints[idx].Config[utils.RavenEnableTunnel] = strconv.FormatBool(enableTunnel) + default: + } + } + return +} diff --git a/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller.go b/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller.go new file mode 100644 index 00000000000..0bc1d19ed09 --- /dev/null +++ b/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller.go @@ -0,0 +1,695 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gatewaypublicservice + +import ( + "context" + "fmt" + "net" + "strconv" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/pkg/apis/raven" + ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + common "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/utils" +) + +const ( + ServiceDeleteFailed = "DeleteServiceFail" +) + +func Format(format string, args ...interface{}) string { + s := fmt.Sprintf(format, args...) + return fmt.Sprintf("%s: %s", common.GatewayPublicServiceController, s) +} + +// Add creates a new Service Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { + return add(mgr, newReconciler(mgr)) +} + +var _ reconcile.Reconciler = &ReconcileService{} + +type serviceInformation struct { + mu sync.Mutex + data map[string]string +} + +func newServiceInfo() *serviceInformation { + return &serviceInformation{data: make(map[string]string, 0)} +} +func (s *serviceInformation) write(key, val string) { + s.mu.Lock() + defer s.mu.Unlock() + s.data[key] = val +} + +func (s *serviceInformation) read(key string) string { + s.mu.Lock() + defer s.mu.Unlock() + return s.data[key] +} + +func (s *serviceInformation) cleanup() { + s.mu.Lock() + defer s.mu.Unlock() + s.data = make(map[string]string) +} + +// ReconcileService reconciles a Gateway object +type ReconcileService struct { + client.Client + scheme *runtime.Scheme + recorder record.EventRecorder + option utils.Option + svcInfo *serviceInformation +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + return &ReconcileService{ + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + recorder: mgr.GetEventRecorderFor(common.GatewayPublicServiceController), + option: utils.NewOption(), + svcInfo: newServiceInfo(), + } +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New(common.GatewayPublicServiceController, mgr, controller.Options{ + Reconciler: r, MaxConcurrentReconciles: common.ConcurrentReconciles, + }) + if err != nil { + return err + } + + // Watch for changes to Gateway + err = c.Watch(&source.Kind{Type: &ravenv1beta1.Gateway{}}, &EnqueueRequestForGatewayEvent{}) + if err != nil { + return err + } + + //Watch for changes to raven agent + err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{}}, &EnqueueRequestForConfigEvent{client: mgr.GetClient()}, predicate.NewPredicateFuncs( + func(object client.Object) bool { + cm, ok := object.(*corev1.ConfigMap) + if !ok { + return false + } + if cm.GetNamespace() != utils.WorkingNamespace { + return false + } + if cm.GetName() != utils.RavenAgentConfig { + return false + } + return true + }, + )) + if err != nil { + return err + } + return nil +} + +// Reconcile reads that state of the cluster for a Gateway object and makes changes based on the state read +// and what is in the Gateway.Spec +func (r *ReconcileService) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + klog.V(2).Info(Format("started reconciling public service for gateway %s", req.Name)) + defer klog.V(2).Info(Format("finished reconciling public service for gateway %s", req.Name)) + gw, err := r.getGateway(ctx, req) + if err != nil && !apierrs.IsNotFound(err) { + klog.Error(Format("failed to get gateway %s, error %s", req.Name, err.Error())) + return reconcile.Result{}, err + } + if apierrs.IsNotFound(err) { + gw = &ravenv1beta1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: req.Name}} + } + r.svcInfo.cleanup() + r.setOptions(ctx, gw, apierrs.IsNotFound(err)) + if err := r.reconcileService(ctx, gw.DeepCopy()); err != nil { + err = fmt.Errorf(Format("unable to reconcile service: %s", err)) + klog.Error(err.Error()) + return reconcile.Result{Requeue: true, RequeueAfter: 2 * time.Second}, err + } + + if err := r.reconcileEndpoints(ctx, gw.DeepCopy()); err != nil { + err = fmt.Errorf(Format("unable to reconcile endpoint: %s", err)) + return reconcile.Result{Requeue: true, RequeueAfter: 2 * time.Second}, err + } + return reconcile.Result{}, nil +} + +func (r *ReconcileService) setOptions(ctx context.Context, gw *ravenv1beta1.Gateway, isNotFound bool) { + r.option.SetProxyOption(true) + r.option.SetTunnelOption(true) + if isNotFound { + r.option.SetProxyOption(false) + r.option.SetTunnelOption(false) + klog.V(4).Info(Format("set option for proxy (%t) and tunnel (%t), reason gateway %s is not found", + false, false, gw.GetName())) + return + } + + if gw.DeletionTimestamp != nil { + r.option.SetProxyOption(false) + r.option.SetTunnelOption(false) + klog.V(4).Info(Format("set option for proxy (%t) and tunnel (%t), reason: gateway %s is deleted ", + false, false, gw.GetName())) + return + } + + if gw.Spec.ExposeType != ravenv1beta1.ExposeTypeLoadBalancer { + r.option.SetProxyOption(false) + r.option.SetTunnelOption(false) + klog.V(4).Info(Format("set option for proxy (%t) and tunnel (%t), reason: gateway %s exposed type is %s ", + false, false, gw.GetName(), gw.Spec.ExposeType)) + return + } + + enableProxy, enableTunnel := utils.CheckServer(ctx, r.Client) + if !enableTunnel { + r.option.SetTunnelOption(enableTunnel) + klog.V(4).Info(Format("set option for tunnel (%t), reason: raven-cfg close tunnel ", false)) + } + + if !enableProxy { + r.option.SetProxyOption(enableProxy) + klog.V(4).Info(Format("set option for tunnel (%t), reason: raven-cfg close proxy ", false)) + } + return +} + +func (r *ReconcileService) getGateway(ctx context.Context, req reconcile.Request) (*ravenv1beta1.Gateway, error) { + var gw ravenv1beta1.Gateway + err := r.Get(ctx, req.NamespacedName, &gw) + if err != nil { + return nil, err + } + return gw.DeepCopy(), nil +} + +func (r *ReconcileService) generateServiceName(services []corev1.Service) { + for _, svc := range services { + epName := svc.Labels[utils.LabelCurrentGatewayEndpoints] + epType := svc.Labels[raven.LabelCurrentGatewayType] + if epName == "" || epType == "" { + continue + } + r.svcInfo.write(formatKey(epName, epType), svc.GetName()) + } + return +} + +func (r *ReconcileService) reconcileService(ctx context.Context, gw *ravenv1beta1.Gateway) error { + enableProxy := r.option.GetProxyOption() + if enableProxy { + klog.V(2).Info(Format("start manage proxy service for gateway %s", gw.GetName())) + defer klog.V(2).Info(Format("finish manage proxy service for gateway %s", gw.GetName())) + if err := r.manageService(ctx, gw, ravenv1beta1.Proxy); err != nil { + return fmt.Errorf("failed to manage service for proxy server %s", err.Error()) + } + } else { + klog.V(2).Info(Format("start clear proxy service for gateway %s", gw.GetName())) + defer klog.V(2).Info(Format("finish clear proxy service for gateway %s", gw.GetName())) + if err := r.clearService(ctx, gw.GetName(), ravenv1beta1.Proxy); err != nil { + return fmt.Errorf("failed to clear service for proxy server %s", err.Error()) + } + } + + enableTunnel := r.option.GetTunnelOption() + if enableTunnel { + klog.V(2).Info(Format("start manage tunnel service for gateway %s", gw.GetName())) + defer klog.V(2).Info(Format("finish manage tunnel service for gateway %s", gw.GetName())) + if err := r.manageService(ctx, gw, ravenv1beta1.Tunnel); err != nil { + return fmt.Errorf("failed to manage service for tunnel server %s", err.Error()) + } + } else { + klog.V(2).Info(Format("start clear tunnel service for gateway %s", gw.GetName())) + defer klog.V(2).Info(Format("finish clear tunnel service for gateway %s", gw.GetName())) + if err := r.clearService(ctx, gw.GetName(), ravenv1beta1.Tunnel); err != nil { + return fmt.Errorf("failed to clear service for tunnel server %s", err.Error()) + } + } + return nil +} + +func (r *ReconcileService) reconcileEndpoints(ctx context.Context, gw *ravenv1beta1.Gateway) error { + enableProxy := r.option.GetProxyOption() + if enableProxy { + klog.V(2).Info(Format("start manage proxy service endpoints for gateway %s", gw.GetName())) + defer klog.V(2).Info(Format("finish manage proxy service endpoints for gateway %s", gw.GetName())) + if err := r.manageEndpoints(ctx, gw, ravenv1beta1.Proxy); err != nil { + return fmt.Errorf("failed to manage endpoints for proxy server %s", err.Error()) + } + } else { + klog.V(2).Info(Format("start clear proxy service endpoints for gateway %s", gw.GetName())) + defer klog.V(2).Info(Format("finish clear proxy service endpoints for gateway %s", gw.GetName())) + if err := r.clearEndpoints(ctx, gw.GetName(), ravenv1beta1.Proxy); err != nil { + return fmt.Errorf("failed to clear endpoints for proxy server %s", err.Error()) + } + } + enableTunnel := r.option.GetTunnelOption() + if enableTunnel { + klog.V(2).Info(Format("start manage tunnel service endpoints for gateway %s", gw.GetName())) + defer klog.V(2).Info(Format("finish manage tunnel service endpoints for gateway %s", gw.GetName())) + if err := r.manageEndpoints(ctx, gw, ravenv1beta1.Tunnel); err != nil { + return fmt.Errorf("failed to manage endpoints for tunnel server %s", err.Error()) + } + } else { + klog.V(2).Info(Format("start clear tunnel service endpoints for gateway %s", gw.GetName())) + defer klog.V(2).Info(Format("finish clear tunnel service endpoints for gateway %s", gw.GetName())) + if err := r.clearEndpoints(ctx, gw.GetName(), ravenv1beta1.Tunnel); err != nil { + return fmt.Errorf("failed to clear endpoints for tunnel server %s", err.Error()) + } + } + return nil +} + +func (r *ReconcileService) clearService(ctx context.Context, gatewayName, gatewayType string) error { + svcList, err := r.listService(ctx, gatewayName, gatewayType) + if err != nil { + return fmt.Errorf("failed to list service for gateway %s", gatewayName) + } + for _, svc := range svcList.Items { + err := r.Delete(ctx, svc.DeepCopy()) + if err != nil { + r.recorder.Event(svc.DeepCopy(), corev1.EventTypeWarning, ServiceDeleteFailed, + fmt.Sprintf("The gateway %s %s server is not need to exposed by loadbalancer, failed to delete service %s/%s", + gatewayName, gatewayType, svc.GetNamespace(), svc.GetName())) + continue + } + } + return nil +} + +func (r *ReconcileService) clearEndpoints(ctx context.Context, gatewayName, gatewayType string) error { + epsList, err := r.listEndpoints(ctx, gatewayName, gatewayType) + if err != nil { + return fmt.Errorf("failed to list endpoints for gateway %s", gatewayName) + } + for _, eps := range epsList.Items { + err := r.Delete(ctx, eps.DeepCopy()) + if err != nil { + r.recorder.Event(eps.DeepCopy(), corev1.EventTypeWarning, ServiceDeleteFailed, + fmt.Sprintf("The gateway %s %s server is not need to exposed by loadbalancer, failed to delete endpoints %s/%s", + gatewayName, gatewayType, eps.GetNamespace(), eps.GetName())) + continue + } + } + return nil +} + +func (r *ReconcileService) manageService(ctx context.Context, gateway *ravenv1beta1.Gateway, gatewayType string) error { + curSvcList, err := r.listService(ctx, gateway.GetName(), gatewayType) + if err != nil { + return fmt.Errorf("failed list service for gateway %s type %s , error %s", gateway.GetName(), gatewayType, err.Error()) + } + proxyPort, tunnelPort := r.getTargetPort() + specSvcList := acquiredSpecService(gateway, gatewayType, proxyPort, tunnelPort) + addSvc, updateSvc, deleteSvc := classifyService(curSvcList, specSvcList) + r.generateServiceName(specSvcList.Items) + for i := 0; i < len(addSvc); i++ { + if err := r.Create(ctx, addSvc[i]); err != nil { + if apierrs.IsAlreadyExists(err) { + klog.V(2).Info(Format("service %s/%s has already exist, ignore creating it", addSvc[i].GetNamespace(), addSvc[i].GetName())) + return nil + } + return fmt.Errorf("failed create service for gateway %s type %s , error %s", gateway.GetName(), gatewayType, err.Error()) + } + } + for i := 0; i < len(updateSvc); i++ { + if err := r.Update(ctx, updateSvc[i]); err != nil { + return fmt.Errorf("failed update service for gateway %s type %s , error %s", gateway.GetName(), gatewayType, err.Error()) + } + } + for i := 0; i < len(deleteSvc); i++ { + if err := r.Delete(ctx, deleteSvc[i]); err != nil { + return fmt.Errorf("failed delete service for gateway %s type %s , error %s", gateway.GetName(), gatewayType, err.Error()) + } + } + return nil +} + +func (r *ReconcileService) manageEndpoints(ctx context.Context, gateway *ravenv1beta1.Gateway, gatewayType string) error { + currEpsList, err := r.listEndpoints(ctx, gateway.GetName(), gatewayType) + if err != nil { + return fmt.Errorf("failed list service for gateway %s type %s , error %s", gateway.GetName(), gatewayType, err.Error()) + } + specEpsList := r.acquiredSpecEndpoints(ctx, gateway, gatewayType) + addEps, updateEps, deleteEps := classifyEndpoints(currEpsList, specEpsList) + for i := 0; i < len(addEps); i++ { + if err := r.Create(ctx, addEps[i]); err != nil { + if apierrs.IsAlreadyExists(err) { + klog.V(2).Info(Format("endpoints %s/%s has already exist, ignore creating it", addEps[i].GetNamespace(), addEps[i].GetName())) + return nil + } + return fmt.Errorf("failed create endpoints for gateway %s type %s , error %s", gateway.GetName(), gatewayType, err.Error()) + } + } + for i := 0; i < len(updateEps); i++ { + if err := r.Update(ctx, updateEps[i]); err != nil { + return fmt.Errorf("failed update endpoints for gateway %s type %s , error %s", gateway.GetName(), gatewayType, err.Error()) + } + } + for i := 0; i < len(deleteEps); i++ { + if err := r.Delete(ctx, deleteEps[i]); err != nil { + return fmt.Errorf("failed delete endpoints for gateway %s type %s , error %s", gateway.GetName(), gatewayType, err.Error()) + } + } + return nil +} + +func (r *ReconcileService) getTargetPort() (proxyPort, tunnelPort int32) { + proxyPort = ravenv1beta1.DefaultProxyServerExposedPort + tunnelPort = ravenv1beta1.DefaultTunnelServerExposedPort + var cm corev1.ConfigMap + err := r.Get(context.TODO(), types.NamespacedName{Namespace: utils.WorkingNamespace, Name: utils.RavenAgentConfig}, &cm) + if err != nil { + return + } + if cm.Data == nil { + return + } + _, proxyExposedPort, err := net.SplitHostPort(cm.Data[utils.ProxyServerExposedPortKey]) + if err == nil { + proxy, _ := strconv.Atoi(proxyExposedPort) + proxyPort = int32(proxy) + } + _, tunnelExposedPort, err := net.SplitHostPort(cm.Data[utils.VPNServerExposedPortKey]) + if err == nil { + tunnel, _ := strconv.Atoi(tunnelExposedPort) + tunnelPort = int32(tunnel) + } + return +} + +func (r *ReconcileService) listService(ctx context.Context, gatewayName, gatewayType string) (*corev1.ServiceList, error) { + var svcList corev1.ServiceList + err := r.List(ctx, &svcList, &client.ListOptions{ + LabelSelector: labels.Set{ + raven.LabelCurrentGateway: gatewayName, + raven.LabelCurrentGatewayType: gatewayType, + }.AsSelector(), + }) + if err != nil { + return nil, err + } + newList := make([]corev1.Service, 0) + for _, val := range svcList.Items { + if val.Spec.Type == corev1.ServiceTypeLoadBalancer { + newList = append(newList, val) + } + } + svcList.Items = newList + return &svcList, nil +} + +func (r *ReconcileService) listEndpoints(ctx context.Context, gatewayName, gatewayType string) (*corev1.EndpointsList, error) { + var epsList corev1.EndpointsList + err := r.List(ctx, &epsList, &client.ListOptions{ + LabelSelector: labels.Set{ + raven.LabelCurrentGateway: gatewayName, + raven.LabelCurrentGatewayType: gatewayType, + }.AsSelector()}) + if err != nil { + return nil, err + } + return &epsList, nil +} + +func (r *ReconcileService) acquiredSpecEndpoints(ctx context.Context, gateway *ravenv1beta1.Gateway, gatewayType string) *corev1.EndpointsList { + proxyPort, tunnelPort := r.getTargetPort() + endpoints := make([]corev1.Endpoints, 0) + for _, aep := range gateway.Status.ActiveEndpoints { + if aep.Type != gatewayType { + continue + } + address, err := r.getEndpointsAddress(ctx, aep.NodeName) + if err != nil { + continue + } + switch aep.Type { + case ravenv1beta1.Proxy: + name := r.svcInfo.read(formatKey(aep.NodeName, ravenv1beta1.Proxy)) + if name == "" { + continue + } + endpoints = append(endpoints, corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: utils.WorkingNamespace, + Labels: map[string]string{ + raven.LabelCurrentGateway: gateway.GetName(), + raven.LabelCurrentGatewayType: ravenv1beta1.Proxy, + utils.LabelCurrentGatewayEndpoints: aep.NodeName, + }, + }, + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{*address}, + Ports: []corev1.EndpointPort{ + { + Port: proxyPort, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + }) + case ravenv1beta1.Tunnel: + name := r.svcInfo.read(formatKey(aep.NodeName, ravenv1beta1.Tunnel)) + if name == "" { + continue + } + endpoints = append(endpoints, corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: utils.WorkingNamespace, + Labels: map[string]string{ + raven.LabelCurrentGateway: gateway.GetName(), + raven.LabelCurrentGatewayType: ravenv1beta1.Tunnel, + utils.LabelCurrentGatewayEndpoints: aep.NodeName, + }, + }, + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{*address}, + Ports: []corev1.EndpointPort{ + { + Port: tunnelPort, + Protocol: corev1.ProtocolUDP, + }, + }, + }, + }, + }) + } + } + return &corev1.EndpointsList{Items: endpoints} +} + +func (r *ReconcileService) getEndpointsAddress(ctx context.Context, name string) (*corev1.EndpointAddress, error) { + var node corev1.Node + err := r.Get(ctx, types.NamespacedName{Name: name}, &node) + if err != nil { + klog.Errorf(Format("failed to get node %s for get active endpoints address, error %s", name, err.Error())) + return nil, err + } + return &corev1.EndpointAddress{NodeName: func(n corev1.Node) *string { return &n.Name }(node), IP: utils.GetNodeInternalIP(node)}, nil +} + +func acquiredSpecService(gateway *ravenv1beta1.Gateway, gatewayType string, proxyPort, tunnelPort int32) *corev1.ServiceList { + services := make([]corev1.Service, 0) + if gateway == nil { + return &corev1.ServiceList{Items: services} + } + for _, aep := range gateway.Status.ActiveEndpoints { + if aep.Type != gatewayType { + continue + } + if aep.Port < 1 || aep.Port > 65535 { + continue + } + switch aep.Type { + case ravenv1beta1.Proxy: + services = append(services, corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: utils.FormatName(fmt.Sprintf("%s-%s", utils.GatewayProxyServiceNamePrefix, gateway.GetName())), + Namespace: utils.WorkingNamespace, + Labels: map[string]string{ + raven.LabelCurrentGateway: gateway.GetName(), + raven.LabelCurrentGatewayType: ravenv1beta1.Proxy, + utils.LabelCurrentGatewayEndpoints: aep.NodeName, + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal, + Ports: []corev1.ServicePort{ + { + Protocol: corev1.ProtocolTCP, + Port: int32(aep.Port), + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: proxyPort, + }, + }, + }, + }, + }) + case ravenv1beta1.Tunnel: + services = append(services, corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: utils.FormatName(fmt.Sprintf("%s-%s", utils.GatewayTunnelServiceNamePrefix, gateway.GetName())), + Namespace: utils.WorkingNamespace, + Labels: map[string]string{ + raven.LabelCurrentGateway: gateway.GetName(), + raven.LabelCurrentGatewayType: ravenv1beta1.Tunnel, + utils.LabelCurrentGatewayEndpoints: aep.NodeName, + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal, + Ports: []corev1.ServicePort{ + { + Protocol: corev1.ProtocolUDP, + Port: int32(aep.Port), + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: tunnelPort, + }, + }, + }, + }, + }) + } + } + return &corev1.ServiceList{Items: services} +} + +func classifyService(current, spec *corev1.ServiceList) (added, updated, deleted []*corev1.Service) { + added = make([]*corev1.Service, 0) + updated = make([]*corev1.Service, 0) + deleted = make([]*corev1.Service, 0) + + getKey := func(svc *corev1.Service) string { + epType := svc.Labels[raven.LabelCurrentGatewayType] + epName := svc.Labels[utils.LabelCurrentGatewayEndpoints] + if epType == "" { + return "" + } + if epName == "" { + return "" + } + return formatKey(epName, epType) + } + + r := make(map[string]int) + for idx, val := range current.Items { + if key := getKey(&val); key != "" { + r[key] = idx + } + } + for _, val := range spec.Items { + if key := getKey(&val); key != "" { + if idx, ok := r[key]; ok { + updatedService := current.Items[idx].DeepCopy() + updatedService.Spec = *val.Spec.DeepCopy() + updated = append(updated, updatedService) + delete(r, key) + } else { + added = append(added, val.DeepCopy()) + } + } + + } + for key, val := range r { + deleted = append(deleted, current.Items[val].DeepCopy()) + delete(r, key) + } + return added, updated, deleted +} + +func classifyEndpoints(current, spec *corev1.EndpointsList) (added, updated, deleted []*corev1.Endpoints) { + added = make([]*corev1.Endpoints, 0) + updated = make([]*corev1.Endpoints, 0) + deleted = make([]*corev1.Endpoints, 0) + + getKey := func(ep *corev1.Endpoints) string { + epType := ep.Labels[raven.LabelCurrentGatewayType] + epName := ep.Labels[utils.LabelCurrentGatewayEndpoints] + if epType == "" { + return "" + } + if epName == "" { + return "" + } + return formatKey(epName, epType) + } + + r := make(map[string]int) + for idx, val := range current.Items { + if key := getKey(&val); key != "" { + r[key] = idx + } + } + for _, val := range spec.Items { + if key := getKey(&val); key != "" { + if idx, ok := r[key]; ok { + updatedEndpoints := current.Items[idx].DeepCopy() + updatedEndpoints.Subsets = val.DeepCopy().Subsets + updated = append(updated, updatedEndpoints) + delete(r, key) + } else { + added = append(added, val.DeepCopy()) + } + } + } + for key, val := range r { + deleted = append(deleted, current.Items[val].DeepCopy()) + delete(r, key) + } + return added, updated, deleted +} + +func formatKey(endpointName, endpointType string) string { + return fmt.Sprintf("%s-%s", endpointName, endpointType) +} diff --git a/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller_test.go b/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller_test.go new file mode 100644 index 00000000000..6a9e6de30ce --- /dev/null +++ b/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller_test.go @@ -0,0 +1,217 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gatewaypublicservice + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/openyurtio/openyurt/pkg/apis" + "github.com/openyurtio/openyurt/pkg/apis/raven" + ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/utils" +) + +const ( + Node1Name = "node-1" + Node2Name = "node-2" + Node3Name = "node-3" + Node4Name = "node-4" + Node1Address = "192.168.0.1" + Node2Address = "192.168.0.2" + Node3Address = "192.168.0.3" + Node4Address = "192.168.0.4" + MockGateway = "gw-mock" +) + +func MockReconcile() *ReconcileService { + nodeList := &corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: Node1Name, + Labels: map[string]string{ + raven.LabelCurrentGateway: MockGateway, + }, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: Node1Address, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: Node2Name, + Labels: map[string]string{ + raven.LabelCurrentGateway: MockGateway, + }, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: Node2Address, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: Node3Name, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: Node3Address, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: Node4Name, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: Node4Address, + }, + }, + }, + }, + }, + } + configmaps := &corev1.ConfigMapList{ + Items: []corev1.ConfigMap{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RavenGlobalConfig, + Namespace: utils.WorkingNamespace, + }, + Data: map[string]string{ + utils.RavenEnableProxy: "true", + utils.RavenEnableTunnel: "true", + }, + }, + }, + } + gateways := &ravenv1beta1.GatewayList{ + Items: []ravenv1beta1.Gateway{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: MockGateway, + }, + Spec: ravenv1beta1.GatewaySpec{ + NodeSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + raven.LabelCurrentGateway: MockGateway, + }, + }, + ProxyConfig: ravenv1beta1.ProxyConfiguration{ + Replicas: 2, + ProxyHTTPPort: "10266,10267,10255,9100", + ProxyHTTPSPort: "10250,9445", + }, + TunnelConfig: ravenv1beta1.TunnelConfiguration{ + Replicas: 1, + }, + Endpoints: []ravenv1beta1.Endpoint{ + { + NodeName: Node1Name, + Type: ravenv1beta1.Proxy, + Port: ravenv1beta1.DefaultProxyServerExposedPort, + UnderNAT: false, + }, + { + NodeName: Node2Name, + Type: ravenv1beta1.Proxy, + Port: ravenv1beta1.DefaultProxyServerExposedPort, + UnderNAT: false, + }, + }, + ExposeType: ravenv1beta1.ExposeTypeLoadBalancer, + }, + Status: ravenv1beta1.GatewayStatus{ + Nodes: []ravenv1beta1.NodeInfo{ + { + NodeName: Node1Name, + PrivateIP: Node1Address, + }, + { + NodeName: Node2Name, + PrivateIP: Node2Address, + }, + }, + ActiveEndpoints: []*ravenv1beta1.Endpoint{ + { + NodeName: Node1Name, + Type: ravenv1beta1.Proxy, + Port: ravenv1beta1.DefaultProxyServerExposedPort, + UnderNAT: false, + }, + { + NodeName: Node2Name, + Type: ravenv1beta1.Proxy, + Port: ravenv1beta1.DefaultProxyServerExposedPort, + UnderNAT: false, + }, + }, + }, + }, + }, + } + objs := []runtime.Object{nodeList, gateways, configmaps} + scheme := runtime.NewScheme() + err := clientgoscheme.AddToScheme(scheme) + if err != nil { + return nil + } + err = apis.AddToScheme(scheme) + if err != nil { + return nil + } + + return &ReconcileService{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objs...).Build(), + recorder: record.NewFakeRecorder(100), + option: utils.NewOption(), + svcInfo: newServiceInfo(), + } +} + +func TestReconcileService_Reconcile(t *testing.T) { + r := MockReconcile() + _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: types.NamespacedName{Name: MockGateway}}) + if err != nil { + t.Errorf("failed to reconcile service %s", MockGateway) + } +} diff --git a/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_enqueue_handlers.go b/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_enqueue_handlers.go new file mode 100644 index 00000000000..2185365f0ac --- /dev/null +++ b/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_enqueue_handlers.go @@ -0,0 +1,163 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gatewaypublicservice + +import ( + "context" + "net" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + + ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/utils" +) + +type EnqueueRequestForGatewayEvent struct{} + +func (h *EnqueueRequestForGatewayEvent) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) { + gw, ok := e.Object.(*ravenv1beta1.Gateway) + if !ok { + klog.Error(Format("fail to assert runtime Object %s/%s to v1beta1.Gateway,", e.Object.GetNamespace(), e.Object.GetName())) + return + } + if gw.Spec.ExposeType == "" { + return + } + klog.V(2).Infof(Format("enqueue gateway %s as create event", gw.GetName())) + utils.AddGatewayToWorkQueue(gw.GetName(), q) +} + +func (h *EnqueueRequestForGatewayEvent) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) { + newGw, ok := e.ObjectNew.(*ravenv1beta1.Gateway) + if !ok { + klog.Error(Format("fail to assert runtime Object %s/%s to v1beta1.Gateway,", e.ObjectNew.GetNamespace(), e.ObjectNew.GetName())) + return + } + oldGw, ok := e.ObjectOld.(*ravenv1beta1.Gateway) + if !ok { + klog.Error(Format("fail to assert runtime Object %s/%s to v1beta1.Gateway,", e.ObjectOld.GetNamespace(), e.ObjectOld.GetName())) + return + } + if needUpdate(newGw, oldGw) { + klog.V(2).Infof(Format("enqueue gateway %s as update event", newGw.GetName())) + utils.AddGatewayToWorkQueue(newGw.GetName(), q) + } +} + +func (h *EnqueueRequestForGatewayEvent) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) { + gw, ok := e.Object.(*ravenv1beta1.Gateway) + if !ok { + klog.Error(Format("fail to assert runtime Object %s/%s to v1beta1.Gateway,", e.Object.GetNamespace(), e.Object.GetName())) + return + } + if gw.Spec.ExposeType == "" { + return + } + if gw.DeletionTimestamp != nil { + return + } + klog.V(2).Infof(Format("enqueue gateway %s as delete event", gw.GetName())) + utils.AddGatewayToWorkQueue(gw.GetName(), q) +} + +func (h *EnqueueRequestForGatewayEvent) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) { + return +} + +func needUpdate(newObj, oldObj *ravenv1beta1.Gateway) bool { + if newObj.Spec.ExposeType != oldObj.Spec.ExposeType { + return true + } + if utils.HashObject(newObj.Status.ActiveEndpoints) != utils.HashObject(oldObj.Status.ActiveEndpoints) { + return true + } + return false +} + +type EnqueueRequestForConfigEvent struct { + client client.Client +} + +func (h *EnqueueRequestForConfigEvent) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) { + cm, ok := e.Object.(*corev1.ConfigMap) + if !ok { + klog.Error(Format("fail to assert runtime Object %s/%s to v1.Configmap,", e.Object.GetNamespace(), e.Object.GetName())) + return + } + if cm.Data == nil { + return + } + if _, _, err := net.SplitHostPort(cm.Data[utils.ProxyServerExposedPortKey]); err == nil { + h.addExposedGateway(q) + return + } + if _, _, err := net.SplitHostPort(cm.Data[utils.VPNServerExposedPortKey]); err == nil { + h.addExposedGateway(q) + return + } +} + +func (h *EnqueueRequestForConfigEvent) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) { + newCm, ok := e.ObjectNew.(*corev1.ConfigMap) + if !ok { + klog.Error(Format("fail to assert runtime Object %s/%s to v1.Configmap,", e.ObjectNew.GetNamespace(), e.ObjectNew.GetName())) + return + } + oldCm, ok := e.ObjectOld.(*corev1.ConfigMap) + if !ok { + klog.Error(Format("fail to assert runtime Object %s/%s to v1.Configmap,", e.ObjectOld.GetNamespace(), e.ObjectOld.GetName())) + return + } + _, newProxyPort, newErr := net.SplitHostPort(newCm.Data[utils.ProxyServerExposedPortKey]) + _, oldProxyPort, oldErr := net.SplitHostPort(oldCm.Data[utils.ProxyServerExposedPortKey]) + if newErr == nil && oldErr == nil && newProxyPort != oldProxyPort { + h.addExposedGateway(q) + return + } + _, newTunnelPort, newErr := net.SplitHostPort(newCm.Data[utils.VPNServerExposedPortKey]) + _, oldTunnelPort, oldErr := net.SplitHostPort(oldCm.Data[utils.VPNServerExposedPortKey]) + if newErr == nil && oldErr == nil && newTunnelPort != oldTunnelPort { + h.addExposedGateway(q) + return + } +} + +func (h *EnqueueRequestForConfigEvent) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) { + return +} + +func (h *EnqueueRequestForConfigEvent) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) { + return +} + +func (h *EnqueueRequestForConfigEvent) addExposedGateway(q workqueue.RateLimitingInterface) { + var gwList ravenv1beta1.GatewayList + err := h.client.List(context.TODO(), &gwList) + if err != nil { + return + } + for _, gw := range gwList.Items { + if gw.Spec.ExposeType == ravenv1beta1.ExposeTypeLoadBalancer { + klog.V(2).Infof(Format("enqueue gateway %s", gw.GetName())) + utils.AddGatewayToWorkQueue(gw.GetName(), q) + } + } +} diff --git a/pkg/yurtmanager/controller/raven/utils/utils.go b/pkg/yurtmanager/controller/raven/utils/utils.go index a5ecdfe9c89..b47ad097f41 100644 --- a/pkg/yurtmanager/controller/raven/utils/utils.go +++ b/pkg/yurtmanager/controller/raven/utils/utils.go @@ -51,13 +51,8 @@ const ( ProxyServerInsecurePortKey = "proxy-internal-insecure-addr" ProxyServerExposedPortKey = "proxy-external-addr" VPNServerExposedPortKey = "tunnel-bind-addr" - RavenEnableProxy = "EnableL7Proxy" - RavenEnableTunnel = "EnableL3Tunnel" - - KubeletSecurePort = 10250 - KubeletInsecurePort = 10255 - PrometheusSecurePort = 9100 - PrometheusInsecurePort = 9445 + RavenEnableProxy = "enable-l7-proxy" + RavenEnableTunnel = "enable-l3-tunnel" ) // GetNodeInternalIP returns internal ip of the given `node`. diff --git a/pkg/yurtmanager/webhook/server.go b/pkg/yurtmanager/webhook/server.go index a57ed20f149..47547c10a26 100644 --- a/pkg/yurtmanager/webhook/server.go +++ b/pkg/yurtmanager/webhook/server.go @@ -73,7 +73,7 @@ func addControllerWebhook(name string, handler SetupWebhookWithManager) { } func init() { - addControllerWebhook(raven.ControllerName, &v1beta1gateway.GatewayHandler{}) + addControllerWebhook(raven.GatewayPickupControllerName, &v1beta1gateway.GatewayHandler{}) addControllerWebhook(nodepool.ControllerName, &v1beta1nodepool.NodePoolHandler{}) addControllerWebhook(yurtstaticset.ControllerName, &v1alpha1yurtstaticset.YurtStaticSetHandler{}) addControllerWebhook(yurtappset.ControllerName, &v1alpha1yurtappset.YurtAppSetHandler{}) From 8b8c6e11fc5bfc53bf38f8c9c05b48abfca46145 Mon Sep 17 00:00:00 2001 From: rambohe Date: Mon, 4 Sep 2023 09:49:36 +0800 Subject: [PATCH 06/13] improve controller names (#1687) --- charts/yurt-manager/values.yaml | 2 +- cmd/yurt-manager/app/manager.go | 5 +- cmd/yurt-manager/app/options/generic.go | 48 ++++++--- cmd/yurt-manager/app/options/options.go | 18 ++-- cmd/yurt-manager/names/controller_names.go | 58 +++++++++++ go.mod | 1 + go.sum | 1 + pkg/yurtmanager/controller/controller.go | 98 +++++++++++-------- .../csrapprover/csrapprover_controller.go | 7 +- .../daemon_pod_updater_controller.go | 9 +- .../nodepool/nodepool_controller.go | 11 +-- .../platformadmin/platformadmin_controller.go | 9 +- pkg/yurtmanager/controller/raven/common.go | 7 -- .../controller/raven/dns/dns_controller.go | 7 +- .../gateway_internal_service_controller.go | 7 +- .../gateway_pickup_controller.go | 7 +- .../gateway_public_service_controller.go | 7 +- .../controller/servicetopology/common.go | 21 ---- .../endpoints/endpoints_controller.go | 6 +- .../endpointslice/endpointslice_controller.go | 8 +- .../controller/util/controller_utils.go | 34 ------- .../controller/util/controller_utils_test.go | 57 ----------- .../yurtappdaemon/yurtappdaemon_controller.go | 13 ++- .../yurtappdaemon_controller_test.go | 9 +- .../yurtappset/yurtappset_controller.go | 9 +- .../cert/yurtcoordinatorcert_controller.go | 7 +- .../delegatelease/delegatelease_controller.go | 7 +- .../podbinding/podbinding_controller.go | 9 +- .../yurtstaticset/yurtstaticset_controller.go | 9 +- pkg/yurtmanager/webhook/server.go | 26 +++-- test/e2e/cmd/init/constants/constants.go | 1 + 31 files changed, 244 insertions(+), 274 deletions(-) create mode 100644 cmd/yurt-manager/names/controller_names.go delete mode 100644 pkg/yurtmanager/controller/servicetopology/common.go delete mode 100644 pkg/yurtmanager/controller/util/controller_utils.go delete mode 100644 pkg/yurtmanager/controller/util/controller_utils_test.go diff --git a/charts/yurt-manager/values.yaml b/charts/yurt-manager/values.yaml index f6d9eba5fc2..30033aa508a 100644 --- a/charts/yurt-manager/values.yaml +++ b/charts/yurt-manager/values.yaml @@ -21,7 +21,7 @@ ports: webhook: 10273 # format should be "foo,-bar,*" -controllers: "" +controllers: "*" # format should be "foo,*" disableIndependentWebhooks: "" diff --git a/cmd/yurt-manager/app/manager.go b/cmd/yurt-manager/app/manager.go index 05e2a2c40bd..9c11a76449a 100644 --- a/cmd/yurt-manager/app/manager.go +++ b/cmd/yurt-manager/app/manager.go @@ -36,6 +36,7 @@ import ( "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/options" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" "github.com/openyurtio/openyurt/pkg/apis" "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/util/profile" @@ -93,7 +94,7 @@ current state towards the desired state.`, PrintFlags(cmd.Flags()) - c, err := s.Config() + c, err := s.Config(controller.KnownControllers(), names.YurtManagerControllerAliases()) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) @@ -115,7 +116,7 @@ current state towards the desired state.`, } fs := cmd.Flags() - namedFlagSets := s.Flags() + namedFlagSets := s.Flags(controller.KnownControllers(), controller.ControllersDisabledByDefault.List()) // verflag.AddFlags(namedFlagSets.FlagSet("global")) globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name()) for _, f := range namedFlagSets.FlagSets { diff --git a/cmd/yurt-manager/app/options/generic.go b/cmd/yurt-manager/app/options/generic.go index 919fd504d04..783e74dc3ea 100644 --- a/cmd/yurt-manager/app/options/generic.go +++ b/cmd/yurt-manager/app/options/generic.go @@ -18,15 +18,15 @@ package options import ( "fmt" + "strings" "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" "github.com/openyurtio/openyurt/pkg/features" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/apis/config" ) -const enableAll = "*" - type GenericOptions struct { *config.GenericConfiguration } @@ -43,14 +43,13 @@ func NewGenericOptions() *GenericOptions { RestConfigQPS: 30, RestConfigBurst: 50, WorkingNamespace: "kube-system", - Controllers: []string{enableAll}, DisabledWebhooks: []string{}, }, } } // Validate checks validation of GenericOptions. -func (o *GenericOptions) Validate() []error { +func (o *GenericOptions) Validate(allControllers []string, controllerAliases map[string]string) []error { if o == nil { return nil } @@ -59,11 +58,26 @@ func (o *GenericOptions) Validate() []error { if o.WebhookPort == 0 { errs = append(errs, fmt.Errorf("webhook server can not be switched off with 0")) } + + allControllersSet := sets.NewString(allControllers...) + for _, initialName := range o.Controllers { + if initialName == "*" { + continue + } + initialNameWithoutPrefix := strings.TrimPrefix(initialName, "-") + controllerName := initialNameWithoutPrefix + if canonicalName, ok := controllerAliases[controllerName]; ok { + controllerName = canonicalName + } + if !allControllersSet.Has(controllerName) { + errs = append(errs, fmt.Errorf("%q is not in the list of known controllers", initialNameWithoutPrefix)) + } + } return errs } // ApplyTo fills up generic config with options. -func (o *GenericOptions) ApplyTo(cfg *config.GenericConfiguration) error { +func (o *GenericOptions) ApplyTo(cfg *config.GenericConfiguration, controllerAliases map[string]string) error { if o == nil { return nil } @@ -77,14 +91,26 @@ func (o *GenericOptions) ApplyTo(cfg *config.GenericConfiguration) error { cfg.RestConfigQPS = o.RestConfigQPS cfg.RestConfigBurst = o.RestConfigBurst cfg.WorkingNamespace = o.WorkingNamespace - cfg.Controllers = o.Controllers + + cfg.Controllers = make([]string, len(o.Controllers)) + for i, initialName := range o.Controllers { + initialNameWithoutPrefix := strings.TrimPrefix(initialName, "-") + controllerName := initialNameWithoutPrefix + if canonicalName, ok := controllerAliases[controllerName]; ok { + controllerName = canonicalName + } + if strings.HasPrefix(initialName, "-") { + controllerName = fmt.Sprintf("-%s", controllerName) + } + cfg.Controllers[i] = controllerName + } cfg.DisabledWebhooks = o.DisabledWebhooks return nil } // AddFlags adds flags related to generic for yurt-manager to the specified FlagSet. -func (o *GenericOptions) AddFlags(fs *pflag.FlagSet) { +func (o *GenericOptions) AddFlags(fs *pflag.FlagSet, allControllers, disabledByDefaultControllers []string) { if o == nil { return } @@ -94,14 +120,14 @@ func (o *GenericOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.HealthProbeAddr, "health-probe-addr", o.HealthProbeAddr, "The address the healthz/readyz endpoint binds to.") fs.IntVar(&o.WebhookPort, "webhook-port", o.WebhookPort, "The port on which to serve HTTPS for webhook server. It can't be switched off with 0") fs.BoolVar(&o.EnableLeaderElection, "enable-leader-election", o.EnableLeaderElection, "Whether you need to enable leader election.") - fs.IntVar(&o.RestConfigQPS, "rest-config-qps", o.RestConfigQPS, "rest-config-qps.") fs.IntVar(&o.RestConfigBurst, "rest-config-burst", o.RestConfigBurst, "rest-config-burst.") fs.StringVar(&o.WorkingNamespace, "working-namespace", o.WorkingNamespace, "The namespace where the yurt-manager is working.") - fs.StringSliceVar(&o.Controllers, "controllers", o.Controllers, "A list of controllers to enable. "+ - "'*' enables all on-by-default controllers, 'foo' enables the controller named 'foo', '-foo' disables the controller named 'foo'.") + fs.StringSliceVar(&o.Controllers, "controllers", o.Controllers, fmt.Sprintf("A list of controllers to enable. '*' enables all on-by-default controllers, 'foo' enables the controller "+ + "named 'foo', '-foo' disables the controller named 'foo'.\nAll controllers: %s\nDisabled-by-default controllers: %s", + strings.Join(allControllers, ", "), strings.Join(disabledByDefaultControllers, ", "))) fs.StringSliceVar(&o.DisabledWebhooks, "disable-independent-webhooks", o.DisabledWebhooks, "A list of webhooks to disable. "+ - "'*' disables all webhooks, 'foo' disables the webhook named 'foo'.") + "'*' disables all independent webhooks, 'foo' disables the independent webhook named 'foo'.") features.DefaultMutableFeatureGate.AddFlag(fs) } diff --git a/cmd/yurt-manager/app/options/options.go b/cmd/yurt-manager/app/options/options.go index 1d1237b46b1..e0940d30971 100644 --- a/cmd/yurt-manager/app/options/options.go +++ b/cmd/yurt-manager/app/options/options.go @@ -50,9 +50,9 @@ func NewYurtManagerOptions() (*YurtManagerOptions, error) { return &s, nil } -func (y *YurtManagerOptions) Flags() cliflag.NamedFlagSets { +func (y *YurtManagerOptions) Flags(allControllers, disabledByDefaultControllers []string) cliflag.NamedFlagSets { fss := cliflag.NamedFlagSets{} - y.Generic.AddFlags(fss.FlagSet("generic")) + y.Generic.AddFlags(fss.FlagSet("generic"), allControllers, disabledByDefaultControllers) y.NodePoolController.AddFlags(fss.FlagSet("nodepool controller")) y.GatewayPickupController.AddFlags(fss.FlagSet("gateway controller")) y.YurtStaticSetController.AddFlags(fss.FlagSet("yurtstaticset controller")) @@ -64,9 +64,9 @@ func (y *YurtManagerOptions) Flags() cliflag.NamedFlagSets { } // Validate is used to validate the options and config before launching the yurt-manager -func (y *YurtManagerOptions) Validate() error { +func (y *YurtManagerOptions) Validate(allControllers []string, controllerAliases map[string]string) error { var errs []error - errs = append(errs, y.Generic.Validate()...) + errs = append(errs, y.Generic.Validate(allControllers, controllerAliases)...) errs = append(errs, y.NodePoolController.Validate()...) errs = append(errs, y.GatewayPickupController.Validate()...) errs = append(errs, y.YurtStaticSetController.Validate()...) @@ -76,8 +76,8 @@ func (y *YurtManagerOptions) Validate() error { } // ApplyTo fills up yurt manager config with options. -func (y *YurtManagerOptions) ApplyTo(c *config.Config) error { - if err := y.Generic.ApplyTo(&c.ComponentConfig.Generic); err != nil { +func (y *YurtManagerOptions) ApplyTo(c *config.Config, controllerAliases map[string]string) error { + if err := y.Generic.ApplyTo(&c.ComponentConfig.Generic, controllerAliases); err != nil { return err } if err := y.NodePoolController.ApplyTo(&c.ComponentConfig.NodePoolController); err != nil { @@ -100,13 +100,13 @@ func (y *YurtManagerOptions) ApplyTo(c *config.Config) error { } // Config return a yurt-manager config objective -func (y *YurtManagerOptions) Config() (*config.Config, error) { - if err := y.Validate(); err != nil { +func (y *YurtManagerOptions) Config(allControllers []string, controllerAliases map[string]string) (*config.Config, error) { + if err := y.Validate(allControllers, controllerAliases); err != nil { return nil, err } c := &config.Config{} - if err := y.ApplyTo(c); err != nil { + if err := y.ApplyTo(c, controllerAliases); err != nil { return nil, err } diff --git a/cmd/yurt-manager/names/controller_names.go b/cmd/yurt-manager/names/controller_names.go new file mode 100644 index 00000000000..f00dc8d04d0 --- /dev/null +++ b/cmd/yurt-manager/names/controller_names.go @@ -0,0 +1,58 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package names + +const ( + CsrApproverController = "csr-approver-controller" + DaemonPodUpdaterController = "daemon-pod-updater-controller" + NodePoolController = "nodepool-controller" + PlatformAdminController = "platform-admin-controller" + ServiceTopologyEndpointsController = "service-topology-endpoints-controller" + ServiceTopologyEndpointSliceController = "service-topology-endpointslice-controller" + YurtAppSetController = "yurt-app-set-controller" + YurtAppDaemonController = "yurt-app-daemon-controller" + YurtStaticSetController = "yurt-static-set-controller" + YurtCoordinatorCertController = "yurt-coordinator-cert-controller" + DelegateLeaseController = "delegate-lease-controller" + PodBindingController = "pod-binding-controller" + GatewayPickupController = "gateway-pickup-controller" + GatewayInternalServiceController = "gateway-internal-service-controller" + GatewayPublicServiceController = "gateway-public-service" + GatewayDNSController = "gateway-dns-controller" +) + +func YurtManagerControllerAliases() map[string]string { + // return a new reference to achieve immutability of the mapping + return map[string]string{ + "csrapprover": CsrApproverController, + "daemonpodupdater": DaemonPodUpdaterController, + "nodepool": NodePoolController, + "platformadmin": PlatformAdminController, + "servicetopologyendpoints": ServiceTopologyEndpointsController, + "servicetopologyendpointslices": ServiceTopologyEndpointSliceController, + "yurtappset": YurtAppSetController, + "yurtappdaemon": YurtAppDaemonController, + "yurtstaticset": YurtStaticSetController, + "yurtcoordinatorcert": YurtCoordinatorCertController, + "delegatelease": DelegateLeaseController, + "podbinding": PodBindingController, + "gatewaypickup": GatewayPickupController, + "gatewayinternalservice": GatewayInternalServiceController, + "gatewaypublicservice": GatewayPublicServiceController, + "gatewaydns": GatewayDNSController, + } +} diff --git a/go.mod b/go.mod index 6bdad04c789..3c16f9c42eb 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( k8s.io/cluster-bootstrap v0.22.3 k8s.io/component-base v0.22.3 k8s.io/component-helpers v0.22.3 + k8s.io/controller-manager v0.22.3 k8s.io/klog/v2 v2.9.0 k8s.io/kubectl v0.22.3 k8s.io/kubernetes v1.22.3 diff --git a/go.sum b/go.sum index e3fa03ef033..1a6aba74e96 100644 --- a/go.sum +++ b/go.sum @@ -1173,6 +1173,7 @@ k8s.io/component-base v0.22.3 h1:/+hryAW03u3FpJQww+GSMsArJNUbGjH66lrgxaRynLU= k8s.io/component-base v0.22.3/go.mod h1:kuybv1miLCMoOk3ebrqF93GbQHQx6W2287FC0YEQY6s= k8s.io/component-helpers v0.22.3 h1:08tn+T8HnjRTwDP2ErIBhHGvPcYJf5zWaWW83golHWc= k8s.io/component-helpers v0.22.3/go.mod h1:7OVySVH5elhHKuJKUOxZEfpT1Bm3ChmBQZHmuFfbGHk= +k8s.io/controller-manager v0.22.3 h1:nBKG8MsgtUd/oFaZvE5zAYRIr45+Hn8QkHzq5+CtPOE= k8s.io/controller-manager v0.22.3/go.mod h1:4cvQGMvYf6IpTY08/NigEiI5UrN/cbtOe5e5WepYmcQ= k8s.io/cri-api v0.22.3/go.mod h1:mj5DGUtElRyErU5AZ8EM0ahxbElYsaLAMTPhLPQ40Eg= k8s.io/csi-translation-lib v0.22.3/go.mod h1:YkdI+scWhZJQeA26iNg9XrKO3LhLz6dAcRKsL0RIiUY= diff --git a/pkg/yurtmanager/controller/controller.go b/pkg/yurtmanager/controller/controller.go index 06f16c73b13..d30b7d8e2dd 100644 --- a/pkg/yurtmanager/controller/controller.go +++ b/pkg/yurtmanager/controller/controller.go @@ -17,24 +17,26 @@ limitations under the License. package controller import ( + "fmt" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/controller-manager/app" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/csrapprover" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/daemonpodupdater" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/nodepool" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/dns" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewayinternalservice" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewaypickup" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewaypublicservice" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology" servicetopologyendpoints "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/endpoints" servicetopologyendpointslice "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/endpointslice" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset" yurtcoordinatorcert "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/cert" @@ -43,33 +45,51 @@ import ( "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset" ) -// Note !!! @kadisi -// Do not change the name of the file @kadisi -// Note !!! - -// Don`t Change this Name !!!! @kadisi -// TODO support feature gate @kadisi -type AddControllerFn func(*config.CompletedConfig, manager.Manager) error - -var controllerAddFuncs = make(map[string][]AddControllerFn) - -func init() { - controllerAddFuncs[csrapprover.ControllerName] = []AddControllerFn{csrapprover.Add} - controllerAddFuncs[daemonpodupdater.ControllerName] = []AddControllerFn{daemonpodupdater.Add} - controllerAddFuncs[delegatelease.ControllerName] = []AddControllerFn{delegatelease.Add} - controllerAddFuncs[podbinding.ControllerName] = []AddControllerFn{podbinding.Add} - controllerAddFuncs[nodepool.ControllerName] = []AddControllerFn{nodepool.Add} - controllerAddFuncs[yurtcoordinatorcert.ControllerName] = []AddControllerFn{yurtcoordinatorcert.Add} - controllerAddFuncs[servicetopology.ControllerName] = []AddControllerFn{servicetopologyendpoints.Add, servicetopologyendpointslice.Add} - controllerAddFuncs[yurtstaticset.ControllerName] = []AddControllerFn{yurtstaticset.Add} - controllerAddFuncs[yurtappset.ControllerName] = []AddControllerFn{yurtappset.Add} - controllerAddFuncs[yurtappdaemon.ControllerName] = []AddControllerFn{yurtappdaemon.Add} - controllerAddFuncs[platformadmin.ControllerName] = []AddControllerFn{platformadmin.Add} - - controllerAddFuncs[raven.GatewayPickupControllerName] = []AddControllerFn{gatewaypickup.Add} - controllerAddFuncs[raven.GatewayDNSControllerName] = []AddControllerFn{dns.Add} - controllerAddFuncs[raven.GatewayInternalServiceController] = []AddControllerFn{gatewayinternalservice.Add} - controllerAddFuncs[raven.GatewayPublicServiceController] = []AddControllerFn{gatewaypublicservice.Add} +type InitFunc func(*config.CompletedConfig, manager.Manager) error + +type ControllerInitializersFunc func() (initializers map[string]InitFunc) + +var ( + _ ControllerInitializersFunc = NewControllerInitializers + + // ControllersDisabledByDefault is the set of controllers which is disabled by default + ControllersDisabledByDefault = sets.NewString() +) + +// KnownControllers returns all known controllers's name +func KnownControllers() []string { + ret := sets.StringKeySet(NewControllerInitializers()) + + return ret.List() +} + +func NewControllerInitializers() map[string]InitFunc { + controllers := map[string]InitFunc{} + register := func(name string, fn InitFunc) { + if _, found := controllers[name]; found { + panic(fmt.Sprintf("controller name %q was registered twice", name)) + } + controllers[name] = fn + } + + register(names.CsrApproverController, csrapprover.Add) + register(names.DaemonPodUpdaterController, daemonpodupdater.Add) + register(names.DelegateLeaseController, delegatelease.Add) + register(names.PodBindingController, podbinding.Add) + register(names.NodePoolController, nodepool.Add) + register(names.YurtCoordinatorCertController, yurtcoordinatorcert.Add) + register(names.ServiceTopologyEndpointsController, servicetopologyendpoints.Add) + register(names.ServiceTopologyEndpointSliceController, servicetopologyendpointslice.Add) + register(names.YurtStaticSetController, yurtstaticset.Add) + register(names.YurtAppSetController, yurtappset.Add) + register(names.YurtAppDaemonController, yurtappdaemon.Add) + register(names.PlatformAdminController, platformadmin.Add) + register(names.GatewayPickupController, gatewaypickup.Add) + register(names.GatewayDNSController, dns.Add) + register(names.GatewayInternalServiceController, gatewayinternalservice.Add) + register(names.GatewayPublicServiceController, gatewaypublicservice.Add) + + return controllers } // If you want to add additional RBAC, enter it here !!! @kadisi @@ -78,22 +98,20 @@ func init() { // +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete func SetupWithManager(c *config.CompletedConfig, m manager.Manager) error { - klog.InfoS("SetupWithManager", "len", len(controllerAddFuncs)) - for controllerName, fns := range controllerAddFuncs { - if !util.IsControllerEnabled(controllerName, c.ComponentConfig.Generic.Controllers) { + for controllerName, fn := range NewControllerInitializers() { + if !app.IsControllerEnabled(controllerName, ControllersDisabledByDefault, c.ComponentConfig.Generic.Controllers) { klog.Warningf("Controller %v is disabled", controllerName) continue } - for _, f := range fns { - if err := f(c, m); err != nil { - if kindMatchErr, ok := err.(*meta.NoKindMatchError); ok { - klog.Infof("CRD %v is not installed, its controller will perform noops!", kindMatchErr.GroupKind) - continue - } - return err + if err := fn(c, m); err != nil { + if kindMatchErr, ok := err.(*meta.NoKindMatchError); ok { + klog.Infof("CRD %v is not installed, its controller will perform noops!", kindMatchErr.GroupKind) + continue } + return err } } + return nil } diff --git a/pkg/yurtmanager/controller/csrapprover/csrapprover_controller.go b/pkg/yurtmanager/controller/csrapprover/csrapprover_controller.go index 0316fb3d66a..ab70a65bef2 100644 --- a/pkg/yurtmanager/controller/csrapprover/csrapprover_controller.go +++ b/pkg/yurtmanager/controller/csrapprover/csrapprover_controller.go @@ -43,6 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" yurtcoorrdinatorCert "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/cert" @@ -93,10 +94,6 @@ var ( } ) -const ( - ControllerName = "csrapprover" -) - type csrRecognizer struct { recognize func(csr *certificatesv1.CertificateSigningRequest, x509cr *x509.CertificateRequest) bool successMsg string @@ -107,7 +104,7 @@ type csrRecognizer struct { func Add(_ *appconfig.CompletedConfig, mgr manager.Manager) error { r := &ReconcileCsrApprover{} // Create a new controller - c, err := controller.New(ControllerName, mgr, controller.Options{ + c, err := controller.New(names.CsrApproverController, mgr, controller.Options{ Reconciler: r, MaxConcurrentReconciles: concurrentReconciles, }) if err != nil { diff --git a/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go b/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go index 726acefc586..bc6691813a4 100644 --- a/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go +++ b/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go @@ -49,6 +49,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" k8sutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/daemonpodupdater/kubernetes" ) @@ -64,8 +65,6 @@ var ( ) const ( - ControllerName = "daemonpodupdater" - // UpdateAnnotation is the annotation key used in daemonset spec to indicate // which update strategy is selected. Currently, "OTA" and "AdvancedRollingUpdate" are supported. UpdateAnnotation = "apps.openyurt.io/update-strategy" @@ -95,7 +94,7 @@ const ( func Format(format string, args ...interface{}) string { s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s: %s", ControllerName, s) + return fmt.Sprintf("%s: %s", names.DaemonPodUpdaterController, s) } // Add creates a new Daemonpodupdater Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller @@ -120,7 +119,7 @@ func newReconciler(_ *appconfig.CompletedConfig, mgr manager.Manager) reconcile. return &ReconcileDaemonpodupdater{ Client: mgr.GetClient(), expectations: k8sutil.NewControllerExpectations(), - recorder: mgr.GetEventRecorderFor(ControllerName), + recorder: mgr.GetEventRecorderFor(names.DaemonPodUpdaterController), } } @@ -141,7 +140,7 @@ func (r *ReconcileDaemonpodupdater) InjectConfig(cfg *rest.Config) error { // add adds a new Controller to mgr with r as the reconcile.Reconciler func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller - c, err := controller.New(ControllerName, mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + c, err := controller.New(names.DaemonPodUpdaterController, mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) if err != nil { return err } diff --git a/pkg/yurtmanager/controller/nodepool/nodepool_controller.go b/pkg/yurtmanager/controller/nodepool/nodepool_controller.go index e7372938c10..913fb2e9ba3 100644 --- a/pkg/yurtmanager/controller/nodepool/nodepool_controller.go +++ b/pkg/yurtmanager/controller/nodepool/nodepool_controller.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" "github.com/openyurtio/openyurt/pkg/apis/apps" appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" poolconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/nodepool/config" @@ -43,13 +44,9 @@ var ( controllerResource = appsv1beta1.SchemeGroupVersion.WithResource("nodepools") ) -const ( - ControllerName = "nodepool" -) - func Format(format string, args ...interface{}) string { s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s: %s", ControllerName, s) + return fmt.Sprintf("%s: %s", names.NodePoolController, s) } // ReconcileNodePool reconciles a NodePool object @@ -78,11 +75,11 @@ func Add(c *config.CompletedConfig, mgr manager.Manager) error { klog.Infof("nodepool-controller add controller %s", controllerResource.String()) r := &ReconcileNodePool{ cfg: c.ComponentConfig.NodePoolController, - recorder: mgr.GetEventRecorderFor(ControllerName), + recorder: mgr.GetEventRecorderFor(names.NodePoolController), } // Create a new controller - ctrl, err := controller.New(ControllerName, mgr, controller.Options{ + ctrl, err := controller.New(names.NodePoolController, mgr, controller.Options{ Reconciler: r, MaxConcurrentReconciles: concurrentReconciles, }) if err != nil { diff --git a/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go b/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go index 6d01a0ea155..a2e8b932a8d 100644 --- a/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go +++ b/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go @@ -45,6 +45,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" "github.com/openyurtio/openyurt/pkg/apis/apps" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" @@ -59,7 +60,7 @@ func init() { func Format(format string, args ...interface{}) string { s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s: %s", ControllerName, s) + return fmt.Sprintf("%s: %s", names.PlatformAdminController, s) } var ( @@ -68,8 +69,6 @@ var ( ) const ( - ControllerName = "PlatformAdmin" - LabelConfigmap = "Configmap" LabelService = "Service" LabelDeployment = "Deployment" @@ -135,7 +134,7 @@ func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile. return &ReconcilePlatformAdmin{ Client: mgr.GetClient(), scheme: mgr.GetScheme(), - recorder: mgr.GetEventRecorderFor(ControllerName), + recorder: mgr.GetEventRecorderFor(names.PlatformAdminController), yamlSerializer: kjson.NewSerializerWithOptions(kjson.DefaultMetaFactory, scheme.Scheme, scheme.Scheme, kjson.SerializerOptions{Yaml: true, Pretty: true}), Configration: c.ComponentConfig.PlatformAdminController, } @@ -144,7 +143,7 @@ func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile. // add adds a new Controller to mgr with r as the reconcile.Reconciler func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller - c, err := controller.New(ControllerName, mgr, controller.Options{ + c, err := controller.New(names.PlatformAdminController, mgr, controller.Options{ Reconciler: r, MaxConcurrentReconciles: concurrentReconciles, }) if err != nil { diff --git a/pkg/yurtmanager/controller/raven/common.go b/pkg/yurtmanager/controller/raven/common.go index 9b94c11f0fc..b69473e47c5 100644 --- a/pkg/yurtmanager/controller/raven/common.go +++ b/pkg/yurtmanager/controller/raven/common.go @@ -19,10 +19,3 @@ package raven var ( ConcurrentReconciles = 1 ) - -const ( - GatewayPickupControllerName = "raven-gateway-pickup" - GatewayInternalServiceController = "raven-gateway-internal-service" - GatewayPublicServiceController = "raven-gateway-public-service" - GatewayDNSControllerName = "raven-dns" -) diff --git a/pkg/yurtmanager/controller/raven/dns/dns_controller.go b/pkg/yurtmanager/controller/raven/dns/dns_controller.go index 43a3e4435c7..17eb950e7c7 100644 --- a/pkg/yurtmanager/controller/raven/dns/dns_controller.go +++ b/pkg/yurtmanager/controller/raven/dns/dns_controller.go @@ -40,13 +40,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" common "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/utils" ) func Format(format string, args ...interface{}) string { s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s: %s", common.GatewayDNSControllerName, s) + return fmt.Sprintf("%s: %s", names.GatewayDNSController, s) } // Add creates a new Ravendns Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller @@ -68,14 +69,14 @@ func newReconciler(mgr manager.Manager) reconcile.Reconciler { return &ReconcileDns{ Client: mgr.GetClient(), scheme: mgr.GetScheme(), - recorder: mgr.GetEventRecorderFor(common.GatewayDNSControllerName), + recorder: mgr.GetEventRecorderFor(names.GatewayDNSController), } } // add adds a new Controller to mgr with r as the reconcile.Reconciler func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller - c, err := controller.New(common.GatewayDNSControllerName, mgr, controller.Options{ + c, err := controller.New(names.GatewayDNSController, mgr, controller.Options{ Reconciler: r, MaxConcurrentReconciles: common.ConcurrentReconciles, }) if err != nil { diff --git a/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller.go b/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller.go index 0f1e7c8946c..15277e96218 100644 --- a/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller.go +++ b/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller.go @@ -43,6 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" common "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/utils" @@ -55,7 +56,7 @@ const ( func Format(format string, args ...interface{}) string { s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s: %s", common.GatewayInternalServiceController, s) + return fmt.Sprintf("%s: %s", names.GatewayInternalServiceController, s) } // Add creates a new Service Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller @@ -79,7 +80,7 @@ func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile. return &ReconcileService{ Client: mgr.GetClient(), scheme: mgr.GetScheme(), - recorder: mgr.GetEventRecorderFor(common.GatewayInternalServiceController), + recorder: mgr.GetEventRecorderFor(names.GatewayInternalServiceController), option: utils.NewOption(), } } @@ -87,7 +88,7 @@ func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile. // add adds a new Controller to mgr with r as the reconcile.Reconciler func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller - c, err := controller.New(common.GatewayInternalServiceController, mgr, controller.Options{ + c, err := controller.New(names.GatewayInternalServiceController, mgr, controller.Options{ Reconciler: r, MaxConcurrentReconciles: common.ConcurrentReconciles, }) if err != nil { diff --git a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go index 3f56bb8ea0f..4a6c2455d60 100644 --- a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go +++ b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go @@ -41,6 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" "github.com/openyurtio/openyurt/pkg/apis/raven" ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" common "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven" @@ -55,7 +56,7 @@ var ( func Format(format string, args ...interface{}) string { s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s: %s", common.GatewayPickupControllerName, s) + return fmt.Sprintf("%s: %s", names.GatewayPickupController, s) } const ( @@ -90,7 +91,7 @@ func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile. return &ReconcileGateway{ Client: mgr.GetClient(), scheme: mgr.GetScheme(), - recorder: mgr.GetEventRecorderFor(common.GatewayPickupControllerName), + recorder: mgr.GetEventRecorderFor(names.GatewayPickupController), Configration: c.ComponentConfig.GatewayPickupController, } } @@ -98,7 +99,7 @@ func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile. // add is used to add a new Controller to mgr func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller - c, err := controller.New(common.GatewayPickupControllerName, mgr, controller.Options{ + c, err := controller.New(names.GatewayPickupController, mgr, controller.Options{ Reconciler: r, MaxConcurrentReconciles: common.ConcurrentReconciles, }) if err != nil { diff --git a/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller.go b/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller.go index 0bc1d19ed09..ad43608c58a 100644 --- a/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller.go +++ b/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller.go @@ -41,6 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" "github.com/openyurtio/openyurt/pkg/apis/raven" ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" common "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven" @@ -53,7 +54,7 @@ const ( func Format(format string, args ...interface{}) string { s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s: %s", common.GatewayPublicServiceController, s) + return fmt.Sprintf("%s: %s", names.GatewayPublicServiceController, s) } // Add creates a new Service Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller @@ -104,7 +105,7 @@ func newReconciler(mgr manager.Manager) reconcile.Reconciler { return &ReconcileService{ Client: mgr.GetClient(), scheme: mgr.GetScheme(), - recorder: mgr.GetEventRecorderFor(common.GatewayPublicServiceController), + recorder: mgr.GetEventRecorderFor(names.GatewayPublicServiceController), option: utils.NewOption(), svcInfo: newServiceInfo(), } @@ -113,7 +114,7 @@ func newReconciler(mgr manager.Manager) reconcile.Reconciler { // add adds a new Controller to mgr with r as the reconcile.Reconciler func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller - c, err := controller.New(common.GatewayPublicServiceController, mgr, controller.Options{ + c, err := controller.New(names.GatewayPublicServiceController, mgr, controller.Options{ Reconciler: r, MaxConcurrentReconciles: common.ConcurrentReconciles, }) if err != nil { diff --git a/pkg/yurtmanager/controller/servicetopology/common.go b/pkg/yurtmanager/controller/servicetopology/common.go deleted file mode 100644 index fa58c1c01ef..00000000000 --- a/pkg/yurtmanager/controller/servicetopology/common.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the License); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an AS IS BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package servicetopology - -const ( - ControllerName = "servicetopology" -) diff --git a/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_controller.go b/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_controller.go index 39adacb3088..bb0d3407016 100644 --- a/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_controller.go +++ b/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_controller.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - common "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/adapter" ) @@ -46,7 +46,7 @@ var ( func Format(format string, args ...interface{}) string { s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s-endpoints: %s", common.ControllerName, s) + return fmt.Sprintf("%s: %s", names.ServiceTopologyEndpointsController, s) } // Add creates a new Servicetopology endpoints Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller @@ -78,7 +78,7 @@ func (r *ReconcileServicetopologyEndpoints) InjectClient(c client.Client) error // add adds a new Controller to mgr with r as the reconcile.Reconciler func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller - c, err := controller.New(fmt.Sprintf("%s-endpoints", common.ControllerName), mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + c, err := controller.New(names.ServiceTopologyEndpointsController, mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) if err != nil { return err } diff --git a/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_controller.go b/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_controller.go index 8a26925f55f..f5541322571 100644 --- a/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_controller.go +++ b/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_controller.go @@ -34,7 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - common "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/adapter" ) @@ -49,14 +49,14 @@ var ( func Format(format string, args ...interface{}) string { s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s-endpointslice: %s", common.ControllerName, s) + return fmt.Sprintf("%s: %s", names.ServiceTopologyEndpointSliceController, s) } // Add creates a new Servicetopology endpointslice Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(_ *appconfig.CompletedConfig, mgr manager.Manager) error { r := &ReconcileServiceTopologyEndpointSlice{} - c, err := controller.New(fmt.Sprintf("%s-endpointslice", common.ControllerName), mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + c, err := controller.New(names.ServiceTopologyEndpointSliceController, mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) if err != nil { return err } @@ -74,7 +74,7 @@ func Add(_ *appconfig.CompletedConfig, mgr manager.Manager) error { return err } - klog.Infof("%s-endpointslice controller is added", common.ControllerName) + klog.Infof("%s controller is added", names.ServiceTopologyEndpointSliceController) return nil } diff --git a/pkg/yurtmanager/controller/util/controller_utils.go b/pkg/yurtmanager/controller/util/controller_utils.go deleted file mode 100644 index 09879cbb825..00000000000 --- a/pkg/yurtmanager/controller/util/controller_utils.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -// IsControllerEnabled check if a specified controller enabled or not. -func IsControllerEnabled(name string, controllers []string) bool { - for _, ctrl := range controllers { - if ctrl == name { - return true - } - if ctrl == "-"+name { - return false - } - if ctrl == "*" { - return true - } - } - - return false -} diff --git a/pkg/yurtmanager/controller/util/controller_utils_test.go b/pkg/yurtmanager/controller/util/controller_utils_test.go deleted file mode 100644 index 6eb75008e35..00000000000 --- a/pkg/yurtmanager/controller/util/controller_utils_test.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the License); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an AS IS BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import "testing" - -func TestIsControllerEnabled(t *testing.T) { - testcases := map[string]struct { - name string - controllers []string - result bool - }{ - "enable specified controller": { - name: "foo", - controllers: []string{"foo", "bar"}, - result: true, - }, - "disable specified controller": { - name: "foo", - controllers: []string{"-foo", "bar"}, - result: false, - }, - "enable controller in default": { - name: "foo", - controllers: []string{"bar", "*"}, - result: true, - }, - "controller doesn't exist": { - name: "unknown", - controllers: []string{"foo", "bar"}, - result: false, - }, - } - - for k, tc := range testcases { - t.Run(k, func(t *testing.T) { - result := IsControllerEnabled(tc.name, tc.controllers) - if tc.result != result { - t.Errorf("expect controller enabled: %v, but got %v", tc.result, result) - } - }) - } -} diff --git a/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go index 097a092ef07..05df1435db7 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go @@ -38,6 +38,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller" @@ -49,7 +50,6 @@ var ( ) const ( - ControllerName = "yurtappdaemon" slowStartInitialBatchSize = 1 eventTypeRevisionProvision = "RevisionProvision" @@ -66,7 +66,7 @@ func init() { func Format(format string, args ...interface{}) string { s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s: %s", ControllerName, s) + return fmt.Sprintf("%s: %s", names.YurtAppDaemonController, s) } // Add creates a new YurtAppDaemon Controller and adds it to the Manager with default RBAC. @@ -85,7 +85,7 @@ func Add(c *config.CompletedConfig, mgr manager.Manager) error { // add adds a new Controller to mgr with r as the reconcile.Reconciler func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller - c, err := controller.New(ControllerName, mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + c, err := controller.New(names.YurtAppDaemonController, mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) if err != nil { return err } @@ -118,10 +118,9 @@ type ReconcileYurtAppDaemon struct { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) reconcile.Reconciler { return &ReconcileYurtAppDaemon{ - Client: mgr.GetClient(), - scheme: mgr.GetScheme(), - - recorder: mgr.GetEventRecorderFor(ControllerName), + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + recorder: mgr.GetEventRecorderFor(names.YurtAppDaemonController), controls: map[unitv1alpha1.TemplateType]workloadcontroller.WorkloadController{ // unitv1alpha1.StatefulSetTemplateType: &StatefulSetControllor{Client: mgr.GetClient(), scheme: mgr.GetScheme()}, unitv1alpha1.DeploymentTemplateType: &workloadcontroller.DeploymentControllor{Client: mgr.GetClient(), Scheme: mgr.GetScheme()}, diff --git a/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go index bfdf9091125..5611e911552 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" "github.com/openyurtio/openyurt/pkg/apis/apps" unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller" @@ -125,7 +126,7 @@ func TestUpdateStatus(t *testing.T) { "equal", yad, &unitv1alpha1.YurtAppDaemonStatus{ - CurrentRevision: ControllerName, + CurrentRevision: names.YurtAppDaemonController, CollisionCount: &int1, TemplateType: "StatefulSet", ObservedGeneration: 1, @@ -139,7 +140,7 @@ func TestUpdateStatus(t *testing.T) { }, }, &unitv1alpha1.YurtAppDaemonStatus{ - CurrentRevision: ControllerName, + CurrentRevision: names.YurtAppDaemonController, CollisionCount: &int1, TemplateType: "StatefulSet", ObservedGeneration: 1, @@ -196,7 +197,7 @@ func TestUpdateYurtAppDaemon(t *testing.T) { "equal", yad, &unitv1alpha1.YurtAppDaemonStatus{ - CurrentRevision: ControllerName, + CurrentRevision: names.YurtAppDaemonController, CollisionCount: &int1, TemplateType: "StatefulSet", ObservedGeneration: 1, @@ -210,7 +211,7 @@ func TestUpdateYurtAppDaemon(t *testing.T) { }, }, &unitv1alpha1.YurtAppDaemonStatus{ - CurrentRevision: ControllerName, + CurrentRevision: names.YurtAppDaemonController, CollisionCount: &int1, TemplateType: "StatefulSet", ObservedGeneration: 1, diff --git a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go index 2d3edaa2dd5..750e57a8f05 100644 --- a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go +++ b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go @@ -41,6 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/adapter" ) @@ -55,8 +56,6 @@ var ( ) const ( - ControllerName = "yurtappset" - eventTypeRevisionProvision = "RevisionProvision" eventTypeFindPools = "FindPools" eventTypeDupPoolsDelete = "DeleteDuplicatedPools" @@ -68,7 +67,7 @@ const ( func Format(format string, args ...interface{}) string { s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s: %s", ControllerName, s) + return fmt.Sprintf("%s: %s", names.YurtAppSetController, s) } // Add creates a new YurtAppSet Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller @@ -89,7 +88,7 @@ func newReconciler(c *config.CompletedConfig, mgr manager.Manager) reconcile.Rec Client: mgr.GetClient(), scheme: mgr.GetScheme(), - recorder: mgr.GetEventRecorderFor(ControllerName), + recorder: mgr.GetEventRecorderFor(names.YurtAppSetController), poolControls: map[unitv1alpha1.TemplateType]ControlInterface{ unitv1alpha1.StatefulSetTemplateType: &PoolControl{Client: mgr.GetClient(), scheme: mgr.GetScheme(), adapter: &adapter.StatefulSetAdapter{Client: mgr.GetClient(), Scheme: mgr.GetScheme()}}, @@ -102,7 +101,7 @@ func newReconciler(c *config.CompletedConfig, mgr manager.Manager) reconcile.Rec // add adds a new Controller to mgr with r as the reconcile.Reconciler func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller - c, err := controller.New(ControllerName, mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + c, err := controller.New(names.YurtAppSetController, mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) if err != nil { return err } diff --git a/pkg/yurtmanager/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller.go b/pkg/yurtmanager/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller.go index 414e96f6839..dc127095225 100644 --- a/pkg/yurtmanager/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller.go +++ b/pkg/yurtmanager/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller.go @@ -39,6 +39,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" certfactory "github.com/openyurtio/openyurt/pkg/util/certmanager/factory" "github.com/openyurtio/openyurt/pkg/util/ip" ) @@ -53,8 +54,6 @@ var ( ) const ( - ControllerName = "yurtcoordinatorcert" - // tmp file directory for certmanager to write cert files certDir = "/tmp" @@ -199,7 +198,7 @@ var ( func Format(format string, args ...interface{}) string { s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s: %s", ControllerName, s) + return fmt.Sprintf("%s: %s", names.YurtCoordinatorCertController, s) } // Add creates a new YurtCoordinatorcert Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller @@ -208,7 +207,7 @@ func Add(cfg *appconfig.CompletedConfig, mgr manager.Manager) error { r := &ReconcileYurtCoordinatorCert{} // Create a new controller - c, err := controller.New(ControllerName, mgr, controller.Options{ + c, err := controller.New(names.YurtCoordinatorCertController, mgr, controller.Options{ Reconciler: r, MaxConcurrentReconciles: concurrentReconciles, }) if err != nil { diff --git a/pkg/yurtmanager/controller/yurtcoordinator/delegatelease/delegatelease_controller.go b/pkg/yurtmanager/controller/yurtcoordinator/delegatelease/delegatelease_controller.go index 7420f6007a1..d06ff113d71 100644 --- a/pkg/yurtmanager/controller/yurtcoordinator/delegatelease/delegatelease_controller.go +++ b/pkg/yurtmanager/controller/yurtcoordinator/delegatelease/delegatelease_controller.go @@ -36,6 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" nodeutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/node" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/constant" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/utils" @@ -45,10 +46,6 @@ func init() { flag.IntVar(&concurrentReconciles, "delegatelease-controller", concurrentReconciles, "Max concurrent workers for delegatelease-controller controller.") } -const ( - ControllerName = "delegatelease" -) - var ( concurrentReconciles = 5 ) @@ -67,7 +64,7 @@ func Add(_ *appconfig.CompletedConfig, mgr manager.Manager) error { ldc: utils.NewLeaseDelegatedCounter(), delLdc: utils.NewLeaseDelegatedCounter(), } - c, err := controller.New(ControllerName, mgr, controller.Options{ + c, err := controller.New(names.DelegateLeaseController, mgr, controller.Options{ Reconciler: r, MaxConcurrentReconciles: concurrentReconciles, }) if err != nil { diff --git a/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller.go b/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller.go index cdd02666f58..bcea5dbabd7 100644 --- a/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller.go +++ b/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/constant" ) @@ -41,10 +42,6 @@ func init() { flag.IntVar(&concurrentReconciles, "podbinding-controller", concurrentReconciles, "Max concurrent workers for podbinding-controller controller.") } -const ( - ControllerName = "podbinding" -) - var ( controllerKind = appsv1.SchemeGroupVersion.WithKind("Node") concurrentReconciles = 5 @@ -65,7 +62,7 @@ var ( func Format(format string, args ...interface{}) string { s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s: %s", ControllerName, s) + return fmt.Sprintf("%s: %s", names.PodBindingController, s) } type ReconcilePodBinding struct { @@ -86,7 +83,7 @@ func newReconciler(_ *appconfig.CompletedConfig, mgr manager.Manager) reconcile. // add adds a new Controller to mgr with r as the reconcile.Reconciler func add(mgr manager.Manager, r reconcile.Reconciler) error { - c, err := controller.New(ControllerName, mgr, controller.Options{ + c, err := controller.New(names.PodBindingController, mgr, controller.Options{ Reconciler: r, MaxConcurrentReconciles: concurrentReconciles, }) if err != nil { diff --git a/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go b/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go index 16e76fb9092..01bbff4d14b 100644 --- a/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go +++ b/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go @@ -41,6 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/config" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo" @@ -58,8 +59,6 @@ var ( ) const ( - ControllerName = "yurtstaticset" - StaticPodHashAnnotation = "openyurt.io/static-pod-hash" hostPathVolumeName = "hostpath" @@ -120,7 +119,7 @@ var ( func Format(format string, args ...interface{}) string { s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s: %s", ControllerName, s) + return fmt.Sprintf("%s: %s", names.YurtStaticSetController, s) } // Add creates a new YurtStaticSet Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller @@ -150,7 +149,7 @@ func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile. return &ReconcileYurtStaticSet{ Client: mgr.GetClient(), scheme: mgr.GetScheme(), - recorder: mgr.GetEventRecorderFor(ControllerName), + recorder: mgr.GetEventRecorderFor(names.YurtStaticSetController), Configuration: c.ComponentConfig.YurtStaticSetController, } } @@ -158,7 +157,7 @@ func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile. // add adds a new Controller to mgr with r as the reconcile.Reconciler func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller - c, err := controller.New(ControllerName, mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + c, err := controller.New(names.YurtStaticSetController, mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) if err != nil { return err } diff --git a/pkg/yurtmanager/webhook/server.go b/pkg/yurtmanager/webhook/server.go index 47547c10a26..4ed3ff08e5f 100644 --- a/pkg/yurtmanager/webhook/server.go +++ b/pkg/yurtmanager/webhook/server.go @@ -22,18 +22,14 @@ import ( "time" "k8s.io/client-go/rest" + "k8s.io/controller-manager/app" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/nodepool" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven" - ctrlutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller" v1beta1gateway "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/gateway/v1beta1" v1node "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/node/v1" v1beta1nodepool "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/nodepool/v1beta1" @@ -73,13 +69,13 @@ func addControllerWebhook(name string, handler SetupWebhookWithManager) { } func init() { - addControllerWebhook(raven.GatewayPickupControllerName, &v1beta1gateway.GatewayHandler{}) - addControllerWebhook(nodepool.ControllerName, &v1beta1nodepool.NodePoolHandler{}) - addControllerWebhook(yurtstaticset.ControllerName, &v1alpha1yurtstaticset.YurtStaticSetHandler{}) - addControllerWebhook(yurtappset.ControllerName, &v1alpha1yurtappset.YurtAppSetHandler{}) - addControllerWebhook(yurtappdaemon.ControllerName, &v1alpha1yurtappdaemon.YurtAppDaemonHandler{}) - addControllerWebhook(platformadmin.ControllerName, &v1alpha1platformadmin.PlatformAdminHandler{}) - addControllerWebhook(platformadmin.ControllerName, &v1alpha2platformadmin.PlatformAdminHandler{}) + addControllerWebhook(names.GatewayPickupController, &v1beta1gateway.GatewayHandler{}) + addControllerWebhook(names.NodePoolController, &v1beta1nodepool.NodePoolHandler{}) + addControllerWebhook(names.YurtStaticSetController, &v1alpha1yurtstaticset.YurtStaticSetHandler{}) + addControllerWebhook(names.YurtAppSetController, &v1alpha1yurtappset.YurtAppSetHandler{}) + addControllerWebhook(names.YurtAppDaemonController, &v1alpha1yurtappdaemon.YurtAppDaemonHandler{}) + addControllerWebhook(names.PlatformAdminController, &v1alpha1platformadmin.PlatformAdminHandler{}) + addControllerWebhook(names.PlatformAdminController, &v1alpha2platformadmin.PlatformAdminHandler{}) independentWebhooks[v1pod.WebhookName] = &v1pod.PodHandler{} independentWebhooks[v1node.WebhookName] = &v1node.NodeHandler{} @@ -125,7 +121,7 @@ func SetupWithManager(c *config.CompletedConfig, mgr manager.Manager) error { // set up controller webhooks for controllerName, list := range controllerWebhooks { - if !ctrlutil.IsControllerEnabled(controllerName, c.ComponentConfig.Generic.Controllers) { + if !app.IsControllerEnabled(controllerName, controller.ControllersDisabledByDefault, c.ComponentConfig.Generic.Controllers) { klog.Warningf("Webhook for %v is disabled", controllerName) continue } diff --git a/test/e2e/cmd/init/constants/constants.go b/test/e2e/cmd/init/constants/constants.go index 228bc16efad..3e3504ec57b 100644 --- a/test/e2e/cmd/init/constants/constants.go +++ b/test/e2e/cmd/init/constants/constants.go @@ -100,6 +100,7 @@ spec: - --metrics-addr=:10271 - --health-probe-addr=:10272 - --webhook-port=10273 + - --controllers=* - --logtostderr=true - --v=4 command: From 61171de4aa7845415d4d0657a9dcb9369890e548 Mon Sep 17 00:00:00 2001 From: lilongfeng Date: Mon, 11 Sep 2023 11:52:38 +0800 Subject: [PATCH 07/13] unified expression about DaemonSet (#1698) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * misspelling about log info * invalid connection * remove cmd/yurt-app-manager * unified expression about DaemonSet Signed-off-by: 李龙峰 --------- Signed-off-by: 李龙峰 --- .../daemon_pod_updater_controller.go | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go b/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go index bc6691813a4..da69f146ca0 100644 --- a/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go +++ b/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go @@ -65,15 +65,15 @@ var ( ) const ( - // UpdateAnnotation is the annotation key used in daemonset spec to indicate + // UpdateAnnotation is the annotation key used in DaemonSet spec to indicate // which update strategy is selected. Currently, "OTA" and "AdvancedRollingUpdate" are supported. UpdateAnnotation = "apps.openyurt.io/update-strategy" - // OTAUpdate set daemonset to over-the-air update mode. + // OTAUpdate set DaemonSet to over-the-air update mode. // In daemonPodUpdater controller, we add PodNeedUpgrade condition to pods. OTAUpdate = "OTA" - // AutoUpdate set daemonset to Auto update mode. - // In this mode, daemonset will keep updating even if there are not-ready nodes. + // AutoUpdate set DaemonSet to Auto update mode. + // In this mode, DaemonSet will keep updating even if there are not-ready nodes. // For more details, see https://github.com/openyurtio/openyurt/pull/921. AutoUpdate = "Auto" AdvancedRollingUpdate = "AdvancedRollingUpdate" @@ -81,7 +81,7 @@ const ( // PodNeedUpgrade indicates whether the pod is able to upgrade. PodNeedUpgrade corev1.PodConditionType = "PodNeedUpgrade" - // MaxUnavailableAnnotation is the annotation key added to daemonset to indicate + // MaxUnavailableAnnotation is the annotation key added to DaemonSet to indicate // the max unavailable pods number. It's used with "apps.openyurt.io/update-strategy=AdvancedRollingUpdate". // If this annotation is not explicitly stated, it will be set to the default value 1. MaxUnavailableAnnotation = "apps.openyurt.io/max-unavailable" @@ -176,7 +176,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { return nil } -// daemonsetUpdate filter events: daemonset update with customized annotation +// daemonsetUpdate filter events: DaemonSet update with customized annotation func daemonsetUpdate(evt event.UpdateEvent) bool { if _, ok := evt.ObjectOld.(*appsv1.DaemonSet); !ok { return false @@ -185,7 +185,7 @@ func daemonsetUpdate(evt event.UpdateEvent) bool { oldDS := evt.ObjectOld.(*appsv1.DaemonSet) newDS := evt.ObjectNew.(*appsv1.DaemonSet) - // Only handle daemonset meets prerequisites + // Only handle DaemonSet meets prerequisites if !checkPrerequisites(newDS) { return false } @@ -194,7 +194,7 @@ func daemonsetUpdate(evt event.UpdateEvent) bool { return false } - klog.V(5).Infof("Got daemonset udpate event: %v", newDS.Name) + klog.V(5).Infof("Got DaemonSet update event: %v", newDS.Name) return true } @@ -225,8 +225,8 @@ func (r *ReconcileDaemonpodupdater) Reconcile(_ context.Context, request reconci return reconcile.Result{}, nil } - // Only process daemonset that meets expectations - // Otherwise, wait native daemonset controller reconciling + // Only process DaemonSet that meets expectations + // Otherwise, wait native DaemonSet controller reconciling if !r.expectations.SatisfiedExpectations(request.NamespacedName.String()) { return reconcile.Result{}, nil } @@ -234,7 +234,7 @@ func (r *ReconcileDaemonpodupdater) Reconcile(_ context.Context, request reconci // Recheck required annotation v, ok := instance.Annotations[UpdateAnnotation] if !ok { - klog.V(4).Infof("won't sync daemonset %q without annotation 'apps.openyurt.io/update-strategy'", + klog.V(4).Infof("won't sync DaemonSet %q without annotation 'apps.openyurt.io/update-strategy'", request.NamespacedName) return reconcile.Result{}, nil } @@ -266,7 +266,7 @@ func (r *ReconcileDaemonpodupdater) deletePod(evt event.DeleteEvent, _ workqueue return } - klog.V(5).Infof("Daemonset pod %s deleted.", pod.Name) + klog.V(5).Infof("DaemonSet pod %s deleted.", pod.Name) controllerRef := metav1.GetControllerOf(pod) if controllerRef == nil { @@ -278,7 +278,7 @@ func (r *ReconcileDaemonpodupdater) deletePod(evt event.DeleteEvent, _ workqueue return } - // Only care daemonset meets prerequisites + // Only care DaemonSet meets prerequisites if !checkPrerequisites(ds) { return } @@ -291,8 +291,8 @@ func (r *ReconcileDaemonpodupdater) deletePod(evt event.DeleteEvent, _ workqueue r.expectations.DeletionObserved(dsKey) } -// otaUpdate compare every pod to its owner daemonset to check if pod is updatable -// If pod is in line with the latest daemonset spec, set pod condition "PodNeedUpgrade" to "false" +// otaUpdate compare every pod to its owner DaemonSet to check if pod is updatable +// If pod is in line with the latest DaemonSet spec, set pod condition "PodNeedUpgrade" to "false" // while not, set pod condition "PodNeedUpgrade" to "true" func (r *ReconcileDaemonpodupdater) otaUpdate(ds *appsv1.DaemonSet) error { pods, err := GetDaemonsetPods(r.Client, ds) @@ -328,7 +328,7 @@ func (r *ReconcileDaemonpodupdater) advancedRollingUpdate(ds *appsv1.DaemonSet) for nodeName, pods := range nodeToDaemonPods { // Check if node is ready, ignore not-ready node - // this is a significant difference from the native daemonset controller + // this is a significant difference from the native DaemonSet controller ready, err := NodeReadyByName(r.Client, nodeName) if err != nil { return fmt.Errorf("couldn't check node %q ready status, %v", nodeName, err) From 0dcd845cd5027c48c020d319a86ace66919a81d9 Mon Sep 17 00:00:00 2001 From: guangwu Date: Mon, 11 Sep 2023 14:12:38 +0800 Subject: [PATCH 08/13] chore: slice loop replace (#1690) Signed-off-by: guoguangwu --- cmd/yurt-tunnel-server/app/options/options.go | 4 +--- pkg/yurtmanager/controller/yurtappset/adapter/adapter_util.go | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/cmd/yurt-tunnel-server/app/options/options.go b/cmd/yurt-tunnel-server/app/options/options.go index 1833b576770..32b045a7471 100644 --- a/cmd/yurt-tunnel-server/app/options/options.go +++ b/cmd/yurt-tunnel-server/app/options/options.go @@ -127,9 +127,7 @@ func (o *ServerOptions) Config() (*config.Config, error) { } if o.CertDNSNames != "" { - for _, name := range strings.Split(o.CertDNSNames, ",") { - cfg.CertDNSNames = append(cfg.CertDNSNames, name) - } + cfg.CertDNSNames = append(cfg.CertDNSNames, strings.Split(o.CertDNSNames, ",")...) } if o.CertIPs != "" { diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util.go b/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util.go index 5b51077965f..e62094596c4 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util.go @@ -84,9 +84,7 @@ func attachTolerations(podSpec *corev1.PodSpec, poolConfig *appsv1alpha1.Pool) { podSpec.Tolerations = []corev1.Toleration{} } - for _, toleration := range poolConfig.Tolerations { - podSpec.Tolerations = append(podSpec.Tolerations, toleration) - } + podSpec.Tolerations = append(podSpec.Tolerations, poolConfig.Tolerations...) return } From f9aaa3904f17e3f6d5206dd79f926318a37205ff Mon Sep 17 00:00:00 2001 From: Zhen Zhao <413621396@qq.com> Date: Mon, 11 Sep 2023 14:17:37 +0800 Subject: [PATCH 09/13] modify the logic for judging static pods and combine the utils for pod and node (#1689) --- .../daemon_pod_updater_controller.go | 5 +- .../controller/daemonpodupdater/util.go | 6 +- .../controller/daemonpodupdater/util_test.go | 53 +++++++++++ .../controller/util/node/controller_utils.go | 53 ----------- .../kubernetes => util/pod}/pod_util.go | 53 ++++++++++- .../kubernetes => util/pod}/pod_util_test.go | 92 ++++++++++++++++++- .../yurtstaticset/upgradeinfo/upgrade_info.go | 3 +- .../upgradeinfo/upgrade_info_test.go | 10 +- .../controller/yurtstaticset/util/util.go | 82 +---------------- .../yurtstaticset/yurtstaticset_controller.go | 8 +- 10 files changed, 222 insertions(+), 143 deletions(-) rename pkg/yurtmanager/controller/{daemonpodupdater/kubernetes => util/pod}/pod_util.go (62%) rename pkg/yurtmanager/controller/{daemonpodupdater/kubernetes => util/pod}/pod_util_test.go (50%) diff --git a/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go b/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go index da69f146ca0..7e9cc55c7ba 100644 --- a/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go +++ b/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go @@ -51,6 +51,7 @@ import ( appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" "github.com/openyurtio/openyurt/cmd/yurt-manager/names" k8sutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/daemonpodupdater/kubernetes" + podutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/pod" ) func init() { @@ -350,14 +351,14 @@ func (r *ReconcileDaemonpodupdater) advancedRollingUpdate(ds *appsv1.DaemonSet) numUnavailable++ case newPod != nil: // This pod is up-to-date, check its availability - if !k8sutil.IsPodAvailable(newPod, ds.Spec.MinReadySeconds, metav1.Time{Time: time.Now()}) { + if !podutil.IsPodAvailable(newPod, ds.Spec.MinReadySeconds, metav1.Time{Time: time.Now()}) { // An unavailable new pod is counted against maxUnavailable numUnavailable++ } default: // This pod is old, it is an update candidate switch { - case !k8sutil.IsPodAvailable(oldPod, ds.Spec.MinReadySeconds, metav1.Time{Time: time.Now()}): + case !podutil.IsPodAvailable(oldPod, ds.Spec.MinReadySeconds, metav1.Time{Time: time.Now()}): // The old pod isn't available, so it needs to be replaced klog.V(5).Infof("DaemonSet %s/%s pod %s on node %s is out of date and not available, allowing replacement", ds.Namespace, ds.Name, oldPod.Name, nodeName) // Record the replacement diff --git a/pkg/yurtmanager/controller/daemonpodupdater/util.go b/pkg/yurtmanager/controller/daemonpodupdater/util.go index be7f5fe41e3..84b1b030aae 100644 --- a/pkg/yurtmanager/controller/daemonpodupdater/util.go +++ b/pkg/yurtmanager/controller/daemonpodupdater/util.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/daemonpodupdater/kubernetes" - util "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/node" + podutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/pod" ) // GetDaemonsetPods get all pods belong to the given daemonset @@ -127,7 +127,7 @@ func SetPodUpgradeCondition(c client.Client, ds *appsv1.DaemonSet, pod *corev1.P Type: PodNeedUpgrade, Status: status, } - if change := util.UpdatePodCondition(&pod.Status, cond); change { + if change := podutil.UpdatePodCondition(&pod.Status, cond); change { if err := c.Status().Update(context.TODO(), pod, &client.UpdateOptions{}); err != nil { return err @@ -243,6 +243,6 @@ func IsPodUpgradeConditionTrue(status corev1.PodStatus) bool { // GetPodUpgradeCondition extracts the pod upgrade condition from the given status and returns that. // Returns nil if the condition is not present. func GetPodUpgradeCondition(status corev1.PodStatus) *corev1.PodCondition { - _, condition := kubernetes.GetPodCondition(&status, PodNeedUpgrade) + _, condition := podutil.GetPodCondition(&status, PodNeedUpgrade) return condition } diff --git a/pkg/yurtmanager/controller/daemonpodupdater/util_test.go b/pkg/yurtmanager/controller/daemonpodupdater/util_test.go index 1fc66343850..d4cf7602795 100644 --- a/pkg/yurtmanager/controller/daemonpodupdater/util_test.go +++ b/pkg/yurtmanager/controller/daemonpodupdater/util_test.go @@ -17,6 +17,7 @@ limitations under the License. package daemonpodupdater import ( + "reflect" "testing" "github.com/stretchr/testify/assert" @@ -223,3 +224,55 @@ func TestGetTargetNodeName(t *testing.T) { }) } } + +func TestGetPodUpgradeCondition(t *testing.T) { + pod1 := newPod("pod1", "", nil, nil) + pod1.Status = corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: PodNeedUpgrade, + Status: corev1.ConditionTrue, + }, + }, + } + + pod2 := pod1.DeepCopy() + pod2.Status = corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: PodNeedUpgrade, + Status: corev1.ConditionFalse, + }, + }, + } + + tests := []struct { + name string + status corev1.PodStatus + want *corev1.PodCondition + }{ + { + name: "pod1", + status: pod1.Status, + want: &corev1.PodCondition{ + Type: PodNeedUpgrade, + Status: corev1.ConditionTrue, + }, + }, + { + name: "pod2", + status: pod2.Status, + want: &corev1.PodCondition{ + Type: PodNeedUpgrade, + Status: corev1.ConditionFalse, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetPodUpgradeCondition(tt.status); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetPodUpgradeCondition() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/yurtmanager/controller/util/node/controller_utils.go b/pkg/yurtmanager/controller/util/node/controller_utils.go index f83c0b994e4..b96981d06ad 100644 --- a/pkg/yurtmanager/controller/util/node/controller_utils.go +++ b/pkg/yurtmanager/controller/util/node/controller_utils.go @@ -19,61 +19,8 @@ package node import ( v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// GetPodCondition extracts the provided condition from the given status and returns that. -// Returns nil and -1 if the condition is not present, and the index of the located condition. -func GetPodCondition(status *v1.PodStatus, conditionType v1.PodConditionType) (int, *v1.PodCondition) { - if status == nil { - return -1, nil - } - return GetPodConditionFromList(status.Conditions, conditionType) -} - -// GetPodConditionFromList extracts the provided condition from the given list of condition and -// returns the index of the condition and the condition. Returns -1 and nil if the condition is not present. -func GetPodConditionFromList(conditions []v1.PodCondition, conditionType v1.PodConditionType) (int, *v1.PodCondition) { - if conditions == nil { - return -1, nil - } - for i := range conditions { - if conditions[i].Type == conditionType { - return i, &conditions[i] - } - } - return -1, nil -} - -// UpdatePodCondition updates existing pod condition or creates a new one. Sets LastTransitionTime to now if the -// status has changed. -// Returns true if pod condition has changed or has been added. -func UpdatePodCondition(status *v1.PodStatus, condition *v1.PodCondition) bool { - condition.LastTransitionTime = metav1.Now() - // Try to find this pod condition. - conditionIndex, oldCondition := GetPodCondition(status, condition.Type) - - if oldCondition == nil { - // We are adding new pod condition. - status.Conditions = append(status.Conditions, *condition) - return true - } - // We are updating an existing condition, so we need to check if it has changed. - if condition.Status == oldCondition.Status { - condition.LastTransitionTime = oldCondition.LastTransitionTime - } - - isEqual := condition.Status == oldCondition.Status && - condition.Reason == oldCondition.Reason && - condition.Message == oldCondition.Message && - condition.LastProbeTime.Equal(&oldCondition.LastProbeTime) && - condition.LastTransitionTime.Equal(&oldCondition.LastTransitionTime) - - status.Conditions[conditionIndex] = *condition - // Return true if one of the fields have changed. - return !isEqual -} - // GetNodeCondition extracts the provided condition from the given status and returns that. // Returns nil and -1 if the condition is not present, and the index of the located condition. func GetNodeCondition(status *v1.NodeStatus, conditionType v1.NodeConditionType) (int, *v1.NodeCondition) { diff --git a/pkg/yurtmanager/controller/daemonpodupdater/kubernetes/pod_util.go b/pkg/yurtmanager/controller/util/pod/pod_util.go similarity index 62% rename from pkg/yurtmanager/controller/daemonpodupdater/kubernetes/pod_util.go rename to pkg/yurtmanager/controller/util/pod/pod_util.go index dd50ab3bfaa..9a6da0359ea 100644 --- a/pkg/yurtmanager/controller/daemonpodupdater/kubernetes/pod_util.go +++ b/pkg/yurtmanager/controller/util/pod/pod_util.go @@ -14,15 +14,21 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubernetes +package pod import ( + "fmt" "time" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + ApiserverSource = "api" + ConfigSourceAnnotationKey = "kubernetes.io/config.source" +) + // IsPodAvailable returns true if a pod is available; false otherwise. // Precondition for an available pod is that it must be ready. On top // of that, there are two cases when a pod can be considered available: @@ -81,3 +87,48 @@ func GetPodConditionFromList(conditions []v1.PodCondition, conditionType v1.PodC } return -1, nil } + +// UpdatePodCondition updates existing pod condition or creates a new one. Sets LastTransitionTime to now if the +// status has changed. +// Returns true if pod condition has changed or has been added. +func UpdatePodCondition(status *v1.PodStatus, condition *v1.PodCondition) bool { + condition.LastTransitionTime = metav1.Now() + // Try to find this pod condition. + conditionIndex, oldCondition := GetPodCondition(status, condition.Type) + + if oldCondition == nil { + // We are adding new pod condition. + status.Conditions = append(status.Conditions, *condition) + return true + } + // We are updating an existing condition, so we need to check if it has changed. + if condition.Status == oldCondition.Status { + condition.LastTransitionTime = oldCondition.LastTransitionTime + } + + isEqual := condition.Status == oldCondition.Status && + condition.Reason == oldCondition.Reason && + condition.Message == oldCondition.Message && + condition.LastProbeTime.Equal(&oldCondition.LastProbeTime) && + condition.LastTransitionTime.Equal(&oldCondition.LastTransitionTime) + + status.Conditions[conditionIndex] = *condition + // Return true if one of the fields have changed. + return !isEqual +} + +// IsStaticPod returns true if the pod is a static pod. +func IsStaticPod(pod *v1.Pod) bool { + source, err := GetPodSource(pod) + return err == nil && source != ApiserverSource +} + +// GetPodSource returns the source of the pod based on the annotation. +func GetPodSource(pod *v1.Pod) (string, error) { + if pod.Annotations != nil { + if source, ok := pod.Annotations[ConfigSourceAnnotationKey]; ok { + return source, nil + } + } + return "", fmt.Errorf("cannot get source of pod %q", pod.UID) +} diff --git a/pkg/yurtmanager/controller/daemonpodupdater/kubernetes/pod_util_test.go b/pkg/yurtmanager/controller/util/pod/pod_util_test.go similarity index 50% rename from pkg/yurtmanager/controller/daemonpodupdater/kubernetes/pod_util_test.go rename to pkg/yurtmanager/controller/util/pod/pod_util_test.go index e9d55223cb2..077c3d0e9bd 100644 --- a/pkg/yurtmanager/controller/daemonpodupdater/kubernetes/pod_util_test.go +++ b/pkg/yurtmanager/controller/util/pod/pod_util_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubernetes +package pod import ( "testing" @@ -22,6 +22,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" ) func newPod(now metav1.Time, ready bool, beforeSec int) *v1.Pod { @@ -78,3 +79,92 @@ func TestIsPodAvailable(t *testing.T) { } } } + +func newStaticPod(podName string, nodeName string, namespace string, isStaticPod bool) *v1.Pod { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + ConfigSourceAnnotationKey: "true", + }, + Name: podName + "-" + rand.String(10), + Namespace: namespace, + }, + Spec: v1.PodSpec{NodeName: nodeName}, + } + + if isStaticPod { + pod.Name = podName + "-" + nodeName + pod.ObjectMeta.OwnerReferences = []metav1.OwnerReference{{Kind: "Node"}} + } else { + pod.Annotations = nil + } + + return pod +} + +func TestIsStaticPod(t *testing.T) { + tests := []struct { + name string + pod *v1.Pod + want bool + }{ + { + name: "test1", + pod: newStaticPod("test1", "test1", "", true), + want: true, + }, + { + name: "test2", + pod: newStaticPod("test2", "test2", "", false), + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsStaticPod(tt.pod); got != tt.want { + t.Errorf("IsStaticPod() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestUpdatePodCondition(t *testing.T) { + now := metav1.Now() + tests := []struct { + name string + status *v1.PodStatus + condition *v1.PodCondition + want bool + }{ + { + name: "test1", + status: &newPod(now, false, 0).Status, + condition: &v1.PodCondition{}, + want: true, + }, + { + name: "test2", + status: &newPod(now, true, 0).Status, + condition: &v1.PodCondition{ + Type: v1.PodReady, + }, + want: true, + }, + { + name: "test3", + status: &newPod(now, true, 0).Status, + condition: &v1.PodCondition{ + Type: v1.PodReady, + Status: "True", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := UpdatePodCondition(tt.status, tt.condition); got != tt.want { + t.Errorf("UpdatePodCondition() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info.go b/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info.go index 6027fe708c1..ebb9e3b3a37 100644 --- a/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info.go +++ b/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info.go @@ -28,6 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + podutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/pod" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/util" ) @@ -81,7 +82,7 @@ func New(c client.Client, instance *appsv1alpha1.YurtStaticSet, workerPodName, h } // The name format of mirror static pod is `StaticPodName-NodeName` - if util.Hyphen(instance.Name, nodeName) == pod.Name && util.IsStaticPod(&pod) { + if util.Hyphen(instance.Name, nodeName) == pod.Name && podutil.IsStaticPod(&pod) { // initialize static pod info if err := initStaticPodInfo(instance, c, nodeName, hash, &podList.Items[i], infos); err != nil { return nil, err diff --git a/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info_test.go b/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info_test.go index f69c0e4efc0..17cc3fd471c 100644 --- a/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info_test.go +++ b/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info_test.go @@ -28,6 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + podutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/pod" ) const ( @@ -60,6 +61,9 @@ func newNodes(nodeNames []string) []client.Object { func newPod(podName string, nodeName string, namespace string, isStaticPod bool) *corev1.Pod { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + podutil.ConfigSourceAnnotationKey: "true", + }, Name: UpgradeWorkerPodPrefix + podName + "-" + rand.String(10), Namespace: namespace, }, @@ -86,8 +90,12 @@ func newStaticPod() *appsv1alpha1.YurtStaticSet { return &appsv1alpha1.YurtStaticSet{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + podutil.ConfigSourceAnnotationKey: "true", + }, Name: fakeStaticPodName, - Namespace: metav1.NamespaceDefault}, + Namespace: metav1.NamespaceDefault, + }, Spec: appsv1alpha1.YurtStaticSetSpec{}, Status: appsv1alpha1.YurtStaticSetStatus{}, } diff --git a/pkg/yurtmanager/controller/yurtstaticset/util/util.go b/pkg/yurtmanager/controller/yurtstaticset/util/util.go index 0c5075b2c2e..41cacde8854 100644 --- a/pkg/yurtmanager/controller/yurtstaticset/util/util.go +++ b/pkg/yurtmanager/controller/yurtstaticset/util/util.go @@ -33,6 +33,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + nodeutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/node" + podutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/pod" ) const ( @@ -113,32 +115,18 @@ func NodeReadyByName(c client.Client, nodeName string) (bool, error) { return false, err } - _, nc := GetNodeCondition(&node.Status, corev1.NodeReady) + _, nc := nodeutil.GetNodeCondition(&node.Status, corev1.NodeReady) return nc != nil && nc.Status == corev1.ConditionTrue, nil } -// GetNodeCondition extracts the provided condition from the given status and returns that. -// Returns nil and -1 if the condition is not present, and the index of the located condition. -func GetNodeCondition(status *corev1.NodeStatus, conditionType corev1.NodeConditionType) (int, *corev1.NodeCondition) { - if status == nil { - return -1, nil - } - for i := range status.Conditions { - if status.Conditions[i].Type == conditionType { - return i, &status.Conditions[i] - } - } - return -1, nil -} - // SetPodUpgradeCondition set pod condition `PodNeedUpgrade` to the specified value func SetPodUpgradeCondition(c client.Client, status corev1.ConditionStatus, pod *corev1.Pod) error { cond := &corev1.PodCondition{ Type: PodNeedUpgrade, Status: status, } - if change := UpdatePodCondition(&pod.Status, cond); change { + if change := podutil.UpdatePodCondition(&pod.Status, cond); change { if err := c.Status().Update(context.TODO(), pod, &client.UpdateOptions{}); err != nil { return err } @@ -146,65 +134,3 @@ func SetPodUpgradeCondition(c client.Client, status corev1.ConditionStatus, pod return nil } - -// UpdatePodCondition updates existing pod condition or creates a new one. Sets LastTransitionTime to now if the -// status has changed. -// Returns true if pod condition has changed or has been added. -func UpdatePodCondition(status *corev1.PodStatus, condition *corev1.PodCondition) bool { - condition.LastTransitionTime = metav1.Now() - // Try to find this pod condition. - conditionIndex, oldCondition := GetPodCondition(status, condition.Type) - - if oldCondition == nil { - // We are adding new pod condition. - status.Conditions = append(status.Conditions, *condition) - return true - } - // We are updating an existing condition, so we need to check if it has changed. - if condition.Status == oldCondition.Status { - condition.LastTransitionTime = oldCondition.LastTransitionTime - } - - isEqual := condition.Status == oldCondition.Status && - condition.Reason == oldCondition.Reason && - condition.Message == oldCondition.Message && - condition.LastProbeTime.Equal(&oldCondition.LastProbeTime) && - condition.LastTransitionTime.Equal(&oldCondition.LastTransitionTime) - - status.Conditions[conditionIndex] = *condition - // Return true if one of the fields have changed. - return !isEqual -} - -// GetPodCondition extracts the provided condition from the given status and returns that. -// Returns nil and -1 if the condition is not present, and the index of the located condition. -func GetPodCondition(status *corev1.PodStatus, conditionType corev1.PodConditionType) (int, *corev1.PodCondition) { - if status == nil { - return -1, nil - } - return GetPodConditionFromList(status.Conditions, conditionType) -} - -// GetPodConditionFromList extracts the provided condition from the given list of condition and -// returns the index of the condition and the condition. Returns -1 and nil if the condition is not present. -func GetPodConditionFromList(conditions []corev1.PodCondition, conditionType corev1.PodConditionType) (int, *corev1.PodCondition) { - if conditions == nil { - return -1, nil - } - for i := range conditions { - if conditions[i].Type == conditionType { - return i, &conditions[i] - } - } - return -1, nil -} - -// IsStaticPod judges whether a pod is static by its OwnerReference -func IsStaticPod(pod *corev1.Pod) bool { - for _, ownerRef := range pod.GetOwnerReferences() { - if ownerRef.Kind == "Node" { - return true - } - } - return false -} diff --git a/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go b/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go index 01bbff4d14b..4fdd470ca8b 100644 --- a/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go +++ b/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go @@ -43,6 +43,8 @@ import ( appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" "github.com/openyurtio/openyurt/cmd/yurt-manager/names" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + nodeutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/node" + podutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/pod" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/config" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/util" @@ -221,7 +223,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { return reqs } - if !util.IsStaticPod(pod) { + if !podutil.IsStaticPod(pod) { return reqs } @@ -252,8 +254,8 @@ func nodeTurnReady(evt event.UpdateEvent) bool { oldNode := evt.ObjectOld.(*corev1.Node) newNode := evt.ObjectNew.(*corev1.Node) - _, onc := util.GetNodeCondition(&oldNode.Status, corev1.NodeReady) - _, nnc := util.GetNodeCondition(&newNode.Status, corev1.NodeReady) + _, onc := nodeutil.GetNodeCondition(&oldNode.Status, corev1.NodeReady) + _, nnc := nodeutil.GetNodeCondition(&newNode.Status, corev1.NodeReady) oldReady := (onc != nil) && ((onc.Status == corev1.ConditionFalse) || (onc.Status == corev1.ConditionUnknown)) newReady := (nnc != nil) && (nnc.Status == corev1.ConditionTrue) From 7697c53e46a7386166090e0ff4a89cbbcfb92e6f Mon Sep 17 00:00:00 2001 From: vie-serendipity <60083692+vie-serendipity@users.noreply.github.com> Date: Tue, 12 Sep 2023 16:10:38 +0800 Subject: [PATCH 10/13] Add yurtappoverrider (#1684) * adjust CRD and add wildcard * add configuration of deployment webhook * add register of webhook and controller * call deployment webhook directly * remove flatternYaml and add more info * modify deployment webhook failurePolicy * do some modifies * rename missing configrender to overrider * remove the outer of * * fix problems in the comments * fix error in e2e of yurtappoverrider * fix error in e2e of yurtappoverrider * fix error in validating webhook of yurtappoverrider * add more UT * gci writting ut * fix error in ut * add more UT * fix problems in the comments * fix problems in the comments * fix problems in the comments --- .../apps.openyurt.io_yurtappoverriders.yaml | 144 ++++++++ .../yurt-manager-auto-generated.yaml | 71 ++++ cmd/yurt-manager/app/options/options.go | 36 +- .../app/options/yurtappoverridercontroller.go | 60 ++++ cmd/yurt-manager/names/controller_names.go | 2 + go.mod | 2 +- .../apps/v1alpha1/yurtappoverrider_types.go | 107 ++++++ .../apps/v1alpha1/zz_generated.deepcopy.go | 171 +++++++++ .../controller/apis/config/types.go | 4 + pkg/yurtmanager/controller/controller.go | 2 + .../yurtappoverrider/config/types.go | 21 ++ .../yurtappoverrider_controller.go | 150 ++++++++ .../yurtappoverrider_controller_test.go | 118 +++++++ .../v1alpha1/deploymentrender_default.go | 156 +++++++++ .../v1alpha1/deploymentrender_handler.go | 56 +++ .../v1alpha1/deploymentrender_webhook_test.go | 326 ++++++++++++++++++ .../deploymentrender/v1alpha1/item_control.go | 43 +++ .../v1alpha1/item_control_test.go | 104 ++++++ .../v1alpha1/patch_control.go | 71 ++++ .../v1alpha1/patch_control_test.go | 94 +++++ pkg/yurtmanager/webhook/server.go | 4 + .../v1alpha1/yurtappoverrider_default.go | 36 ++ .../v1alpha1/yurtappoverrider_handler.go | 56 +++ .../v1alpha1/yurtappoverrider_validation.go | 111 ++++++ test/e2e/yurt/yurtappoverrider.go | 293 ++++++++++++++++ 25 files changed, 2222 insertions(+), 16 deletions(-) create mode 100644 charts/yurt-manager/crds/apps.openyurt.io_yurtappoverriders.yaml create mode 100644 cmd/yurt-manager/app/options/yurtappoverridercontroller.go create mode 100644 pkg/apis/apps/v1alpha1/yurtappoverrider_types.go create mode 100644 pkg/yurtmanager/controller/yurtappoverrider/config/types.go create mode 100644 pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go create mode 100644 pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller_test.go create mode 100644 pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_default.go create mode 100644 pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_handler.go create mode 100644 pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_webhook_test.go create mode 100644 pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control.go create mode 100644 pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control_test.go create mode 100644 pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control.go create mode 100644 pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control_test.go create mode 100644 pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_default.go create mode 100644 pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_handler.go create mode 100644 pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_validation.go create mode 100644 test/e2e/yurt/yurtappoverrider.go diff --git a/charts/yurt-manager/crds/apps.openyurt.io_yurtappoverriders.yaml b/charts/yurt-manager/crds/apps.openyurt.io_yurtappoverriders.yaml new file mode 100644 index 00000000000..d2b08cbd0a0 --- /dev/null +++ b/charts/yurt-manager/crds/apps.openyurt.io_yurtappoverriders.yaml @@ -0,0 +1,144 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + name: yurtappoverriders.apps.openyurt.io +spec: + group: apps.openyurt.io + names: + kind: YurtAppOverrider + listKind: YurtAppOverriderList + plural: yurtappoverriders + shortNames: + - yao + singular: yurtappoverrider + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The subject kind of this overrider. + jsonPath: .subject.kind + name: Subject + type: string + - description: The subject name of this overrider. + jsonPath: .subject.name + name: Name + type: string + - description: CreationTimestamp is a timestamp representing the server time when + this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. + jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + entries: + items: + description: Describe detailed multi-region configuration of the subject + Entry describe a set of nodepools and their shared or identical configurations + properties: + items: + items: + description: Item represents configuration to be injected. Only + one of its members may be specified. + properties: + image: + description: ImageItem specifies the corresponding container + and the claimed image + properties: + containerName: + description: ContainerName represents name of the container + in which the Image will be replaced + type: string + imageClaim: + description: ImageClaim represents the claimed image name + which is injected into the container above + type: string + required: + - containerName + - imageClaim + type: object + replicas: + format: int32 + type: integer + type: object + type: array + patches: + description: Convert Patch struct into json patch operation + items: + properties: + operation: + description: Operation represents the operation + enum: + - add + - remove + - replace + type: string + path: + description: Path represents the path in the json patch + type: string + value: + description: Indicates the value of json patch + x-kubernetes-preserve-unknown-fields: true + required: + - operation + - path + type: object + type: array + pools: + items: + type: string + type: array + required: + - pools + type: object + type: array + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + subject: + description: Describe the object Entries belongs + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: Name is the name of YurtAppSet or YurtAppDaemon + type: string + required: + - name + type: object + required: + - entries + - subject + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml index 46800849e86..1b452b66bcb 100644 --- a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml +++ b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml @@ -159,6 +159,14 @@ rules: - get - patch - update +- apiGroups: + - apps.openyurt.io + resources: + - yurtappoverriders + verbs: + - get + - list + - watch - apiGroups: - apps.openyurt.io resources: @@ -516,6 +524,26 @@ metadata: creationTimestamp: null name: yurt-manager-mutating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: yurt-manager-webhook-service + namespace: {{ .Release.Namespace }} + path: /mutate-apps-v1-deployment + failurePolicy: Ignore + name: mutate.apps.v1.deployment + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 @@ -618,6 +646,27 @@ webhooks: resources: - yurtappdaemons sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: yurt-manager-webhook-service + namespace: {{ .Release.Namespace }} + path: /mutate-apps-openyurt-io-v1alpha1-yurtappoverrider + failurePolicy: Fail + name: mutate.apps.v1alpha1.yurtappoverrider.openyurt.io + rules: + - apiGroups: + - apps.openyurt.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - yurtappoverriders + sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -789,6 +838,28 @@ webhooks: resources: - yurtappdaemons sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: yurt-manager-webhook-service + namespace: {{ .Release.Namespace }} + path: /validate-apps-openyurt-io-v1alpha1-yurtappoverrider + failurePolicy: Fail + name: validate.apps.v1alpha1.yurtappoverrider.openyurt.io + rules: + - apiGroups: + - apps.openyurt.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - yurtappoverriders + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/cmd/yurt-manager/app/options/options.go b/cmd/yurt-manager/app/options/options.go index e0940d30971..a0f8f161f11 100644 --- a/cmd/yurt-manager/app/options/options.go +++ b/cmd/yurt-manager/app/options/options.go @@ -25,26 +25,28 @@ import ( // YurtManagerOptions is the main context object for the yurt-manager. type YurtManagerOptions struct { - Generic *GenericOptions - NodePoolController *NodePoolControllerOptions - GatewayPickupController *GatewayPickupControllerOptions - YurtStaticSetController *YurtStaticSetControllerOptions - YurtAppSetController *YurtAppSetControllerOptions - YurtAppDaemonController *YurtAppDaemonControllerOptions - PlatformAdminController *PlatformAdminControllerOptions + Generic *GenericOptions + NodePoolController *NodePoolControllerOptions + GatewayPickupController *GatewayPickupControllerOptions + YurtStaticSetController *YurtStaticSetControllerOptions + YurtAppSetController *YurtAppSetControllerOptions + YurtAppDaemonController *YurtAppDaemonControllerOptions + PlatformAdminController *PlatformAdminControllerOptions + YurtAppOverriderController *YurtAppOverriderControllerOptions } // NewYurtManagerOptions creates a new YurtManagerOptions with a default config. func NewYurtManagerOptions() (*YurtManagerOptions, error) { s := YurtManagerOptions{ - Generic: NewGenericOptions(), - NodePoolController: NewNodePoolControllerOptions(), - GatewayPickupController: NewGatewayPickupControllerOptions(), - YurtStaticSetController: NewYurtStaticSetControllerOptions(), - YurtAppSetController: NewYurtAppSetControllerOptions(), - YurtAppDaemonController: NewYurtAppDaemonControllerOptions(), - PlatformAdminController: NewPlatformAdminControllerOptions(), + Generic: NewGenericOptions(), + NodePoolController: NewNodePoolControllerOptions(), + GatewayPickupController: NewGatewayPickupControllerOptions(), + YurtStaticSetController: NewYurtStaticSetControllerOptions(), + YurtAppSetController: NewYurtAppSetControllerOptions(), + YurtAppDaemonController: NewYurtAppDaemonControllerOptions(), + PlatformAdminController: NewPlatformAdminControllerOptions(), + YurtAppOverriderController: NewYurtAppOverriderControllerOptions(), } return &s, nil @@ -58,6 +60,7 @@ func (y *YurtManagerOptions) Flags(allControllers, disabledByDefaultControllers y.YurtStaticSetController.AddFlags(fss.FlagSet("yurtstaticset controller")) y.YurtAppDaemonController.AddFlags(fss.FlagSet("yurtappdaemon controller")) y.PlatformAdminController.AddFlags(fss.FlagSet("iot controller")) + y.YurtAppOverriderController.AddFlags(fss.FlagSet("yurtappoverrider controller")) // Please Add Other controller flags @kadisi return fss @@ -72,6 +75,7 @@ func (y *YurtManagerOptions) Validate(allControllers []string, controllerAliases errs = append(errs, y.YurtStaticSetController.Validate()...) errs = append(errs, y.YurtAppDaemonController.Validate()...) errs = append(errs, y.PlatformAdminController.Validate()...) + errs = append(errs, y.YurtAppOverriderController.Validate()...) return utilerrors.NewAggregate(errs) } @@ -92,10 +96,12 @@ func (y *YurtManagerOptions) ApplyTo(c *config.Config, controllerAliases map[str if err := y.PlatformAdminController.ApplyTo(&c.ComponentConfig.PlatformAdminController); err != nil { return err } + if err := y.YurtAppOverriderController.ApplyTo(&c.ComponentConfig.YurtAppOverriderController); err != nil { + return err + } if err := y.GatewayPickupController.ApplyTo(&c.ComponentConfig.GatewayPickupController); err != nil { return err } - return nil } diff --git a/cmd/yurt-manager/app/options/yurtappoverridercontroller.go b/cmd/yurt-manager/app/options/yurtappoverridercontroller.go new file mode 100644 index 00000000000..da9fd8f60ee --- /dev/null +++ b/cmd/yurt-manager/app/options/yurtappoverridercontroller.go @@ -0,0 +1,60 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "github.com/spf13/pflag" + + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappoverrider/config" +) + +type YurtAppOverriderControllerOptions struct { + *config.YurtAppOverriderControllerConfiguration +} + +func NewYurtAppOverriderControllerOptions() *YurtAppOverriderControllerOptions { + return &YurtAppOverriderControllerOptions{ + &config.YurtAppOverriderControllerConfiguration{}, + } +} + +// AddFlags adds flags related to nodepool for yurt-manager to the specified FlagSet. +func (n *YurtAppOverriderControllerOptions) AddFlags(fs *pflag.FlagSet) { + if n == nil { + return + } + + //fs.BoolVar(&n.CreateDefaultPool, "create-default-pool", n.CreateDefaultPool, "Create default cloud/edge pools if indicated.") +} + +// ApplyTo fills up nodepool config with options. +func (o *YurtAppOverriderControllerOptions) ApplyTo(cfg *config.YurtAppOverriderControllerConfiguration) error { + if o == nil { + return nil + } + + return nil +} + +// Validate checks validation of YurtAppOverriderControllerOptions. +func (o *YurtAppOverriderControllerOptions) Validate() []error { + if o == nil { + return nil + } + errs := []error{} + return errs +} diff --git a/cmd/yurt-manager/names/controller_names.go b/cmd/yurt-manager/names/controller_names.go index f00dc8d04d0..9749256c90f 100644 --- a/cmd/yurt-manager/names/controller_names.go +++ b/cmd/yurt-manager/names/controller_names.go @@ -25,6 +25,7 @@ const ( ServiceTopologyEndpointSliceController = "service-topology-endpointslice-controller" YurtAppSetController = "yurt-app-set-controller" YurtAppDaemonController = "yurt-app-daemon-controller" + YurtAppOverriderController = "yurt-app-overrider-controller" YurtStaticSetController = "yurt-static-set-controller" YurtCoordinatorCertController = "yurt-coordinator-cert-controller" DelegateLeaseController = "delegate-lease-controller" @@ -47,6 +48,7 @@ func YurtManagerControllerAliases() map[string]string { "yurtappset": YurtAppSetController, "yurtappdaemon": YurtAppDaemonController, "yurtstaticset": YurtStaticSetController, + "yurtappoverrider": YurtAppOverriderController, "yurtcoordinatorcert": YurtCoordinatorCertController, "delegatelease": DelegateLeaseController, "podbinding": PodBindingController, diff --git a/go.mod b/go.mod index 3c16f9c42eb..bda596c041b 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/edgexfoundry/go-mod-core-contracts/v2 v2.3.0 github.com/edgexfoundry/go-mod-core-contracts/v3 v3.0.0 + github.com/evanphx/json-patch v5.6.0+incompatible github.com/go-resty/resty/v2 v2.7.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/uuid v1.3.0 @@ -87,7 +88,6 @@ require ( github.com/cyphar/filepath-securejoin v0.2.3 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/emicklei/go-restful v2.16.0+incompatible // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/fatih/camelcase v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect diff --git a/pkg/apis/apps/v1alpha1/yurtappoverrider_types.go b/pkg/apis/apps/v1alpha1/yurtappoverrider_types.go new file mode 100644 index 00000000000..c7d658b7bb8 --- /dev/null +++ b/pkg/apis/apps/v1alpha1/yurtappoverrider_types.go @@ -0,0 +1,107 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// ImageItem specifies the corresponding container and the claimed image +type ImageItem struct { + // ContainerName represents name of the container + // in which the Image will be replaced + ContainerName string `json:"containerName"` + // ImageClaim represents the claimed image name + //which is injected into the container above + ImageClaim string `json:"imageClaim"` +} + +// Item represents configuration to be injected. +// Only one of its members may be specified. +type Item struct { + // +optional + Image *ImageItem `json:"image,omitempty"` + // +optional + Replicas *int32 `json:"replicas,omitempty"` +} + +type Operation string + +const ( + ADD Operation = "add" // json patch + REMOVE Operation = "remove" // json patch + REPLACE Operation = "replace" // json patch +) + +type Patch struct { + // Path represents the path in the json patch + Path string `json:"path"` + // Operation represents the operation + // +kubebuilder:validation:Enum=add;remove;replace + Operation Operation `json:"operation"` + // Indicates the value of json patch + // +optional + Value apiextensionsv1.JSON `json:"value,omitempty"` +} + +// Describe detailed multi-region configuration of the subject +// Entry describe a set of nodepools and their shared or identical configurations +type Entry struct { + Pools []string `json:"pools"` + // +optional + Items []Item `json:"items,omitempty"` + // Convert Patch struct into json patch operation + // +optional + Patches []Patch `json:"patches,omitempty"` +} + +// Describe the object Entries belongs +type Subject struct { + metav1.TypeMeta `json:",inline"` + // Name is the name of YurtAppSet or YurtAppDaemon + Name string `json:"name"` +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:resource:shortName=yao +// +kubebuilder:printcolumn:name="Subject",type="string",JSONPath=".subject.kind",description="The subject kind of this overrider." +// +kubebuilder:printcolumn:name="Name",type="string",JSONPath=".subject.name",description="The subject name of this overrider." +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC." + +type YurtAppOverrider struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Subject Subject `json:"subject"` + Entries []Entry `json:"entries"` +} + +// YurtAppOverriderList contains a list of YurtAppOverrider +// +kubebuilder:object:root=true +type YurtAppOverriderList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []YurtAppOverrider `json:"items"` +} + +func init() { + SchemeBuilder.Register(&YurtAppOverrider{}, &YurtAppOverriderList{}) +} diff --git a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go index f732bdcaa9b..bbd03f00dfb 100644 --- a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -45,6 +45,80 @@ func (in *DeploymentTemplateSpec) DeepCopy() *DeploymentTemplateSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Entry) DeepCopyInto(out *Entry) { + *out = *in + if in.Pools != nil { + in, out := &in.Pools, &out.Pools + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Item, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Patches != nil { + in, out := &in.Patches, &out.Patches + *out = make([]Patch, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Entry. +func (in *Entry) DeepCopy() *Entry { + if in == nil { + return nil + } + out := new(Entry) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImageItem) DeepCopyInto(out *ImageItem) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageItem. +func (in *ImageItem) DeepCopy() *ImageItem { + if in == nil { + return nil + } + out := new(ImageItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Item) DeepCopyInto(out *Item) { + *out = *in + if in.Image != nil { + in, out := &in.Image, &out.Image + *out = new(ImageItem) + **out = **in + } + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Item. +func (in *Item) DeepCopy() *Item { + if in == nil { + return nil + } + out := new(Item) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodePool) DeepCopyInto(out *NodePool) { *out = *in @@ -165,6 +239,22 @@ func (in *NodePoolStatus) DeepCopy() *NodePoolStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Patch) DeepCopyInto(out *Patch) { + *out = *in + in.Value.DeepCopyInto(&out.Value) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Patch. +func (in *Patch) DeepCopy() *Patch { + if in == nil { + return nil + } + out := new(Patch) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Pool) DeepCopyInto(out *Pool) { *out = *in @@ -215,6 +305,22 @@ func (in *StatefulSetTemplateSpec) DeepCopy() *StatefulSetTemplateSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Subject) DeepCopyInto(out *Subject) { + *out = *in + out.TypeMeta = in.TypeMeta +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Subject. +func (in *Subject) DeepCopy() *Subject { + if in == nil { + return nil + } + out := new(Subject) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Topology) DeepCopyInto(out *Topology) { *out = *in @@ -400,6 +506,71 @@ func (in *YurtAppDaemonStatus) DeepCopy() *YurtAppDaemonStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *YurtAppOverrider) DeepCopyInto(out *YurtAppOverrider) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Subject = in.Subject + if in.Entries != nil { + in, out := &in.Entries, &out.Entries + *out = make([]Entry, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new YurtAppOverrider. +func (in *YurtAppOverrider) DeepCopy() *YurtAppOverrider { + if in == nil { + return nil + } + out := new(YurtAppOverrider) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *YurtAppOverrider) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *YurtAppOverriderList) DeepCopyInto(out *YurtAppOverriderList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]YurtAppOverrider, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new YurtAppOverriderList. +func (in *YurtAppOverriderList) DeepCopy() *YurtAppOverriderList { + if in == nil { + return nil + } + out := new(YurtAppOverriderList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *YurtAppOverriderList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *YurtAppSet) DeepCopyInto(out *YurtAppSet) { *out = *in diff --git a/pkg/yurtmanager/controller/apis/config/types.go b/pkg/yurtmanager/controller/apis/config/types.go index 3be8f661277..aaeb974bf98 100644 --- a/pkg/yurtmanager/controller/apis/config/types.go +++ b/pkg/yurtmanager/controller/apis/config/types.go @@ -23,6 +23,7 @@ import ( platformadminconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin/config" gatewaypickupconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewaypickup/config" yurtappdaemonconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon/config" + yurtappoverriderconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappoverrider/config" yurtappsetconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/config" yurtstaticsetconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/config" ) @@ -48,6 +49,9 @@ type YurtManagerConfiguration struct { // PlatformAdminControllerConfiguration holds configuration for PlatformAdminController related features. PlatformAdminController platformadminconfig.PlatformAdminControllerConfiguration + + // YurtAppOverriderControllerConfiguration holds configuration for YurtAppOverriderController related features. + YurtAppOverriderController yurtappoverriderconfig.YurtAppOverriderControllerConfiguration } type GenericConfiguration struct { diff --git a/pkg/yurtmanager/controller/controller.go b/pkg/yurtmanager/controller/controller.go index d30b7d8e2dd..aaf66932148 100644 --- a/pkg/yurtmanager/controller/controller.go +++ b/pkg/yurtmanager/controller/controller.go @@ -38,6 +38,7 @@ import ( servicetopologyendpoints "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/endpoints" servicetopologyendpointslice "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/endpointslice" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappoverrider" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset" yurtcoordinatorcert "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/cert" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/delegatelease" @@ -83,6 +84,7 @@ func NewControllerInitializers() map[string]InitFunc { register(names.YurtStaticSetController, yurtstaticset.Add) register(names.YurtAppSetController, yurtappset.Add) register(names.YurtAppDaemonController, yurtappdaemon.Add) + register(names.YurtAppOverriderController, yurtappoverrider.Add) register(names.PlatformAdminController, platformadmin.Add) register(names.GatewayPickupController, gatewaypickup.Add) register(names.GatewayDNSController, dns.Add) diff --git a/pkg/yurtmanager/controller/yurtappoverrider/config/types.go b/pkg/yurtmanager/controller/yurtappoverrider/config/types.go new file mode 100644 index 00000000000..e11801f07f3 --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappoverrider/config/types.go @@ -0,0 +1,21 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +// YurtAppOverriderControllerConfiguration contains elements describing YurtAppOverriderController. +type YurtAppOverriderControllerConfiguration struct { +} diff --git a/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go b/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go new file mode 100644 index 00000000000..0fd375a3ce8 --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go @@ -0,0 +1,150 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yurtappoverrider + +import ( + "context" + "flag" + "fmt" + "time" + + v1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappoverrider/config" +) + +func init() { + flag.IntVar(&concurrentReconciles, "yurtappoverrider-workers", concurrentReconciles, "Max concurrent workers for YurtAppOverrider controller.") +} + +var ( + concurrentReconciles = 3 + controllerResource = appsv1alpha1.SchemeGroupVersion.WithResource("yurtappoverriders") +) + +const ( + ControllerName = "yurtappoverrider" +) + +func Format(format string, args ...interface{}) string { + s := fmt.Sprintf(format, args...) + return fmt.Sprintf("%s: %s", ControllerName, s) +} + +// Add creates a new YurtAppOverrider Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { + if _, err := mgr.GetRESTMapper().KindFor(controllerResource); err != nil { + klog.Infof("resource %s doesn't exist", controllerResource.String()) + return err + } + + return add(mgr, newReconciler(c, mgr)) +} + +var _ reconcile.Reconciler = &ReconcileYurtAppOverrider{} + +// ReconcileYurtAppOverrider reconciles a YurtAppOverrider object +type ReconcileYurtAppOverrider struct { + client.Client + Configuration config.YurtAppOverriderControllerConfiguration +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { + return &ReconcileYurtAppOverrider{ + Client: mgr.GetClient(), + Configuration: c.ComponentConfig.YurtAppOverriderController, + } +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New(ControllerName, mgr, controller.Options{ + Reconciler: r, MaxConcurrentReconciles: concurrentReconciles, + }) + if err != nil { + return err + } + + // Watch for changes to YurtAppOverrider + err = c.Watch(&source.Kind{Type: &appsv1alpha1.YurtAppOverrider{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + + return nil +} + +// +kubebuilder:rbac:groups=apps.openyurt.io,resources=yurtappoverriders,verbs=get;list;watch +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=list;watch;update +// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete + +// Reconcile reads that state of the cluster for a YurtAppOverrider object and makes changes based on the state read +// and what is in the YurtAppOverrider.Spec +func (r *ReconcileYurtAppOverrider) Reconcile(_ context.Context, request reconcile.Request) (reconcile.Result, error) { + + // Note !!!!!!!!!! + // We strongly recommend use Format() to encapsulation because Format() can print logs by module + // @kadisi + klog.Infof(Format("Reconcile YurtAppOverrider %s/%s", request.Namespace, request.Name)) + + // Fetch the YurtAppOverrider instance + instance := &appsv1alpha1.YurtAppOverrider{} + err := r.Get(context.TODO(), request.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + + if instance.DeletionTimestamp != nil { + return reconcile.Result{}, nil + } + + deployments := v1.DeploymentList{} + + if err := r.List(context.TODO(), &deployments); err != nil { + return reconcile.Result{}, err + } + + for _, deployment := range deployments.Items { + if deployment.OwnerReferences[0].Kind == instance.Subject.Kind && deployment.OwnerReferences[0].Name == instance.Subject.Name { + if deployment.Annotations == nil { + deployment.Annotations = make(map[string]string) + } + deployment.Annotations["LastOverrideTime"] = time.Now().String() + if err := r.Client.Update(context.TODO(), &deployment); err != nil { + return reconcile.Result{}, err + } + } + } + + return reconcile.Result{}, nil +} diff --git a/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller_test.go b/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller_test.go new file mode 100644 index 00000000000..646ce4f5d89 --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller_test.go @@ -0,0 +1,118 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package yurtappoverrider + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +var ( + replica int32 = 3 +) + +var daemonDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps.openyurt.io/v1alpha1", + Kind: "YurtAppDaemon", + Name: "yurtappdaemon", + }}, + Labels: map[string]string{ + "apps.openyurt.io/pool-name": "nodepool-test", + }, + }, + Status: appsv1.DeploymentStatus{}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replica, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, +} + +var overrider = &v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo", + Namespace: "default", + }, + Subject: v1alpha1.Subject{ + Name: "yurtappdaemon", + TypeMeta: metav1.TypeMeta{ + Kind: "YurtAppDaemon", + APIVersion: "apps.openyurt.io/v1alpha1", + }, + }, + Entries: []v1alpha1.Entry{ + { + Pools: []string{"*"}, + }, + }, +} + +func TestReconcile(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Logf("failed to add yurt custom resource") + return + } + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Logf("failed to add kubernetes clint-go custom resource") + return + } + reconciler := ReconcileYurtAppOverrider{ + Client: fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(daemonDeployment, overrider).Build(), + } + _, err := reconciler.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: "default", + Name: "demo", + }, + }) + if err != nil { + t.Logf("fail to call Reconcile: %v", err) + } +} diff --git a/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_default.go b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_default.go new file mode 100644 index 00000000000..e1433cf75ce --- /dev/null +++ b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_default.go @@ -0,0 +1,156 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "fmt" + "strings" + + v1 "k8s.io/api/apps/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/adapter" +) + +var ( + resources = []string{"YurtAppSet", "YurtAppDaemon"} +) + +func contain(kind string, resources []string) bool { + for _, v := range resources { + if kind == v { + return true + } + } + return false +} + +// Default satisfies the defaulting webhook interface. +func (webhook *DeploymentRenderHandler) Default(ctx context.Context, obj runtime.Object) error { + deployment, ok := obj.(*v1.Deployment) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a Deployment but got a %T", obj)) + } + if deployment.OwnerReferences == nil { + return nil + } + if !contain(deployment.OwnerReferences[0].Kind, resources) { + return nil + } + // Get YurtAppSet/YurtAppDaemon resource of this deployment + app := deployment.OwnerReferences[0] + var instance client.Object + switch app.Kind { + case "YurtAppSet": + instance = &v1alpha1.YurtAppSet{} + case "YurtAppDaemon": + instance = &v1alpha1.YurtAppDaemon{} + default: + return nil + } + if err := webhook.Client.Get(ctx, client.ObjectKey{ + Namespace: deployment.Namespace, + Name: app.Name, + }, instance); err != nil { + return err + } + // Get nodepool of deployment + nodepool := deployment.Labels["apps.openyurt.io/pool-name"] + + // resume deployment + switch app.Kind { + case "YurtAppSet": + var replicas int32 + yas := instance.(*v1alpha1.YurtAppSet) + revision := yas.Status.CurrentRevision + if yas.Spec.WorkloadTemplate.DeploymentTemplate != nil && yas.Spec.WorkloadTemplate.DeploymentTemplate.Spec.Replicas != nil { + replicas = *yas.Spec.WorkloadTemplate.DeploymentTemplate.Spec.Replicas + } + deploymentAdapter := adapter.DeploymentAdapter{ + Client: webhook.Client, + Scheme: webhook.Scheme, + } + for _, pool := range yas.Spec.Topology.Pools { + if pool.Name == nodepool { + replicas = *pool.Replicas + } + } + if err := deploymentAdapter.ApplyPoolTemplate(yas, nodepool, revision, replicas, deployment); err != nil { + return err + } + case "YurtAppDaemon": + yad := instance.(*v1alpha1.YurtAppDaemon) + yad.Spec.WorkloadTemplate.DeploymentTemplate.Spec.DeepCopyInto(&deployment.Spec) + } + + // Get YurtAppOverrider resource of app(1 to 1) + var allOverriderList v1alpha1.YurtAppOverriderList + //listOptions := client.MatchingFields{"spec.subject.kind": app.Kind, "spec.subject.name": app.Name, "spec.subject.APIVersion": app.APIVersion} + if err := webhook.Client.List(ctx, &allOverriderList, client.InNamespace(deployment.Namespace)); err != nil { + klog.Infof("error in listing YurtAppOverrider: %v", err) + return err + } + var overriders = make([]v1alpha1.YurtAppOverrider, 0) + for _, overrider := range allOverriderList.Items { + if overrider.Subject.Kind == app.Kind && overrider.Subject.Name == app.Name && overrider.Subject.APIVersion == app.APIVersion { + overriders = append(overriders, overrider) + } + } + + if len(overriders) == 0 { + return nil + } + render := overriders[0] + + for _, entry := range render.Entries { + for _, pool := range entry.Pools { + if pool[0] == '-' && pool[1:] == nodepool { + continue + } + if pool == nodepool || pool == "*" { + // Replace items + replaceItems(deployment, entry.Items) + // json patch + for i, patch := range entry.Patches { + if strings.Contains(string(patch.Value.Raw), "{{nodepool}}") { + newPatchString := strings.ReplaceAll(string(patch.Value.Raw), "{{nodepool}}", nodepool) + entry.Patches[i].Value = apiextensionsv1.JSON{Raw: []byte(newPatchString)} + } + } + // Implement injection + dataStruct := v1.Deployment{} + pc := PatchControl{ + patches: entry.Patches, + patchObject: deployment, + dataStruct: dataStruct, + } + if err := pc.jsonMergePatch(); err != nil { + klog.Infof("fail to update patches for deployment: %v", err) + return err + } + break + } + } + } + return nil +} diff --git a/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_handler.go b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_handler.go new file mode 100644 index 00000000000..ffb4fad7d0c --- /dev/null +++ b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_handler.go @@ -0,0 +1,56 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + v1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" +) + +// SetupWebhookWithManager sets up Cluster webhooks. mutate path, validatepath, error +func (webhook *DeploymentRenderHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, string, error) { + // init + webhook.Client = mgr.GetClient() + webhook.Scheme = mgr.GetScheme() + + gvk, err := apiutil.GVKForObject(&v1.Deployment{}, mgr.GetScheme()) + if err != nil { + return "", "", err + } + return util.GenerateMutatePath(gvk), + util.GenerateValidatePath(gvk), + ctrl.NewWebhookManagedBy(mgr). + For(&v1.Deployment{}). + WithDefaulter(webhook). + Complete() +} + +// +kubebuilder:webhook:path=/mutate-apps-v1-deployment,mutating=true,failurePolicy=ignore,groups=apps,resources=deployments,verbs=create;update,versions=v1,name=mutate.apps.v1.deployment,sideEffects=None,admissionReviewVersions=v1 + +// Cluster implements a validating and defaulting webhook for Cluster. +type DeploymentRenderHandler struct { + Client client.Client + Scheme *runtime.Scheme +} + +var _ webhook.CustomDefaulter = &DeploymentRenderHandler{} diff --git a/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_webhook_test.go b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_webhook_test.go new file mode 100644 index 00000000000..1bd80a09cd8 --- /dev/null +++ b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_webhook_test.go @@ -0,0 +1,326 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package v1alpha1 + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +var ( + replica int32 = 3 +) + +var defaultAppSet = &v1alpha1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "yurtappset-patch", + Namespace: "default", + }, + Spec: v1alpha1.YurtAppSetSpec{ + Topology: v1alpha1.Topology{ + Pools: []v1alpha1.Pool{{ + Name: "nodepool-test", + Replicas: &replica}}, + }, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "test"}}, + WorkloadTemplate: v1alpha1.WorkloadTemplate{ + DeploymentTemplate: &v1alpha1.DeploymentTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "test"}, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "test"}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "test"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "nginx", Image: "nginx"}, + }, + Volumes: []corev1.Volume{ + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "configMapSource-nodepool-test", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, +} + +var defaultAppDaemon = &v1alpha1.YurtAppDaemon{ + ObjectMeta: metav1.ObjectMeta{ + Name: "yurtappdaemon", + Namespace: "default", + }, + Spec: v1alpha1.YurtAppDaemonSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "test"}}, + WorkloadTemplate: v1alpha1.WorkloadTemplate{ + DeploymentTemplate: &v1alpha1.DeploymentTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "test"}, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "test"}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "test"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "nginx", Image: "nginx"}, + }, + }, + }, + }, + }, + }, + }, +} + +var defaultDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test1", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps.openyurt.io/v1alpha1", + Kind: "YurtAppSet", + Name: "yurtappset-patch", + }}, + Labels: map[string]string{ + "apps.openyurt.io/pool-name": "nodepool-test", + }, + }, + Status: appsv1.DeploymentStatus{}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replica, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, +} + +var daemonDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test2", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps.openyurt.io/v1alpha1", + Kind: "YurtAppDaemon", + Name: "yurtappdaemon", + }}, + Labels: map[string]string{ + "apps.openyurt.io/pool-name": "nodepool-test", + }, + }, + Status: appsv1.DeploymentStatus{}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replica, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, +} + +var overrider1 = &v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo", + Namespace: "default", + }, + Subject: v1alpha1.Subject{ + Name: "yurtappset-patch", + TypeMeta: metav1.TypeMeta{ + Kind: "YurtAppSet", + APIVersion: "apps.openyurt.io/v1alpha1", + }, + }, + Entries: []v1alpha1.Entry{ + { + Pools: []string{"nodepool-test"}, + Items: []v1alpha1.Item{ + { + Image: &v1alpha1.ImageItem{ + ContainerName: "nginx", + ImageClaim: "nginx:1.18", + }, + }, + }, + Patches: []v1alpha1.Patch{ + { + Operation: v1alpha1.REPLACE, + Path: "/spec/replicas", + Value: apiextensionsv1.JSON{ + Raw: []byte("3"), + }, + }, + }, + }, + }, +} + +var overrider2 = &v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo", + Namespace: "default", + }, + Subject: v1alpha1.Subject{ + Name: "yurtappset-patch", + TypeMeta: metav1.TypeMeta{ + Kind: "YurtAppSet", + APIVersion: "apps.openyurt.io/v1alpha1", + }, + }, + Entries: []v1alpha1.Entry{ + { + Pools: []string{"*"}, + Patches: []v1alpha1.Patch{ + { + Operation: v1alpha1.ADD, + Path: "/spec/template/spec/volumes/-", + Value: apiextensionsv1.JSON{ + Raw: []byte(`{"name":"configmap-{{nodepool}}","configMap":{"name":"demo","items":[{"key": "game.properities","path": "game.properities"}]}}`), + }, + }, + }, + }, + }, +} + +var overrider3 = &v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo", + Namespace: "default", + }, + Subject: v1alpha1.Subject{ + Name: "demo", + TypeMeta: metav1.TypeMeta{ + Kind: "test", + APIVersion: "apps.openyurt.io/v1alpha1", + }, + }, + Entries: []v1alpha1.Entry{ + { + Pools: []string{"*"}, + }, + }, +} + +var overrider4 = &v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo", + Namespace: "default", + }, + Subject: v1alpha1.Subject{ + Name: "yurtappdaemon", + TypeMeta: metav1.TypeMeta{ + Kind: "YurtAppDaemon", + APIVersion: "apps.openyurt.io/v1alpha1", + }, + }, + Entries: []v1alpha1.Entry{ + { + Pools: []string{"*", "-nodepool-test"}, + }, + }, +} + +func TestDeploymentRenderHandler_Default(t *testing.T) { + tcases := []struct { + overrider *v1alpha1.YurtAppOverrider + }{ + {overrider1}, + {overrider2}, + {overrider3}, + {overrider4}, + } + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Logf("failed to add yurt custom resource") + return + } + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Logf("failed to add kubernetes clint-go custom resource") + return + } + for _, tcase := range tcases { + t.Run("", func(t *testing.T) { + webhook := &DeploymentRenderHandler{ + Client: fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(defaultAppSet, daemonDeployment, defaultDeployment, defaultAppDaemon, tcase.overrider).Build(), + Scheme: scheme, + } + if err := webhook.Default(context.TODO(), defaultDeployment); err != nil { + t.Fatal(err) + } + if err := webhook.Default(context.TODO(), daemonDeployment); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control.go b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control.go new file mode 100644 index 00000000000..83e4fcaa862 --- /dev/null +++ b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control.go @@ -0,0 +1,43 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + v1 "k8s.io/api/apps/v1" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +func replaceItems(deployment *v1.Deployment, items []v1alpha1.Item) { + for _, item := range items { + switch { + case item.Replicas != nil: + deployment.Spec.Replicas = item.Replicas + case item.Image != nil: + for i := range deployment.Spec.Template.Spec.Containers { + if deployment.Spec.Template.Spec.Containers[i].Name == item.Image.ContainerName { + deployment.Spec.Template.Spec.Containers[i].Image = item.Image.ImageClaim + } + } + for i := range deployment.Spec.Template.Spec.InitContainers { + if deployment.Spec.Template.Spec.InitContainers[i].Name == item.Image.ContainerName { + deployment.Spec.Template.Spec.InitContainers[i].Image = item.Image.ImageClaim + } + } + } + } +} diff --git a/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control_test.go b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control_test.go new file mode 100644 index 00000000000..7fb5982876f --- /dev/null +++ b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control_test.go @@ -0,0 +1,104 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +var ( + itemReplicas int32 = 3 +) + +var testItemDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Status: appsv1.DeploymentStatus{}, + Spec: appsv1.DeploymentSpec{ + Replicas: &itemReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: "initContainer", + Image: "initOld", + }, + }, + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + Volumes: []corev1.Volume{ + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "configMapSource", + }, + }, + }, + }, + }, + }, + }, + }, +} + +func TestReplaceItems(t *testing.T) { + items := []v1alpha1.Item{ + { + Image: &v1alpha1.ImageItem{ + ContainerName: "nginx", + ImageClaim: "nginx", + }, + }, + { + Image: &v1alpha1.ImageItem{ + ContainerName: "initOld", + ImageClaim: "initNew", + }, + }, + { + Replicas: &itemReplicas, + }, + } + replaceItems(testItemDeployment, items) +} diff --git a/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control.go b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control.go new file mode 100644 index 00000000000..9be0cee032a --- /dev/null +++ b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control.go @@ -0,0 +1,71 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "encoding/json" + + jsonpatch "github.com/evanphx/json-patch" + appsv1 "k8s.io/api/apps/v1" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +type PatchControl struct { + patches []v1alpha1.Patch + patchObject interface{} + // data structure + dataStruct interface{} +} + +type overrider struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value,omitempty"` +} + +// implement json patch +func (pc *PatchControl) jsonMergePatch() error { + // convert into json patch format + var patchOperations []overrider + for _, patch := range pc.patches { + single := overrider{ + Op: string(patch.Operation), + Path: patch.Path, + Value: patch.Value, + } + patchOperations = append(patchOperations, single) + } + patchBytes, err := json.Marshal(patchOperations) + if err != nil { + return err + } + patchedData, err := json.Marshal(pc.patchObject.(*appsv1.Deployment)) + if err != nil { + return err + } + // conduct json patch + patchObj, err := jsonpatch.DecodePatch(patchBytes) + if err != nil { + return err + } + patchedData, err = patchObj.Apply(patchedData) + if err != nil { + return err + } + return json.Unmarshal(patchedData, &pc.patchObject) +} diff --git a/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control_test.go b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control_test.go new file mode 100644 index 00000000000..845369ed137 --- /dev/null +++ b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +var initialReplicas int32 = 2 + +var testPatchDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps.openyurt.io/v1alpha1", + Kind: "YurtAppSet", + Name: "yurtappset-patch", + }}, + }, + Status: appsv1.DeploymentStatus{}, + Spec: appsv1.DeploymentSpec{ + Replicas: &initialReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, +} + +var patchControl = PatchControl{ + patches: []v1alpha1.Patch{ + { + Operation: v1alpha1.REPLACE, + Path: "/spec/template/spec/containers/0/image", + Value: apiextensionsv1.JSON{ + Raw: []byte(`"tomcat:1.18"`), + }, + }, + { + Operation: v1alpha1.ADD, + Path: "/spec/replicas", + Value: apiextensionsv1.JSON{ + Raw: []byte("5"), + }, + }, + }, + patchObject: testPatchDeployment, + dataStruct: appsv1.Deployment{}, +} + +func TestJsonMergePatch(t *testing.T) { + if err := patchControl.jsonMergePatch(); err != nil { + t.Fatalf("fail to call jsonMergePatch") + } + t.Logf("image:%v", testPatchDeployment.Spec.Template.Spec.Containers[0].Name) +} diff --git a/pkg/yurtmanager/webhook/server.go b/pkg/yurtmanager/webhook/server.go index 4ed3ff08e5f..7b43a79c414 100644 --- a/pkg/yurtmanager/webhook/server.go +++ b/pkg/yurtmanager/webhook/server.go @@ -30,6 +30,7 @@ import ( "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" "github.com/openyurtio/openyurt/cmd/yurt-manager/names" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller" + v1alpha1deploymentrender "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/deploymentrender/v1alpha1" v1beta1gateway "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/gateway/v1beta1" v1node "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/node/v1" v1beta1nodepool "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/nodepool/v1beta1" @@ -39,6 +40,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" webhookcontroller "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util/controller" v1alpha1yurtappdaemon "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1" + v1alpha1yurtappoverrider "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1" v1alpha1yurtappset "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtappset/v1alpha1" v1alpha1yurtstaticset "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtstaticset/v1alpha1" ) @@ -76,6 +78,8 @@ func init() { addControllerWebhook(names.YurtAppDaemonController, &v1alpha1yurtappdaemon.YurtAppDaemonHandler{}) addControllerWebhook(names.PlatformAdminController, &v1alpha1platformadmin.PlatformAdminHandler{}) addControllerWebhook(names.PlatformAdminController, &v1alpha2platformadmin.PlatformAdminHandler{}) + addControllerWebhook(names.YurtAppOverriderController, &v1alpha1yurtappoverrider.YurtAppOverriderHandler{}) + addControllerWebhook(names.YurtAppOverriderController, &v1alpha1deploymentrender.DeploymentRenderHandler{}) independentWebhooks[v1pod.WebhookName] = &v1pod.PodHandler{} independentWebhooks[v1node.WebhookName] = &v1node.NodeHandler{} diff --git a/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_default.go b/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_default.go new file mode 100644 index 00000000000..3424b412670 --- /dev/null +++ b/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_default.go @@ -0,0 +1,36 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +// Default satisfies the defaulting webhook interface. +func (webhook *YurtAppOverriderHandler) Default(ctx context.Context, obj runtime.Object) error { + _, ok := obj.(*v1alpha1.YurtAppOverrider) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider but got a %T", obj)) + } + return nil +} diff --git a/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_handler.go b/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_handler.go new file mode 100644 index 00000000000..53df018c966 --- /dev/null +++ b/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_handler.go @@ -0,0 +1,56 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" +) + +// SetupWebhookWithManager sets up Cluster webhooks. mutate path, validatepath, error +func (webhook *YurtAppOverriderHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, string, error) { + // init + webhook.Client = mgr.GetClient() + + gvk, err := apiutil.GVKForObject(&v1alpha1.YurtAppOverrider{}, mgr.GetScheme()) + if err != nil { + return "", "", err + } + return util.GenerateMutatePath(gvk), + util.GenerateValidatePath(gvk), + ctrl.NewWebhookManagedBy(mgr). + For(&v1alpha1.YurtAppOverrider{}). + WithDefaulter(webhook). + WithValidator(webhook). + Complete() +} + +// +kubebuilder:webhook:path=/validate-apps-openyurt-io-v1alpha1-yurtappoverrider,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps.openyurt.io,resources=yurtappoverriders,verbs=create;update;delete,versions=v1alpha1,name=validate.apps.v1alpha1.yurtappoverrider.openyurt.io +// +kubebuilder:webhook:path=/mutate-apps-openyurt-io-v1alpha1-yurtappoverrider,mutating=true,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps.openyurt.io,resources=yurtappoverriders,verbs=create;update,versions=v1alpha1,name=mutate.apps.v1alpha1.yurtappoverrider.openyurt.io + +// Cluster implements a validating and defaulting webhook for Cluster. +type YurtAppOverriderHandler struct { + Client client.Client +} + +var _ webhook.CustomDefaulter = &YurtAppOverriderHandler{} +var _ webhook.CustomValidator = &YurtAppOverriderHandler{} diff --git a/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_validation.go b/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_validation.go new file mode 100644 index 00000000000..b319bf68fcb --- /dev/null +++ b/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_validation.go @@ -0,0 +1,111 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *YurtAppOverriderHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { + overrider, ok := obj.(*v1alpha1.YurtAppOverrider) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider but got a %T", obj)) + } + + // validate + if err := webhook.validateOneToOneBinding(ctx, overrider); err != nil { + return err + } + return nil +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *YurtAppOverriderHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { + oldOverrider, ok := oldObj.(*v1alpha1.YurtAppOverrider) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider but got a %T", newObj)) + } + newOverrider, ok := newObj.(*v1alpha1.YurtAppOverrider) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider} but got a %T", oldObj)) + } + if oldOverrider.Namespace != newOverrider.Namespace || newOverrider.Name != oldOverrider.Name { + return fmt.Errorf("unable to change metadata after %s is created", oldOverrider.Name) + } + if newOverrider.Subject.Kind != oldOverrider.Subject.Kind || newOverrider.Subject.Name != oldOverrider.Subject.Name { + return fmt.Errorf("unable to modify subject after %s is created", oldOverrider.Name) + } + // validate + if err := webhook.validateOneToOneBinding(ctx, newOverrider); err != nil { + return err + } + return nil +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *YurtAppOverriderHandler) ValidateDelete(ctx context.Context, obj runtime.Object) error { + overrider, ok := obj.(*v1alpha1.YurtAppOverrider) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider but got a %T", obj)) + } + switch overrider.Subject.Kind { + case "YurtAppSet": + appSet := &v1alpha1.YurtAppSet{} + err := webhook.Client.Get(ctx, client.ObjectKey{Name: overrider.Subject.Name, Namespace: overrider.Namespace}, appSet) + if err == nil { + return fmt.Errorf("namespace: %s, unable to delete YurtAppOverrider when subject resource exists: %s", overrider.Namespace, appSet.Name) + } + case "YurtAppDaemon": + appDaemon := &v1alpha1.YurtAppDaemon{} + err := webhook.Client.Get(ctx, client.ObjectKey{Name: overrider.Subject.Name, Namespace: overrider.Namespace}, appDaemon) + if err == nil { + return fmt.Errorf("namespace: %s, unable to delete YurtAppOverrider when subject resource exists: %s", overrider.Namespace, appDaemon.Name) + } + } + return nil +} + +// YurtAppOverrider and YurtAppSet are one-to-one relationship +func (webhook *YurtAppOverriderHandler) validateOneToOneBinding(ctx context.Context, app *v1alpha1.YurtAppOverrider) error { + var allOverriderList v1alpha1.YurtAppOverriderList + if err := webhook.Client.List(ctx, &allOverriderList, client.InNamespace(app.Namespace)); err != nil { + klog.Infof("could not list YurtAppOverrider, %v", err) + return err + } + duplicatedOverriders := make([]v1alpha1.YurtAppOverrider, 0) + for _, overrider := range allOverriderList.Items { + if overrider.Name == app.Name { + continue + } + if overrider.Subject.Kind == app.Subject.Kind && overrider.Subject.Name == app.Subject.Name { + duplicatedOverriders = append(duplicatedOverriders, overrider) + } + } + if len(duplicatedOverriders) > 0 { + return fmt.Errorf("unable to bind multiple yurtappoverriders to one subject resource %s in namespace %s, %s already exists", app.Subject.Name, app.Namespace, app.Name) + } + return nil +} diff --git a/test/e2e/yurt/yurtappoverrider.go b/test/e2e/yurt/yurtappoverrider.go new file mode 100644 index 00000000000..acccfd7ce2b --- /dev/null +++ b/test/e2e/yurt/yurtappoverrider.go @@ -0,0 +1,293 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package yurt + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/test/e2e/util" + ycfg "github.com/openyurtio/openyurt/test/e2e/yurtconfig" +) + +var _ = Describe("YurtAppOverrider Test", func() { + ctx := context.Background() + k8sClient := ycfg.YurtE2eCfg.RuntimeClient + var namespaceName string + timeout := 60 * time.Second + nodePoolName := "nodepool-test" + yurtAppSetName := "yurtappset-test" + yurtAppOverriderName := "yurtappoverrider-test" + var testReplicasOld int32 = 3 + var testReplicasNew int32 = 5 + createNameSpace := func() { + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + } + Eventually( + func() error { + return k8sClient.Delete(ctx, &ns, client.PropagationPolicy(metav1.DeletePropagationForeground)) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + By("make sure namespace are removed") + + res := &corev1.Namespace{} + Eventually(func() error { + return k8sClient.Get(ctx, client.ObjectKey{Name: namespaceName}, res) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(&util.NotFoundMatcher{}) + Eventually(func() error { + return k8sClient.Create(ctx, &ns) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + } + createNodePool := func() { + Eventually(func() error { + return k8sClient.Delete(ctx, &v1alpha1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodePoolName, + Namespace: namespaceName, + }, + }) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + testNodePool := v1alpha1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodePoolName, + Namespace: namespaceName, + }, + } + Eventually(func() error { + return k8sClient.Create(ctx, &testNodePool) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + } + createYurtAppSet := func() { + Eventually(func() error { + return k8sClient.Delete(ctx, &v1alpha1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: yurtAppSetName, + Namespace: namespaceName, + }, + }) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + testYurtAppSet := v1alpha1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: yurtAppSetName, + Namespace: namespaceName, + }, + Spec: v1alpha1.YurtAppSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + WorkloadTemplate: v1alpha1.WorkloadTemplate{ + DeploymentTemplate: &v1alpha1.DeploymentTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "test"}, + }, + Spec: v1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "test"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Image: "nginx-old", + Name: "nginx", + }}, + }, + }, + }, + }, + }, + Topology: v1alpha1.Topology{ + Pools: []v1alpha1.Pool{ + { + Name: nodePoolName, + Replicas: &testReplicasOld, + }, + }, + }, + }, + } + Eventually(func() error { + return k8sClient.Create(ctx, &testYurtAppSet) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + } + createYurtAppOverrider := func() { + Eventually(func() error { + return k8sClient.Delete(ctx, &v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: yurtAppOverriderName, + Namespace: namespaceName, + }, + }) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + testYurtAppOverrider := v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: yurtAppOverriderName, + Namespace: namespaceName, + }, + Subject: v1alpha1.Subject{ + Name: yurtAppSetName, + TypeMeta: metav1.TypeMeta{ + Kind: "YurtAppSet", + APIVersion: "apps.openyurt.io/v1alpha1", + }, + }, + Entries: []v1alpha1.Entry{ + { + Pools: []string{"nodepool-test"}, + Items: []v1alpha1.Item{ + { + Image: &v1alpha1.ImageItem{ + ContainerName: "nginx", + ImageClaim: "nginx-item", + }, + }, + { + Replicas: &testReplicasNew, + }, + }, + Patches: []v1alpha1.Patch{ + { + Operation: v1alpha1.REPLACE, + Path: "/spec/template/spec/containers/0/image", + Value: apiextensionsv1.JSON{ + Raw: []byte(`"nginx-patch"`), + }, + }, + }, + }, + }, + } + Eventually(func() error { + return k8sClient.Create(ctx, &testYurtAppOverrider) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + } + deleteNodePool := func() { + Eventually(func() error { + return k8sClient.Delete(ctx, &v1alpha1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodePoolName, + Namespace: namespaceName, + }, + }) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + } + deleteYurtAppSet := func() { + Eventually(func() error { + return k8sClient.Delete(ctx, &v1alpha1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: yurtAppSetName, + Namespace: namespaceName, + }, + }) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + } + deleteYurtAppOverrider := func() { + Eventually(func() error { + return k8sClient.Delete(ctx, &v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: yurtAppOverriderName, + Namespace: namespaceName, + }, + }) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + } + + BeforeEach(func() { + By("Start to run YurtAppOverrider test, prepare resources") + namespaceName = "yurtappoverrider-e2e-test" + "-" + rand.String(4) + k8sClient = ycfg.YurtE2eCfg.RuntimeClient + createNameSpace() + + }) + AfterEach(func() { + By("Cleanup resources after test") + Expect(k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}}, client.PropagationPolicy(metav1.DeletePropagationBackground))).Should(Succeed()) + }) + + Describe("Test function of YurtAppOverrider", func() { + It("YurtAppOverrider should work after it is created", func() { + By("validate replicas and image of deployment") + Eventually(func() error { + deploymentList := &v1.DeploymentList{} + if err := k8sClient.List(ctx, deploymentList, client.InNamespace(namespaceName)); err != nil { + return err + } + for _, deployment := range deploymentList.Items { + if deployment.Labels["apps.openyurt.io/pool-name"] == nodePoolName { + if deployment.Spec.Template.Spec.Containers[0].Image != "nginx-patch" { + return fmt.Errorf("the image of nginx is not nginx-patch but %s", deployment.Spec.Template.Spec.Containers[0].Image) + } + if *deployment.Spec.Replicas != 5 { + return fmt.Errorf("the replicas of nginx is not 3 but %d", *deployment.Spec.Replicas) + } + } + } + return nil + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(Succeed()) + }) + It("YurtAppOverrider should refresh template after it is updated", func() { + By("Deployment will be returned to former when the YurtAppOverrider is deleted") + yurtAppOverrider := &v1alpha1.YurtAppOverrider{} + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: yurtAppOverriderName, Namespace: namespaceName}, yurtAppOverrider) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(Succeed()) + for _, entry := range yurtAppOverrider.Entries { + entry.Pools = []string{} + } + Expect(k8sClient.Update(ctx, yurtAppOverrider)).Should(Succeed()) + Eventually(func() error { + deploymentList := &v1.DeploymentList{} + if err := k8sClient.List(ctx, deploymentList, client.MatchingLabels{ + "apps.openyurt.io/pool-name": nodePoolName, + }); err != nil { + return err + } + for _, deployment := range deploymentList.Items { + if deployment.Spec.Template.Spec.Containers[0].Image != "nginx-old" { + return fmt.Errorf("the image of nginx is not nginx but %s", deployment.Spec.Template.Spec.Containers[0].Image) + } + if *deployment.Spec.Replicas != 3 { + return fmt.Errorf("the replicas of nginx is not 3 but %d", *deployment.Spec.Replicas) + } + } + return nil + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(Succeed()) + }) + BeforeEach(func() { + createNodePool() + createYurtAppSet() + createYurtAppOverrider() + }) + AfterEach(func() { + deleteNodePool() + deleteYurtAppSet() + deleteYurtAppOverrider() + }) + }) +}) From 893f117f12e7ea5414aabb311b9fc112fbaad862 Mon Sep 17 00:00:00 2001 From: lu chen <1813927768@qq.com> Date: Thu, 14 Sep 2023 12:03:50 +0800 Subject: [PATCH 11/13] Fix work dir nested `yurthub/yurthub` (#1693) Co-authored-by: luchen --- pkg/yurthub/certificate/manager/manager.go | 8 +++++--- pkg/yurthub/certificate/manager/manager_test.go | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/yurthub/certificate/manager/manager.go b/pkg/yurthub/certificate/manager/manager.go index 9d1f8cfff40..c75cb12e6c1 100644 --- a/pkg/yurthub/certificate/manager/manager.go +++ b/pkg/yurthub/certificate/manager/manager.go @@ -46,7 +46,7 @@ var ( apiServerClientCertNotReadyError = errors.New("APIServer client certificate") caCertIsNotReadyError = errors.New("ca.crt file") - DefaultRootDir = "/var/lib" + DefaultWorkDir = filepath.Join("/var/lib", projectinfo.GetHubName()) ) type yurtHubCertManager struct { @@ -58,10 +58,12 @@ type yurtHubCertManager struct { func NewYurtHubCertManager(options *options.YurtHubOptions, remoteServers []*url.URL) (hubCert.YurtCertificateManager, error) { var clientCertManager hubCert.YurtClientCertificateManager var err error + var workDir string - workDir := filepath.Join(options.RootDir, projectinfo.GetHubName()) if len(options.RootDir) == 0 { - workDir = filepath.Join(DefaultRootDir, projectinfo.GetHubName()) + workDir = DefaultWorkDir + } else { + workDir = options.RootDir } if options.BootstrapMode == "kubeletcertificate" { diff --git a/pkg/yurthub/certificate/manager/manager_test.go b/pkg/yurthub/certificate/manager/manager_test.go index 5a462c2fce4..fc5483fc772 100644 --- a/pkg/yurthub/certificate/manager/manager_test.go +++ b/pkg/yurthub/certificate/manager/manager_test.go @@ -45,7 +45,7 @@ func TestGetHubServerCertFile(t *testing.T) { }, "define root dir": { rootDir: "/tmp", - path: filepath.Join("/tmp", projectinfo.GetHubName(), "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), + path: filepath.Join("/tmp", "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), }, } From 6006a943522b32cc281288fa025216a69b2b46aa Mon Sep 17 00:00:00 2001 From: chenlong <50588775+Pluviophile225@users.noreply.github.com> Date: Thu, 14 Sep 2023 14:45:24 +0800 Subject: [PATCH 12/13] fix: add the logic of removing the finalizer (#1695) --- pkg/yurtiotdock/controllers/device_syncer.go | 11 +++++++++++ pkg/yurtiotdock/controllers/deviceprofile_syncer.go | 11 +++++++++++ pkg/yurtiotdock/controllers/deviceservice_syncer.go | 11 +++++++++++ 3 files changed, 33 insertions(+) diff --git a/pkg/yurtiotdock/controllers/device_syncer.go b/pkg/yurtiotdock/controllers/device_syncer.go index 64e30c8dc36..2e695aa6552 100644 --- a/pkg/yurtiotdock/controllers/device_syncer.go +++ b/pkg/yurtiotdock/controllers/device_syncer.go @@ -18,10 +18,12 @@ package controllers import ( "context" + "encoding/json" "strings" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" @@ -198,6 +200,15 @@ func (ds *DeviceSyncer) deleteDevices(redundantKubeDevices map[string]*iotv1alph "DeviceName", kd.Name) return err } + patchData, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": []string{}, + }, + }) + if err := ds.Client.Patch(context.TODO(), kd, client.RawPatch(types.MergePatchType, patchData)); err != nil { + klog.V(5).ErrorS(err, "fail to remove finalizer of Device on Kubernetes", "Device", kd.Name) + return err + } } return nil } diff --git a/pkg/yurtiotdock/controllers/deviceprofile_syncer.go b/pkg/yurtiotdock/controllers/deviceprofile_syncer.go index 5ae3390f8e5..4a6fe2f0757 100644 --- a/pkg/yurtiotdock/controllers/deviceprofile_syncer.go +++ b/pkg/yurtiotdock/controllers/deviceprofile_syncer.go @@ -18,10 +18,12 @@ package controllers import ( "context" + "encoding/json" "strings" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" @@ -213,6 +215,15 @@ func (dps *DeviceProfileSyncer) deleteDeviceProfiles(redundantKubeDeviceProfiles "DeviceProfile", kdp.Name) return err } + patchData, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": []string{}, + }, + }) + if err := dps.Client.Patch(context.TODO(), kdp, client.RawPatch(types.MergePatchType, patchData)); err != nil { + klog.V(5).ErrorS(err, "fail to remove finalizer of DeviceProfile on Kubernetes", "DeviceProfile", kdp.Name) + return err + } } return nil } diff --git a/pkg/yurtiotdock/controllers/deviceservice_syncer.go b/pkg/yurtiotdock/controllers/deviceservice_syncer.go index 30a6e59b5c4..e048cd7449d 100644 --- a/pkg/yurtiotdock/controllers/deviceservice_syncer.go +++ b/pkg/yurtiotdock/controllers/deviceservice_syncer.go @@ -17,10 +17,12 @@ package controllers import ( "context" + "encoding/json" "strings" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" @@ -197,6 +199,15 @@ func (ds *DeviceServiceSyncer) deleteDeviceServices(redundantKubeDeviceServices "DeviceService", kds.Name) return err } + patchData, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": []string{}, + }, + }) + if err := ds.Client.Patch(context.TODO(), kds, client.RawPatch(types.MergePatchType, patchData)); err != nil { + klog.V(5).ErrorS(err, "fail to remove finalizer of DeviceService on Kubernetes", "DeviceService", kds.Name) + return err + } } return nil } From f7820210e7b0e0863b6227d27a2ddaf02c809f56 Mon Sep 17 00:00:00 2001 From: vie-serendipity <60083692+vie-serendipity@users.noreply.github.com> Date: Sun, 17 Sep 2023 18:29:50 +0800 Subject: [PATCH 13/13] Add status info to YurtAppSet/YurtAppDaemon (#1702) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * adjust CRD and add wildcard * add configuration of deployment webhook * add register of webhook and controller * call deployment webhook directly * remove flatternYaml and add more info * modify deployment webhook failurePolicy * do some modifies * rename missing configrender to overrider * remove the outer of * * fix problems in the comments * fix error in e2e of yurtappoverrider * fix error in e2e of yurtappoverrider * fix error in validating webhook of yurtappoverrider * add more UT * gci writting ut * fix error in ut * add more UT * supplement status * supplement status of yurtappset * add status info to yurtappset/yurtappdaemon * feat: manifests * fix some problems * feat: refactor import * fix error of image scan * fix errors in github checks * go mod tidy * feat: manifests * fix errors in the comments * fix errors in the comments * fix errors in the comments * fix gci problems * add poolReplicas field to YurtAppSet * feat: manifests * feat: manifests * delete useless files * fix errors in the comments * add workloadsummaries to yurtappdaemon * feat: manifests * fix problems in the UT file * fix golangci-lint * add omitempty * feat: manifests --------- Co-authored-by: 飞倚 --- .../crds/apps.openyurt.io_yurtappdaemons.yaml | 27 +++++++++++ .../crds/apps.openyurt.io_yurtappsets.yaml | 27 +++++++++++ go.mod | 12 ++--- go.sum | 24 +++++----- pkg/apis/apps/v1alpha1/yurtappdaemon_types.go | 7 +++ pkg/apis/apps/v1alpha1/yurtappset_types.go | 14 ++++++ .../apps/v1alpha1/zz_generated.deepcopy.go | 25 ++++++++++ .../controller/yurtappdaemon/util.go | 5 ++ .../deployment_controller.go | 13 ++++++ .../workloadcontroller/workload.go | 3 ++ .../yurtappdaemon/yurtappdaemon_controller.go | 35 ++++++++++++-- .../yurtappdaemon_controller_test.go | 33 ++++++++----- .../yurtappoverrider_controller.go | 46 +++++++++++++++++-- .../yurtappoverrider_controller_test.go | 22 ++++++++- .../controller/yurtappset/adapter/adapter.go | 3 ++ .../yurtappset/adapter/deployment_adapter.go | 12 +++++ .../yurtappset/adapter/statefulset_adapter.go | 11 +++++ pkg/yurtmanager/controller/yurtappset/pool.go | 4 +- .../controller/yurtappset/pool_control.go | 5 ++ .../yurtappset/yurtappset_controller.go | 23 +++++++++- 20 files changed, 308 insertions(+), 43 deletions(-) diff --git a/charts/yurt-manager/crds/apps.openyurt.io_yurtappdaemons.yaml b/charts/yurt-manager/crds/apps.openyurt.io_yurtappdaemons.yaml index 4fa7d8e6dfa..23f898517e9 100644 --- a/charts/yurt-manager/crds/apps.openyurt.io_yurtappdaemons.yaml +++ b/charts/yurt-manager/crds/apps.openyurt.io_yurtappdaemons.yaml @@ -30,6 +30,10 @@ spec: jsonPath: .metadata.creationTimestamp name: AGE type: date + - description: The name of overrider bound to this yurtappdaemon + jsonPath: .status.overriderRef + name: OverriderRef + type: string name: v1alpha1 schema: openAPIV3Schema: @@ -225,9 +229,32 @@ spec: which is updated on mutation by the API Server. format: int64 type: integer + overriderRef: + type: string templateType: description: TemplateType indicates the type of PoolTemplate type: string + workloadSummary: + description: Records the topology detailed information of each workload. + items: + properties: + availableCondition: + type: string + readyReplicas: + format: int32 + type: integer + replicas: + format: int32 + type: integer + workloadName: + type: string + required: + - availableCondition + - readyReplicas + - replicas + - workloadName + type: object + type: array required: - currentRevision - templateType diff --git a/charts/yurt-manager/crds/apps.openyurt.io_yurtappsets.yaml b/charts/yurt-manager/crds/apps.openyurt.io_yurtappsets.yaml index 93ef1d936c2..559f8967049 100644 --- a/charts/yurt-manager/crds/apps.openyurt.io_yurtappsets.yaml +++ b/charts/yurt-manager/crds/apps.openyurt.io_yurtappsets.yaml @@ -32,6 +32,10 @@ spec: jsonPath: .metadata.creationTimestamp name: AGE type: date + - description: The name of overrider bound to this yurtappset + jsonPath: .status.overriderRef + name: OverriderRef + type: string name: v1alpha1 schema: openAPIV3Schema: @@ -329,6 +333,8 @@ spec: which is updated on mutation by the API Server. format: int64 type: integer + overriderRef: + type: string poolReplicas: additionalProperties: format: int32 @@ -347,6 +353,27 @@ spec: templateType: description: TemplateType indicates the type of PoolTemplate type: string + workloadSummary: + description: Records the topology detailed information of each workload. + items: + properties: + availableCondition: + type: string + readyReplicas: + format: int32 + type: integer + replicas: + format: int32 + type: integer + workloadName: + type: string + required: + - availableCondition + - readyReplicas + - replicas + - workloadName + type: object + type: array required: - currentRevision - replicas diff --git a/go.mod b/go.mod index bda596c041b..7adaca6b76b 100644 --- a/go.mod +++ b/go.mod @@ -28,8 +28,8 @@ require ( go.etcd.io/etcd/api/v3 v3.5.0 go.etcd.io/etcd/client/pkg/v3 v3.5.0 go.etcd.io/etcd/client/v3 v3.5.0 - golang.org/x/net v0.14.0 - golang.org/x/sys v0.11.0 + golang.org/x/net v0.15.0 + golang.org/x/sys v0.12.0 google.golang.org/grpc v1.57.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/square/go-jose.v2 v2.6.0 @@ -150,11 +150,11 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.12.0 // indirect + golang.org/x/crypto v0.13.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/term v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/term v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 1a6aba74e96..e92e2355ed8 100644 --- a/go.sum +++ b/go.sum @@ -773,8 +773,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -855,8 +855,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -873,8 +873,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -939,13 +939,13 @@ golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -954,8 +954,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/apis/apps/v1alpha1/yurtappdaemon_types.go b/pkg/apis/apps/v1alpha1/yurtappdaemon_types.go index 084ee6fa42e..a695d7c8d94 100644 --- a/pkg/apis/apps/v1alpha1/yurtappdaemon_types.go +++ b/pkg/apis/apps/v1alpha1/yurtappdaemon_types.go @@ -73,6 +73,12 @@ type YurtAppDaemonStatus struct { // +optional Conditions []YurtAppDaemonCondition `json:"conditions,omitempty"` + OverriderRef string `json:"overriderRef,omitempty"` + + // Records the topology detailed information of each workload. + // +optional + WorkloadSummaries []WorkloadSummary `json:"workloadSummary,omitempty"` + // TemplateType indicates the type of PoolTemplate TemplateType TemplateType `json:"templateType"` @@ -105,6 +111,7 @@ type YurtAppDaemonCondition struct { // +kubebuilder:resource:scope=Namespaced,path=yurtappdaemons,shortName=yad,categories=all // +kubebuilder:printcolumn:name="WorkloadTemplate",type="string",JSONPath=".status.templateType",description="The WorkloadTemplate Type." // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC." +// +kubebuilder:printcolumn:name="OverriderRef",type="string",JSONPath=".status.overriderRef",description="The name of overrider bound to this yurtappdaemon" // YurtAppDaemon is the Schema for the samples API type YurtAppDaemon struct { diff --git a/pkg/apis/apps/v1alpha1/yurtappset_types.go b/pkg/apis/apps/v1alpha1/yurtappset_types.go index 5af4f460801..4f333d203d7 100644 --- a/pkg/apis/apps/v1alpha1/yurtappset_types.go +++ b/pkg/apis/apps/v1alpha1/yurtappset_types.go @@ -163,6 +163,12 @@ type YurtAppSetStatus struct { // +optional Conditions []YurtAppSetCondition `json:"conditions,omitempty"` + // Records the topology detailed information of each workload. + // +optional + WorkloadSummaries []WorkloadSummary `json:"workloadSummary,omitempty"` + + OverriderRef string `json:"overriderRef,omitempty"` + // Records the topology detail information of the replicas of each pool. // +optional PoolReplicas map[string]int32 `json:"poolReplicas,omitempty"` @@ -178,6 +184,13 @@ type YurtAppSetStatus struct { TemplateType TemplateType `json:"templateType"` } +type WorkloadSummary struct { + AvailableCondition corev1.ConditionStatus `json:"availableCondition"` + Replicas int32 `json:"replicas"` + ReadyReplicas int32 `json:"readyReplicas"` + WorkloadName string `json:"workloadName"` +} + // YurtAppSetCondition describes current state of a YurtAppSet. type YurtAppSetCondition struct { // Type of in place set condition. @@ -203,6 +216,7 @@ type YurtAppSetCondition struct { // +kubebuilder:printcolumn:name="READY",type="integer",JSONPath=".status.readyReplicas",description="The number of pods ready." // +kubebuilder:printcolumn:name="WorkloadTemplate",type="string",JSONPath=".status.templateType",description="The WorkloadTemplate Type." // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC." +// +kubebuilder:printcolumn:name="OverriderRef",type="string",JSONPath=".status.overriderRef",description="The name of overrider bound to this yurtappset" // YurtAppSet is the Schema for the yurtAppSets API type YurtAppSet struct { diff --git a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go index bbd03f00dfb..061cbcc12c0 100644 --- a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -343,6 +343,21 @@ func (in *Topology) DeepCopy() *Topology { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadSummary) DeepCopyInto(out *WorkloadSummary) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadSummary. +func (in *WorkloadSummary) DeepCopy() *WorkloadSummary { + if in == nil { + return nil + } + out := new(WorkloadSummary) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkloadTemplate) DeepCopyInto(out *WorkloadTemplate) { *out = *in @@ -489,6 +504,11 @@ func (in *YurtAppDaemonStatus) DeepCopyInto(out *YurtAppDaemonStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.WorkloadSummaries != nil { + in, out := &in.WorkloadSummaries, &out.WorkloadSummaries + *out = make([]WorkloadSummary, len(*in)) + copy(*out, *in) + } if in.NodePools != nil { in, out := &in.NodePools, &out.NodePools *out = make([]string, len(*in)) @@ -688,6 +708,11 @@ func (in *YurtAppSetStatus) DeepCopyInto(out *YurtAppSetStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.WorkloadSummaries != nil { + in, out := &in.WorkloadSummaries, &out.WorkloadSummaries + *out = make([]WorkloadSummary, len(*in)) + copy(*out, *in) + } if in.PoolReplicas != nil { in, out := &in.PoolReplicas, &out.PoolReplicas *out = make(map[string]int32, len(*in)) diff --git a/pkg/yurtmanager/controller/yurtappdaemon/util.go b/pkg/yurtmanager/controller/yurtappdaemon/util.go index 503f3d7c73e..5b094c7b43b 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/util.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/util.go @@ -57,6 +57,11 @@ func GetYurtAppDaemonCondition(status unitv1alpha1.YurtAppDaemonStatus, condType return nil } +// RemoveYurtAppDaemonCondition removes the YurtAppDaemon condition with the provided type. +func RemoveYurtAppDaemonCondition(status *unitv1alpha1.YurtAppDaemonStatus, condType unitv1alpha1.YurtAppDaemonConditionType) { + status.Conditions = filterOutCondition(status.Conditions, condType) +} + // SetYurtAppDaemonCondition updates the YurtAppDaemon to include the provided condition. If the condition that // we are about to add already exists and has the same status, reason and message then we are not going to update. func SetYurtAppDaemonCondition(status *unitv1alpha1.YurtAppDaemonStatus, condition *unitv1alpha1.YurtAppDaemonCondition) { diff --git a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller.go index a70ad98f03b..d61b2194dab 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller.go @@ -21,6 +21,7 @@ import ( "errors" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -185,6 +186,13 @@ func (d *DeploymentControllor) GetAllWorkloads(yad *v1alpha1.YurtAppDaemon) ([]* for i, o := range objs { deploy := o.(*appsv1.Deployment) spec := deploy.Spec + var availableCondition corev1.ConditionStatus + for _, condition := range deploy.Status.Conditions { + if condition.Type == appsv1.DeploymentAvailable { + availableCondition = condition.Status + break + } + } w := &Workload{ Name: o.GetName(), Namespace: o.GetNamespace(), @@ -194,6 +202,11 @@ func (d *DeploymentControllor) GetAllWorkloads(yad *v1alpha1.YurtAppDaemon) ([]* NodeSelector: spec.Template.Spec.NodeSelector, Tolerations: spec.Template.Spec.Tolerations, }, + Status: WorkloadStatus{ + Replicas: deploy.Status.Replicas, + ReadyReplicas: deploy.Status.ReadyReplicas, + AvailableCondition: availableCondition, + }, } workloads = append(workloads, w) } diff --git a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload.go index 0270f9f0bb0..3eec35289a0 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload.go @@ -40,6 +40,9 @@ type WorkloadSpec struct { // WorkloadStatus stores the observed state of the Workload. type WorkloadStatus struct { + Replicas int32 + ReadyReplicas int32 + AvailableCondition corev1.ConditionStatus } func (w *Workload) GetRevision() string { diff --git a/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go index 05df1435db7..d36bb905cb5 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go @@ -202,13 +202,13 @@ func (r *ReconcileYurtAppDaemon) Reconcile(_ context.Context, request reconcile. return reconcile.Result{}, err } - return r.updateStatus(instance, newStatus, oldStatus, currentRevision, collisionCount, templateType) + return r.updateStatus(instance, newStatus, oldStatus, currentRevision, collisionCount, templateType, currentNPToWorkload) } func (r *ReconcileYurtAppDaemon) updateStatus(instance *unitv1alpha1.YurtAppDaemon, newStatus, oldStatus *unitv1alpha1.YurtAppDaemonStatus, - currentRevision *appsv1.ControllerRevision, collisionCount int32, templateType unitv1alpha1.TemplateType) (reconcile.Result, error) { + currentRevision *appsv1.ControllerRevision, collisionCount int32, templateType unitv1alpha1.TemplateType, currentNodepoolToWorkload map[string]*workloadcontroller.Workload) (reconcile.Result, error) { - newStatus = r.calculateStatus(instance, newStatus, currentRevision, collisionCount, templateType) + newStatus = r.calculateStatus(instance, newStatus, currentRevision, collisionCount, templateType, currentNodepoolToWorkload) _, err := r.updateYurtAppDaemon(instance, oldStatus, newStatus) return reconcile.Result{}, err @@ -254,17 +254,42 @@ func (r *ReconcileYurtAppDaemon) updateYurtAppDaemon(yad *unitv1alpha1.YurtAppDa } func (r *ReconcileYurtAppDaemon) calculateStatus(instance *unitv1alpha1.YurtAppDaemon, newStatus *unitv1alpha1.YurtAppDaemonStatus, - currentRevision *appsv1.ControllerRevision, collisionCount int32, templateType unitv1alpha1.TemplateType) *unitv1alpha1.YurtAppDaemonStatus { + currentRevision *appsv1.ControllerRevision, collisionCount int32, templateType unitv1alpha1.TemplateType, currentNodepoolToWorkload map[string]*workloadcontroller.Workload) *unitv1alpha1.YurtAppDaemonStatus { newStatus.CollisionCount = &collisionCount + var workloadFailure string + overriderList := unitv1alpha1.YurtAppOverriderList{} + if err := r.List(context.TODO(), &overriderList); err != nil { + workloadFailure = fmt.Sprintf("unable to list yurtappoverrider: %v", err) + } + for _, overrider := range overriderList.Items { + if overrider.Subject.Kind == "YurtAppDaemon" && overrider.Subject.Name == instance.Name { + newStatus.OverriderRef = overrider.Name + break + } + } + + newStatus.WorkloadSummaries = make([]unitv1alpha1.WorkloadSummary, 0) + for _, workload := range currentNodepoolToWorkload { + newStatus.WorkloadSummaries = append(newStatus.WorkloadSummaries, unitv1alpha1.WorkloadSummary{ + AvailableCondition: workload.Status.AvailableCondition, + Replicas: workload.Status.Replicas, + ReadyReplicas: workload.Status.ReadyReplicas, + WorkloadName: workload.Name, + }) + } if newStatus.CurrentRevision == "" { // init with current revision newStatus.CurrentRevision = currentRevision.Name } - newStatus.TemplateType = templateType + if workloadFailure == "" { + RemoveYurtAppDaemonCondition(newStatus, unitv1alpha1.WorkLoadFailure) + } else { + SetYurtAppDaemonCondition(newStatus, NewYurtAppDaemonCondition(unitv1alpha1.WorkLoadFailure, corev1.ConditionFalse, "Error", workloadFailure)) + } return newStatus } diff --git a/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go index 5611e911552..da0b909d944 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go @@ -23,6 +23,7 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/openyurtio/openyurt/cmd/yurt-manager/names" @@ -166,9 +167,11 @@ func TestUpdateStatus(t *testing.T) { t.Parallel() t.Logf("\tTestCase: %s", st.name) { - rc := &ReconcileYurtAppDaemon{} + rc := &ReconcileYurtAppDaemon{ + Client: fakeclient.NewClientBuilder().Build(), + } get, _ := rc.updateStatus( - st.instance, st.newStatus, st.oldStatus, st.currentRevision, st.collisionCount, st.templateType) + st.instance, st.newStatus, st.oldStatus, st.currentRevision, st.collisionCount, st.templateType, make(map[string]*workloadcontroller.Workload)) if !reflect.DeepEqual(get, st.expect) { t.Fatalf("\t%s\texpect %v, but get %v", failed, st.expect, get) } @@ -256,13 +259,14 @@ func TestCalculateStatus(t *testing.T) { var cr appsv1.ControllerRevision cr.Name = "a" tests := []struct { - name string - instance *unitv1alpha1.YurtAppDaemon - newStatus *unitv1alpha1.YurtAppDaemonStatus - currentRevision *appsv1.ControllerRevision - collisionCount int32 - templateType unitv1alpha1.TemplateType - expect unitv1alpha1.YurtAppDaemonStatus + name string + instance *unitv1alpha1.YurtAppDaemon + newStatus *unitv1alpha1.YurtAppDaemonStatus + currentNodepoolToWorkload map[string]*workloadcontroller.Workload + currentRevision *appsv1.ControllerRevision + collisionCount int32 + templateType unitv1alpha1.TemplateType + expect unitv1alpha1.YurtAppDaemonStatus }{ { "normal", @@ -281,6 +285,7 @@ func TestCalculateStatus(t *testing.T) { }, }, }, + map[string]*workloadcontroller.Workload{}, &cr, 1, "StatefulSet", @@ -307,8 +312,10 @@ func TestCalculateStatus(t *testing.T) { t.Parallel() t.Logf("\tTestCase: %s", st.name) { - rc := &ReconcileYurtAppDaemon{} - get := rc.calculateStatus(st.instance, st.newStatus, st.currentRevision, st.collisionCount, st.templateType) + rc := &ReconcileYurtAppDaemon{ + Client: fakeclient.NewClientBuilder().Build(), + } + get := rc.calculateStatus(st.instance, st.newStatus, st.currentRevision, st.collisionCount, st.templateType, st.currentNodepoolToWorkload) if !reflect.DeepEqual(get.CurrentRevision, st.expect.CurrentRevision) { t.Fatalf("\t%s\texpect %v, but get %v", failed, st.expect.CurrentRevision, get.CurrentRevision) } @@ -351,7 +358,9 @@ func TestManageWorkloads(t *testing.T) { t.Parallel() t.Logf("\tTestCase: %s", st.name) { - rc := &ReconcileYurtAppDaemon{} + rc := &ReconcileYurtAppDaemon{ + Client: fakeclient.NewClientBuilder().Build(), + } rc.manageWorkloads(st.instance, st.currentNodepoolToWorkload, st.allNameToNodePools, st.expectedRevision, st.templateType) get := st.expect if !reflect.DeepEqual(get, st.expect) { diff --git a/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go b/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go index 0fd375a3ce8..e8328f5a31e 100644 --- a/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go +++ b/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go @@ -20,10 +20,13 @@ import ( "context" "flag" "fmt" + "reflect" "time" v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/tools/record" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -33,6 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappoverrider/config" ) @@ -71,14 +75,18 @@ var _ reconcile.Reconciler = &ReconcileYurtAppOverrider{} // ReconcileYurtAppOverrider reconciles a YurtAppOverrider object type ReconcileYurtAppOverrider struct { client.Client - Configuration config.YurtAppOverriderControllerConfiguration + Configuration config.YurtAppOverriderControllerConfiguration + CacheOverriderMap map[string]*appsv1alpha1.YurtAppOverrider + recorder record.EventRecorder } // newReconciler returns a new reconcile.Reconciler func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { return &ReconcileYurtAppOverrider{ - Client: mgr.GetClient(), - Configuration: c.ComponentConfig.YurtAppOverriderController, + Client: mgr.GetClient(), + Configuration: c.ComponentConfig.YurtAppOverriderController, + CacheOverriderMap: make(map[string]*appsv1alpha1.YurtAppOverrider), + recorder: mgr.GetEventRecorderFor(names.YurtAppOverriderController), } } @@ -128,8 +136,38 @@ func (r *ReconcileYurtAppOverrider) Reconcile(_ context.Context, request reconci return reconcile.Result{}, nil } - deployments := v1.DeploymentList{} + switch instance.Subject.Kind { + case "YurtAppSet": + appSet := &appsv1alpha1.YurtAppSet{} + if err := r.Get(context.TODO(), client.ObjectKey{Namespace: instance.Namespace, Name: instance.Subject.Name}, appSet); err != nil { + return reconcile.Result{}, err + } + if appSet.Spec.WorkloadTemplate.StatefulSetTemplate != nil { + r.recorder.Event(instance.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("unable to override statefulset workload of %s", appSet.Name), "It is not supported to overrider statefulset now") + return reconcile.Result{}, nil + } + case "YurtAppDaemon": + appDaemon := &appsv1alpha1.YurtAppDaemon{} + if err := r.Get(context.TODO(), client.ObjectKey{Namespace: instance.Namespace, Name: instance.Subject.Name}, appDaemon); err != nil { + return reconcile.Result{}, err + } + if appDaemon.Spec.WorkloadTemplate.StatefulSetTemplate != nil { + r.recorder.Event(instance.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("unable to override statefulset workload of %s", appDaemon.Name), "It is not supported to overrider statefulset now") + return reconcile.Result{}, nil + } + default: + return reconcile.Result{}, fmt.Errorf("unsupported kind: %s", instance.Subject.Kind) + } + + if cacheOverrider, ok := r.CacheOverriderMap[instance.Namespace+"/"+instance.Name]; ok { + if reflect.DeepEqual(cacheOverrider.Entries, instance.Entries) { + return reconcile.Result{}, nil + } + } else { + r.CacheOverriderMap[instance.Namespace+"/"+instance.Name] = instance.DeepCopy() + } + deployments := v1.DeploymentList{} if err := r.List(context.TODO(), &deployments); err != nil { return reconcile.Result{}, err } diff --git a/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller_test.go b/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller_test.go index 646ce4f5d89..92d3b005a3c 100644 --- a/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller_test.go +++ b/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller_test.go @@ -35,6 +35,25 @@ var ( replica int32 = 3 ) +var yurtAppDaemon = &v1alpha1.YurtAppDaemon{ + ObjectMeta: metav1.ObjectMeta{ + Name: "yurtappdaemon", + Namespace: "default", + }, + Spec: v1alpha1.YurtAppDaemonSpec{ + NodePoolSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + }, +} + var daemonDeployment = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -104,7 +123,8 @@ func TestReconcile(t *testing.T) { return } reconciler := ReconcileYurtAppOverrider{ - Client: fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(daemonDeployment, overrider).Build(), + Client: fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(daemonDeployment, overrider, yurtAppDaemon).Build(), + CacheOverriderMap: make(map[string]*v1alpha1.YurtAppOverrider), } _, err := reconciler.Reconcile(context.Background(), reconcile.Request{ NamespacedName: types.NamespacedName{ diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/adapter.go b/pkg/yurtmanager/controller/yurtappset/adapter/adapter.go index f460a731314..9ff1110c16a 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/adapter.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/adapter.go @@ -21,6 +21,7 @@ change Adapter interface package adapter import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -36,6 +37,8 @@ type Adapter interface { GetStatusObservedGeneration(pool metav1.Object) int64 // GetDetails returns the replicas information of the pool status. GetDetails(pool metav1.Object) (replicasInfo ReplicasInfo, err error) + // GetAvailableStatus returns the available condition status of the workload + GetAvailableStatus(set metav1.Object) (conditionStatus corev1.ConditionStatus, err error) // GetPoolFailure returns failure information of the pool. GetPoolFailure() *string // ApplyPoolTemplate updates the pool to the latest revision. diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter.go b/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter.go index 72b7971b6cb..4d7ec32a7ff 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter.go @@ -20,6 +20,7 @@ import ( "fmt" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" @@ -68,6 +69,17 @@ func (a *DeploymentAdapter) GetDetails(obj metav1.Object) (ReplicasInfo, error) return replicasInfo, nil } +// GetAvailableStatus returns the available condition status of the workload +func (a *DeploymentAdapter) GetAvailableStatus(obj metav1.Object) (conditionStatus corev1.ConditionStatus, err error) { + dp := obj.(*appsv1.Deployment) + for _, condition := range dp.Status.Conditions { + if condition.Type == appsv1.DeploymentAvailable { + return condition.Status, nil + } + } + return corev1.ConditionUnknown, nil +} + // GetPoolFailure returns the failure information of the pool. // Deployment has no condition. func (a *DeploymentAdapter) GetPoolFailure() *string { diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter.go b/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter.go index be8a2710d4c..555d21be578 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter.go @@ -25,6 +25,7 @@ import ( "fmt" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" @@ -74,6 +75,16 @@ func (a *StatefulSetAdapter) GetDetails(obj metav1.Object) (ReplicasInfo, error) return replicasInfo, nil } +// GetAvailableStatus returns the available condition status of the workload +func (a *StatefulSetAdapter) GetAvailableStatus(obj metav1.Object) (conditionStatus corev1.ConditionStatus, err error) { + set := obj.(*appsv1.StatefulSet) + + if set.Status.AvailableReplicas != set.Status.Replicas { + return corev1.ConditionFalse, nil + } + return corev1.ConditionTrue, nil +} + // GetPoolFailure returns the failure information of the pool. // StatefulSet has no condition. func (a *StatefulSetAdapter) GetPoolFailure() *string { diff --git a/pkg/yurtmanager/controller/yurtappset/pool.go b/pkg/yurtmanager/controller/yurtappset/pool.go index a579a8f3a2f..5960d8afab4 100644 --- a/pkg/yurtmanager/controller/yurtappset/pool.go +++ b/pkg/yurtmanager/controller/yurtappset/pool.go @@ -18,6 +18,7 @@ limitations under the License. package yurtappset import ( + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" @@ -41,7 +42,8 @@ type PoolSpec struct { type PoolStatus struct { ObservedGeneration int64 adapter.ReplicasInfo - PatchInfo string + PatchInfo string + AvailableCondition v1.ConditionStatus } // ResourceRef stores the Pool resource it represents. diff --git a/pkg/yurtmanager/controller/yurtappset/pool_control.go b/pkg/yurtmanager/controller/yurtappset/pool_control.go index 45f07095fb7..6a6ba35d536 100644 --- a/pkg/yurtmanager/controller/yurtappset/pool_control.go +++ b/pkg/yurtmanager/controller/yurtappset/pool_control.go @@ -158,6 +158,10 @@ func (m *PoolControl) convertToPool(set metav1.Object) (*Pool, error) { if err != nil { return nil, err } + conditionStatus, err := m.adapter.GetAvailableStatus(set) + if err != nil { + return nil, err + } pool := &Pool{ Name: poolName, Namespace: set.GetNamespace(), @@ -167,6 +171,7 @@ func (m *PoolControl) convertToPool(set metav1.Object) (*Pool, error) { Status: PoolStatus{ ObservedGeneration: m.adapter.GetStatusObservedGeneration(set), ReplicasInfo: specReplicas, + AvailableCondition: conditionStatus, }, } if data, ok := set.GetAnnotations()[apps.AnnotationPatchKey]; ok { diff --git a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go index 750e57a8f05..8180aad3db7 100644 --- a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go +++ b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go @@ -299,19 +299,38 @@ func (r *ReconcileYurtAppSet) calculateStatus(instance *unitv1alpha1.YurtAppSet, newStatus.CurrentRevision = currentRevision.Name } + // update OverriderRef + var poolFailure *string + overriderList := unitv1alpha1.YurtAppOverriderList{} + if err := r.List(context.TODO(), &overriderList); err != nil { + message := fmt.Sprintf("fail to list yurtappoverrider: %v", err) + poolFailure = &message + } + for _, overrider := range overriderList.Items { + if overrider.Subject.Kind == "YurtAppSet" && overrider.Subject.Name == instance.Name { + newStatus.OverriderRef = overrider.Name + break + } + } // sync from status + newStatus.WorkloadSummaries = make([]unitv1alpha1.WorkloadSummary, 0) newStatus.PoolReplicas = make(map[string]int32) newStatus.ReadyReplicas = 0 newStatus.Replicas = 0 for _, pool := range nameToPool { newStatus.PoolReplicas[pool.Name] = pool.Status.Replicas + newStatus.WorkloadSummaries = append(newStatus.WorkloadSummaries, unitv1alpha1.WorkloadSummary{ + AvailableCondition: pool.Status.AvailableCondition, + Replicas: pool.Status.Replicas, + ReadyReplicas: pool.Status.ReadyReplicas, + WorkloadName: pool.Spec.PoolRef.GetName(), + }) newStatus.Replicas += pool.Status.Replicas newStatus.ReadyReplicas += pool.Status.ReadyReplicas } newStatus.TemplateType = getPoolTemplateType(instance) - var poolFailure *string for _, pool := range nameToPool { failureMessage := control.GetPoolFailure(pool) if failureMessage != nil { @@ -348,7 +367,7 @@ func (r *ReconcileYurtAppSet) updateYurtAppSet(yas *unitv1alpha1.YurtAppSet, old oldStatus.Replicas == newStatus.Replicas && oldStatus.ReadyReplicas == newStatus.ReadyReplicas && yas.Generation == newStatus.ObservedGeneration && - reflect.DeepEqual(oldStatus.PoolReplicas, newStatus.PoolReplicas) && + reflect.DeepEqual(oldStatus.WorkloadSummaries, newStatus.WorkloadSummaries) && reflect.DeepEqual(oldStatus.Conditions, newStatus.Conditions) { return yas, nil }