From 7697c53e46a7386166090e0ff4a89cbbcfb92e6f Mon Sep 17 00:00:00 2001 From: vie-serendipity <60083692+vie-serendipity@users.noreply.github.com> Date: Tue, 12 Sep 2023 16:10:38 +0800 Subject: [PATCH] Add yurtappoverrider (#1684) * adjust CRD and add wildcard * add configuration of deployment webhook * add register of webhook and controller * call deployment webhook directly * remove flatternYaml and add more info * modify deployment webhook failurePolicy * do some modifies * rename missing configrender to overrider * remove the outer of * * fix problems in the comments * fix error in e2e of yurtappoverrider * fix error in e2e of yurtappoverrider * fix error in validating webhook of yurtappoverrider * add more UT * gci writting ut * fix error in ut * add more UT * fix problems in the comments * fix problems in the comments * fix problems in the comments --- .../apps.openyurt.io_yurtappoverriders.yaml | 144 ++++++++ .../yurt-manager-auto-generated.yaml | 71 ++++ cmd/yurt-manager/app/options/options.go | 36 +- .../app/options/yurtappoverridercontroller.go | 60 ++++ cmd/yurt-manager/names/controller_names.go | 2 + go.mod | 2 +- .../apps/v1alpha1/yurtappoverrider_types.go | 107 ++++++ .../apps/v1alpha1/zz_generated.deepcopy.go | 171 +++++++++ .../controller/apis/config/types.go | 4 + pkg/yurtmanager/controller/controller.go | 2 + .../yurtappoverrider/config/types.go | 21 ++ .../yurtappoverrider_controller.go | 150 ++++++++ .../yurtappoverrider_controller_test.go | 118 +++++++ .../v1alpha1/deploymentrender_default.go | 156 +++++++++ .../v1alpha1/deploymentrender_handler.go | 56 +++ .../v1alpha1/deploymentrender_webhook_test.go | 326 ++++++++++++++++++ .../deploymentrender/v1alpha1/item_control.go | 43 +++ .../v1alpha1/item_control_test.go | 104 ++++++ .../v1alpha1/patch_control.go | 71 ++++ .../v1alpha1/patch_control_test.go | 94 +++++ pkg/yurtmanager/webhook/server.go | 4 + .../v1alpha1/yurtappoverrider_default.go | 36 ++ .../v1alpha1/yurtappoverrider_handler.go | 56 +++ .../v1alpha1/yurtappoverrider_validation.go | 111 ++++++ test/e2e/yurt/yurtappoverrider.go | 293 ++++++++++++++++ 25 files changed, 2222 insertions(+), 16 deletions(-) create mode 100644 charts/yurt-manager/crds/apps.openyurt.io_yurtappoverriders.yaml create mode 100644 cmd/yurt-manager/app/options/yurtappoverridercontroller.go create mode 100644 pkg/apis/apps/v1alpha1/yurtappoverrider_types.go create mode 100644 pkg/yurtmanager/controller/yurtappoverrider/config/types.go create mode 100644 pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go create mode 100644 pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller_test.go create mode 100644 pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_default.go create mode 100644 pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_handler.go create mode 100644 pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_webhook_test.go create mode 100644 pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control.go create mode 100644 pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control_test.go create mode 100644 pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control.go create mode 100644 pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control_test.go create mode 100644 pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_default.go create mode 100644 pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_handler.go create mode 100644 pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_validation.go create mode 100644 test/e2e/yurt/yurtappoverrider.go diff --git a/charts/yurt-manager/crds/apps.openyurt.io_yurtappoverriders.yaml b/charts/yurt-manager/crds/apps.openyurt.io_yurtappoverriders.yaml new file mode 100644 index 00000000000..d2b08cbd0a0 --- /dev/null +++ b/charts/yurt-manager/crds/apps.openyurt.io_yurtappoverriders.yaml @@ -0,0 +1,144 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + name: yurtappoverriders.apps.openyurt.io +spec: + group: apps.openyurt.io + names: + kind: YurtAppOverrider + listKind: YurtAppOverriderList + plural: yurtappoverriders + shortNames: + - yao + singular: yurtappoverrider + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The subject kind of this overrider. + jsonPath: .subject.kind + name: Subject + type: string + - description: The subject name of this overrider. + jsonPath: .subject.name + name: Name + type: string + - description: CreationTimestamp is a timestamp representing the server time when + this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. + jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + entries: + items: + description: Describe detailed multi-region configuration of the subject + Entry describe a set of nodepools and their shared or identical configurations + properties: + items: + items: + description: Item represents configuration to be injected. Only + one of its members may be specified. + properties: + image: + description: ImageItem specifies the corresponding container + and the claimed image + properties: + containerName: + description: ContainerName represents name of the container + in which the Image will be replaced + type: string + imageClaim: + description: ImageClaim represents the claimed image name + which is injected into the container above + type: string + required: + - containerName + - imageClaim + type: object + replicas: + format: int32 + type: integer + type: object + type: array + patches: + description: Convert Patch struct into json patch operation + items: + properties: + operation: + description: Operation represents the operation + enum: + - add + - remove + - replace + type: string + path: + description: Path represents the path in the json patch + type: string + value: + description: Indicates the value of json patch + x-kubernetes-preserve-unknown-fields: true + required: + - operation + - path + type: object + type: array + pools: + items: + type: string + type: array + required: + - pools + type: object + type: array + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + subject: + description: Describe the object Entries belongs + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: Name is the name of YurtAppSet or YurtAppDaemon + type: string + required: + - name + type: object + required: + - entries + - subject + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml index 46800849e86..1b452b66bcb 100644 --- a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml +++ b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml @@ -159,6 +159,14 @@ rules: - get - patch - update +- apiGroups: + - apps.openyurt.io + resources: + - yurtappoverriders + verbs: + - get + - list + - watch - apiGroups: - apps.openyurt.io resources: @@ -516,6 +524,26 @@ metadata: creationTimestamp: null name: yurt-manager-mutating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: yurt-manager-webhook-service + namespace: {{ .Release.Namespace }} + path: /mutate-apps-v1-deployment + failurePolicy: Ignore + name: mutate.apps.v1.deployment + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 @@ -618,6 +646,27 @@ webhooks: resources: - yurtappdaemons sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: yurt-manager-webhook-service + namespace: {{ .Release.Namespace }} + path: /mutate-apps-openyurt-io-v1alpha1-yurtappoverrider + failurePolicy: Fail + name: mutate.apps.v1alpha1.yurtappoverrider.openyurt.io + rules: + - apiGroups: + - apps.openyurt.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - yurtappoverriders + sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -789,6 +838,28 @@ webhooks: resources: - yurtappdaemons sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: yurt-manager-webhook-service + namespace: {{ .Release.Namespace }} + path: /validate-apps-openyurt-io-v1alpha1-yurtappoverrider + failurePolicy: Fail + name: validate.apps.v1alpha1.yurtappoverrider.openyurt.io + rules: + - apiGroups: + - apps.openyurt.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - yurtappoverriders + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/cmd/yurt-manager/app/options/options.go b/cmd/yurt-manager/app/options/options.go index e0940d30971..a0f8f161f11 100644 --- a/cmd/yurt-manager/app/options/options.go +++ b/cmd/yurt-manager/app/options/options.go @@ -25,26 +25,28 @@ import ( // YurtManagerOptions is the main context object for the yurt-manager. type YurtManagerOptions struct { - Generic *GenericOptions - NodePoolController *NodePoolControllerOptions - GatewayPickupController *GatewayPickupControllerOptions - YurtStaticSetController *YurtStaticSetControllerOptions - YurtAppSetController *YurtAppSetControllerOptions - YurtAppDaemonController *YurtAppDaemonControllerOptions - PlatformAdminController *PlatformAdminControllerOptions + Generic *GenericOptions + NodePoolController *NodePoolControllerOptions + GatewayPickupController *GatewayPickupControllerOptions + YurtStaticSetController *YurtStaticSetControllerOptions + YurtAppSetController *YurtAppSetControllerOptions + YurtAppDaemonController *YurtAppDaemonControllerOptions + PlatformAdminController *PlatformAdminControllerOptions + YurtAppOverriderController *YurtAppOverriderControllerOptions } // NewYurtManagerOptions creates a new YurtManagerOptions with a default config. func NewYurtManagerOptions() (*YurtManagerOptions, error) { s := YurtManagerOptions{ - Generic: NewGenericOptions(), - NodePoolController: NewNodePoolControllerOptions(), - GatewayPickupController: NewGatewayPickupControllerOptions(), - YurtStaticSetController: NewYurtStaticSetControllerOptions(), - YurtAppSetController: NewYurtAppSetControllerOptions(), - YurtAppDaemonController: NewYurtAppDaemonControllerOptions(), - PlatformAdminController: NewPlatformAdminControllerOptions(), + Generic: NewGenericOptions(), + NodePoolController: NewNodePoolControllerOptions(), + GatewayPickupController: NewGatewayPickupControllerOptions(), + YurtStaticSetController: NewYurtStaticSetControllerOptions(), + YurtAppSetController: NewYurtAppSetControllerOptions(), + YurtAppDaemonController: NewYurtAppDaemonControllerOptions(), + PlatformAdminController: NewPlatformAdminControllerOptions(), + YurtAppOverriderController: NewYurtAppOverriderControllerOptions(), } return &s, nil @@ -58,6 +60,7 @@ func (y *YurtManagerOptions) Flags(allControllers, disabledByDefaultControllers y.YurtStaticSetController.AddFlags(fss.FlagSet("yurtstaticset controller")) y.YurtAppDaemonController.AddFlags(fss.FlagSet("yurtappdaemon controller")) y.PlatformAdminController.AddFlags(fss.FlagSet("iot controller")) + y.YurtAppOverriderController.AddFlags(fss.FlagSet("yurtappoverrider controller")) // Please Add Other controller flags @kadisi return fss @@ -72,6 +75,7 @@ func (y *YurtManagerOptions) Validate(allControllers []string, controllerAliases errs = append(errs, y.YurtStaticSetController.Validate()...) errs = append(errs, y.YurtAppDaemonController.Validate()...) errs = append(errs, y.PlatformAdminController.Validate()...) + errs = append(errs, y.YurtAppOverriderController.Validate()...) return utilerrors.NewAggregate(errs) } @@ -92,10 +96,12 @@ func (y *YurtManagerOptions) ApplyTo(c *config.Config, controllerAliases map[str if err := y.PlatformAdminController.ApplyTo(&c.ComponentConfig.PlatformAdminController); err != nil { return err } + if err := y.YurtAppOverriderController.ApplyTo(&c.ComponentConfig.YurtAppOverriderController); err != nil { + return err + } if err := y.GatewayPickupController.ApplyTo(&c.ComponentConfig.GatewayPickupController); err != nil { return err } - return nil } diff --git a/cmd/yurt-manager/app/options/yurtappoverridercontroller.go b/cmd/yurt-manager/app/options/yurtappoverridercontroller.go new file mode 100644 index 00000000000..da9fd8f60ee --- /dev/null +++ b/cmd/yurt-manager/app/options/yurtappoverridercontroller.go @@ -0,0 +1,60 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "github.com/spf13/pflag" + + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappoverrider/config" +) + +type YurtAppOverriderControllerOptions struct { + *config.YurtAppOverriderControllerConfiguration +} + +func NewYurtAppOverriderControllerOptions() *YurtAppOverriderControllerOptions { + return &YurtAppOverriderControllerOptions{ + &config.YurtAppOverriderControllerConfiguration{}, + } +} + +// AddFlags adds flags related to nodepool for yurt-manager to the specified FlagSet. +func (n *YurtAppOverriderControllerOptions) AddFlags(fs *pflag.FlagSet) { + if n == nil { + return + } + + //fs.BoolVar(&n.CreateDefaultPool, "create-default-pool", n.CreateDefaultPool, "Create default cloud/edge pools if indicated.") +} + +// ApplyTo fills up nodepool config with options. +func (o *YurtAppOverriderControllerOptions) ApplyTo(cfg *config.YurtAppOverriderControllerConfiguration) error { + if o == nil { + return nil + } + + return nil +} + +// Validate checks validation of YurtAppOverriderControllerOptions. +func (o *YurtAppOverriderControllerOptions) Validate() []error { + if o == nil { + return nil + } + errs := []error{} + return errs +} diff --git a/cmd/yurt-manager/names/controller_names.go b/cmd/yurt-manager/names/controller_names.go index f00dc8d04d0..9749256c90f 100644 --- a/cmd/yurt-manager/names/controller_names.go +++ b/cmd/yurt-manager/names/controller_names.go @@ -25,6 +25,7 @@ const ( ServiceTopologyEndpointSliceController = "service-topology-endpointslice-controller" YurtAppSetController = "yurt-app-set-controller" YurtAppDaemonController = "yurt-app-daemon-controller" + YurtAppOverriderController = "yurt-app-overrider-controller" YurtStaticSetController = "yurt-static-set-controller" YurtCoordinatorCertController = "yurt-coordinator-cert-controller" DelegateLeaseController = "delegate-lease-controller" @@ -47,6 +48,7 @@ func YurtManagerControllerAliases() map[string]string { "yurtappset": YurtAppSetController, "yurtappdaemon": YurtAppDaemonController, "yurtstaticset": YurtStaticSetController, + "yurtappoverrider": YurtAppOverriderController, "yurtcoordinatorcert": YurtCoordinatorCertController, "delegatelease": DelegateLeaseController, "podbinding": PodBindingController, diff --git a/go.mod b/go.mod index 3c16f9c42eb..bda596c041b 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/edgexfoundry/go-mod-core-contracts/v2 v2.3.0 github.com/edgexfoundry/go-mod-core-contracts/v3 v3.0.0 + github.com/evanphx/json-patch v5.6.0+incompatible github.com/go-resty/resty/v2 v2.7.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/uuid v1.3.0 @@ -87,7 +88,6 @@ require ( github.com/cyphar/filepath-securejoin v0.2.3 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/emicklei/go-restful v2.16.0+incompatible // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/fatih/camelcase v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect diff --git a/pkg/apis/apps/v1alpha1/yurtappoverrider_types.go b/pkg/apis/apps/v1alpha1/yurtappoverrider_types.go new file mode 100644 index 00000000000..c7d658b7bb8 --- /dev/null +++ b/pkg/apis/apps/v1alpha1/yurtappoverrider_types.go @@ -0,0 +1,107 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// ImageItem specifies the corresponding container and the claimed image +type ImageItem struct { + // ContainerName represents name of the container + // in which the Image will be replaced + ContainerName string `json:"containerName"` + // ImageClaim represents the claimed image name + //which is injected into the container above + ImageClaim string `json:"imageClaim"` +} + +// Item represents configuration to be injected. +// Only one of its members may be specified. +type Item struct { + // +optional + Image *ImageItem `json:"image,omitempty"` + // +optional + Replicas *int32 `json:"replicas,omitempty"` +} + +type Operation string + +const ( + ADD Operation = "add" // json patch + REMOVE Operation = "remove" // json patch + REPLACE Operation = "replace" // json patch +) + +type Patch struct { + // Path represents the path in the json patch + Path string `json:"path"` + // Operation represents the operation + // +kubebuilder:validation:Enum=add;remove;replace + Operation Operation `json:"operation"` + // Indicates the value of json patch + // +optional + Value apiextensionsv1.JSON `json:"value,omitempty"` +} + +// Describe detailed multi-region configuration of the subject +// Entry describe a set of nodepools and their shared or identical configurations +type Entry struct { + Pools []string `json:"pools"` + // +optional + Items []Item `json:"items,omitempty"` + // Convert Patch struct into json patch operation + // +optional + Patches []Patch `json:"patches,omitempty"` +} + +// Describe the object Entries belongs +type Subject struct { + metav1.TypeMeta `json:",inline"` + // Name is the name of YurtAppSet or YurtAppDaemon + Name string `json:"name"` +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:resource:shortName=yao +// +kubebuilder:printcolumn:name="Subject",type="string",JSONPath=".subject.kind",description="The subject kind of this overrider." +// +kubebuilder:printcolumn:name="Name",type="string",JSONPath=".subject.name",description="The subject name of this overrider." +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC." + +type YurtAppOverrider struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Subject Subject `json:"subject"` + Entries []Entry `json:"entries"` +} + +// YurtAppOverriderList contains a list of YurtAppOverrider +// +kubebuilder:object:root=true +type YurtAppOverriderList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []YurtAppOverrider `json:"items"` +} + +func init() { + SchemeBuilder.Register(&YurtAppOverrider{}, &YurtAppOverriderList{}) +} diff --git a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go index f732bdcaa9b..bbd03f00dfb 100644 --- a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -45,6 +45,80 @@ func (in *DeploymentTemplateSpec) DeepCopy() *DeploymentTemplateSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Entry) DeepCopyInto(out *Entry) { + *out = *in + if in.Pools != nil { + in, out := &in.Pools, &out.Pools + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Item, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Patches != nil { + in, out := &in.Patches, &out.Patches + *out = make([]Patch, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Entry. +func (in *Entry) DeepCopy() *Entry { + if in == nil { + return nil + } + out := new(Entry) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImageItem) DeepCopyInto(out *ImageItem) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageItem. +func (in *ImageItem) DeepCopy() *ImageItem { + if in == nil { + return nil + } + out := new(ImageItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Item) DeepCopyInto(out *Item) { + *out = *in + if in.Image != nil { + in, out := &in.Image, &out.Image + *out = new(ImageItem) + **out = **in + } + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Item. +func (in *Item) DeepCopy() *Item { + if in == nil { + return nil + } + out := new(Item) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodePool) DeepCopyInto(out *NodePool) { *out = *in @@ -165,6 +239,22 @@ func (in *NodePoolStatus) DeepCopy() *NodePoolStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Patch) DeepCopyInto(out *Patch) { + *out = *in + in.Value.DeepCopyInto(&out.Value) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Patch. +func (in *Patch) DeepCopy() *Patch { + if in == nil { + return nil + } + out := new(Patch) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Pool) DeepCopyInto(out *Pool) { *out = *in @@ -215,6 +305,22 @@ func (in *StatefulSetTemplateSpec) DeepCopy() *StatefulSetTemplateSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Subject) DeepCopyInto(out *Subject) { + *out = *in + out.TypeMeta = in.TypeMeta +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Subject. +func (in *Subject) DeepCopy() *Subject { + if in == nil { + return nil + } + out := new(Subject) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Topology) DeepCopyInto(out *Topology) { *out = *in @@ -400,6 +506,71 @@ func (in *YurtAppDaemonStatus) DeepCopy() *YurtAppDaemonStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *YurtAppOverrider) DeepCopyInto(out *YurtAppOverrider) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Subject = in.Subject + if in.Entries != nil { + in, out := &in.Entries, &out.Entries + *out = make([]Entry, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new YurtAppOverrider. +func (in *YurtAppOverrider) DeepCopy() *YurtAppOverrider { + if in == nil { + return nil + } + out := new(YurtAppOverrider) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *YurtAppOverrider) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *YurtAppOverriderList) DeepCopyInto(out *YurtAppOverriderList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]YurtAppOverrider, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new YurtAppOverriderList. +func (in *YurtAppOverriderList) DeepCopy() *YurtAppOverriderList { + if in == nil { + return nil + } + out := new(YurtAppOverriderList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *YurtAppOverriderList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *YurtAppSet) DeepCopyInto(out *YurtAppSet) { *out = *in diff --git a/pkg/yurtmanager/controller/apis/config/types.go b/pkg/yurtmanager/controller/apis/config/types.go index 3be8f661277..aaeb974bf98 100644 --- a/pkg/yurtmanager/controller/apis/config/types.go +++ b/pkg/yurtmanager/controller/apis/config/types.go @@ -23,6 +23,7 @@ import ( platformadminconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin/config" gatewaypickupconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewaypickup/config" yurtappdaemonconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon/config" + yurtappoverriderconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappoverrider/config" yurtappsetconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/config" yurtstaticsetconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/config" ) @@ -48,6 +49,9 @@ type YurtManagerConfiguration struct { // PlatformAdminControllerConfiguration holds configuration for PlatformAdminController related features. PlatformAdminController platformadminconfig.PlatformAdminControllerConfiguration + + // YurtAppOverriderControllerConfiguration holds configuration for YurtAppOverriderController related features. + YurtAppOverriderController yurtappoverriderconfig.YurtAppOverriderControllerConfiguration } type GenericConfiguration struct { diff --git a/pkg/yurtmanager/controller/controller.go b/pkg/yurtmanager/controller/controller.go index d30b7d8e2dd..aaf66932148 100644 --- a/pkg/yurtmanager/controller/controller.go +++ b/pkg/yurtmanager/controller/controller.go @@ -38,6 +38,7 @@ import ( servicetopologyendpoints "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/endpoints" servicetopologyendpointslice "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/endpointslice" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappoverrider" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset" yurtcoordinatorcert "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/cert" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/delegatelease" @@ -83,6 +84,7 @@ func NewControllerInitializers() map[string]InitFunc { register(names.YurtStaticSetController, yurtstaticset.Add) register(names.YurtAppSetController, yurtappset.Add) register(names.YurtAppDaemonController, yurtappdaemon.Add) + register(names.YurtAppOverriderController, yurtappoverrider.Add) register(names.PlatformAdminController, platformadmin.Add) register(names.GatewayPickupController, gatewaypickup.Add) register(names.GatewayDNSController, dns.Add) diff --git a/pkg/yurtmanager/controller/yurtappoverrider/config/types.go b/pkg/yurtmanager/controller/yurtappoverrider/config/types.go new file mode 100644 index 00000000000..e11801f07f3 --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappoverrider/config/types.go @@ -0,0 +1,21 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +// YurtAppOverriderControllerConfiguration contains elements describing YurtAppOverriderController. +type YurtAppOverriderControllerConfiguration struct { +} diff --git a/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go b/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go new file mode 100644 index 00000000000..0fd375a3ce8 --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go @@ -0,0 +1,150 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yurtappoverrider + +import ( + "context" + "flag" + "fmt" + "time" + + v1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappoverrider/config" +) + +func init() { + flag.IntVar(&concurrentReconciles, "yurtappoverrider-workers", concurrentReconciles, "Max concurrent workers for YurtAppOverrider controller.") +} + +var ( + concurrentReconciles = 3 + controllerResource = appsv1alpha1.SchemeGroupVersion.WithResource("yurtappoverriders") +) + +const ( + ControllerName = "yurtappoverrider" +) + +func Format(format string, args ...interface{}) string { + s := fmt.Sprintf(format, args...) + return fmt.Sprintf("%s: %s", ControllerName, s) +} + +// Add creates a new YurtAppOverrider Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { + if _, err := mgr.GetRESTMapper().KindFor(controllerResource); err != nil { + klog.Infof("resource %s doesn't exist", controllerResource.String()) + return err + } + + return add(mgr, newReconciler(c, mgr)) +} + +var _ reconcile.Reconciler = &ReconcileYurtAppOverrider{} + +// ReconcileYurtAppOverrider reconciles a YurtAppOverrider object +type ReconcileYurtAppOverrider struct { + client.Client + Configuration config.YurtAppOverriderControllerConfiguration +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { + return &ReconcileYurtAppOverrider{ + Client: mgr.GetClient(), + Configuration: c.ComponentConfig.YurtAppOverriderController, + } +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New(ControllerName, mgr, controller.Options{ + Reconciler: r, MaxConcurrentReconciles: concurrentReconciles, + }) + if err != nil { + return err + } + + // Watch for changes to YurtAppOverrider + err = c.Watch(&source.Kind{Type: &appsv1alpha1.YurtAppOverrider{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + + return nil +} + +// +kubebuilder:rbac:groups=apps.openyurt.io,resources=yurtappoverriders,verbs=get;list;watch +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=list;watch;update +// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete + +// Reconcile reads that state of the cluster for a YurtAppOverrider object and makes changes based on the state read +// and what is in the YurtAppOverrider.Spec +func (r *ReconcileYurtAppOverrider) Reconcile(_ context.Context, request reconcile.Request) (reconcile.Result, error) { + + // Note !!!!!!!!!! + // We strongly recommend use Format() to encapsulation because Format() can print logs by module + // @kadisi + klog.Infof(Format("Reconcile YurtAppOverrider %s/%s", request.Namespace, request.Name)) + + // Fetch the YurtAppOverrider instance + instance := &appsv1alpha1.YurtAppOverrider{} + err := r.Get(context.TODO(), request.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + + if instance.DeletionTimestamp != nil { + return reconcile.Result{}, nil + } + + deployments := v1.DeploymentList{} + + if err := r.List(context.TODO(), &deployments); err != nil { + return reconcile.Result{}, err + } + + for _, deployment := range deployments.Items { + if deployment.OwnerReferences[0].Kind == instance.Subject.Kind && deployment.OwnerReferences[0].Name == instance.Subject.Name { + if deployment.Annotations == nil { + deployment.Annotations = make(map[string]string) + } + deployment.Annotations["LastOverrideTime"] = time.Now().String() + if err := r.Client.Update(context.TODO(), &deployment); err != nil { + return reconcile.Result{}, err + } + } + } + + return reconcile.Result{}, nil +} diff --git a/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller_test.go b/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller_test.go new file mode 100644 index 00000000000..646ce4f5d89 --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller_test.go @@ -0,0 +1,118 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package yurtappoverrider + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +var ( + replica int32 = 3 +) + +var daemonDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps.openyurt.io/v1alpha1", + Kind: "YurtAppDaemon", + Name: "yurtappdaemon", + }}, + Labels: map[string]string{ + "apps.openyurt.io/pool-name": "nodepool-test", + }, + }, + Status: appsv1.DeploymentStatus{}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replica, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, +} + +var overrider = &v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo", + Namespace: "default", + }, + Subject: v1alpha1.Subject{ + Name: "yurtappdaemon", + TypeMeta: metav1.TypeMeta{ + Kind: "YurtAppDaemon", + APIVersion: "apps.openyurt.io/v1alpha1", + }, + }, + Entries: []v1alpha1.Entry{ + { + Pools: []string{"*"}, + }, + }, +} + +func TestReconcile(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Logf("failed to add yurt custom resource") + return + } + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Logf("failed to add kubernetes clint-go custom resource") + return + } + reconciler := ReconcileYurtAppOverrider{ + Client: fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(daemonDeployment, overrider).Build(), + } + _, err := reconciler.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: "default", + Name: "demo", + }, + }) + if err != nil { + t.Logf("fail to call Reconcile: %v", err) + } +} diff --git a/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_default.go b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_default.go new file mode 100644 index 00000000000..e1433cf75ce --- /dev/null +++ b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_default.go @@ -0,0 +1,156 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "fmt" + "strings" + + v1 "k8s.io/api/apps/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/adapter" +) + +var ( + resources = []string{"YurtAppSet", "YurtAppDaemon"} +) + +func contain(kind string, resources []string) bool { + for _, v := range resources { + if kind == v { + return true + } + } + return false +} + +// Default satisfies the defaulting webhook interface. +func (webhook *DeploymentRenderHandler) Default(ctx context.Context, obj runtime.Object) error { + deployment, ok := obj.(*v1.Deployment) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a Deployment but got a %T", obj)) + } + if deployment.OwnerReferences == nil { + return nil + } + if !contain(deployment.OwnerReferences[0].Kind, resources) { + return nil + } + // Get YurtAppSet/YurtAppDaemon resource of this deployment + app := deployment.OwnerReferences[0] + var instance client.Object + switch app.Kind { + case "YurtAppSet": + instance = &v1alpha1.YurtAppSet{} + case "YurtAppDaemon": + instance = &v1alpha1.YurtAppDaemon{} + default: + return nil + } + if err := webhook.Client.Get(ctx, client.ObjectKey{ + Namespace: deployment.Namespace, + Name: app.Name, + }, instance); err != nil { + return err + } + // Get nodepool of deployment + nodepool := deployment.Labels["apps.openyurt.io/pool-name"] + + // resume deployment + switch app.Kind { + case "YurtAppSet": + var replicas int32 + yas := instance.(*v1alpha1.YurtAppSet) + revision := yas.Status.CurrentRevision + if yas.Spec.WorkloadTemplate.DeploymentTemplate != nil && yas.Spec.WorkloadTemplate.DeploymentTemplate.Spec.Replicas != nil { + replicas = *yas.Spec.WorkloadTemplate.DeploymentTemplate.Spec.Replicas + } + deploymentAdapter := adapter.DeploymentAdapter{ + Client: webhook.Client, + Scheme: webhook.Scheme, + } + for _, pool := range yas.Spec.Topology.Pools { + if pool.Name == nodepool { + replicas = *pool.Replicas + } + } + if err := deploymentAdapter.ApplyPoolTemplate(yas, nodepool, revision, replicas, deployment); err != nil { + return err + } + case "YurtAppDaemon": + yad := instance.(*v1alpha1.YurtAppDaemon) + yad.Spec.WorkloadTemplate.DeploymentTemplate.Spec.DeepCopyInto(&deployment.Spec) + } + + // Get YurtAppOverrider resource of app(1 to 1) + var allOverriderList v1alpha1.YurtAppOverriderList + //listOptions := client.MatchingFields{"spec.subject.kind": app.Kind, "spec.subject.name": app.Name, "spec.subject.APIVersion": app.APIVersion} + if err := webhook.Client.List(ctx, &allOverriderList, client.InNamespace(deployment.Namespace)); err != nil { + klog.Infof("error in listing YurtAppOverrider: %v", err) + return err + } + var overriders = make([]v1alpha1.YurtAppOverrider, 0) + for _, overrider := range allOverriderList.Items { + if overrider.Subject.Kind == app.Kind && overrider.Subject.Name == app.Name && overrider.Subject.APIVersion == app.APIVersion { + overriders = append(overriders, overrider) + } + } + + if len(overriders) == 0 { + return nil + } + render := overriders[0] + + for _, entry := range render.Entries { + for _, pool := range entry.Pools { + if pool[0] == '-' && pool[1:] == nodepool { + continue + } + if pool == nodepool || pool == "*" { + // Replace items + replaceItems(deployment, entry.Items) + // json patch + for i, patch := range entry.Patches { + if strings.Contains(string(patch.Value.Raw), "{{nodepool}}") { + newPatchString := strings.ReplaceAll(string(patch.Value.Raw), "{{nodepool}}", nodepool) + entry.Patches[i].Value = apiextensionsv1.JSON{Raw: []byte(newPatchString)} + } + } + // Implement injection + dataStruct := v1.Deployment{} + pc := PatchControl{ + patches: entry.Patches, + patchObject: deployment, + dataStruct: dataStruct, + } + if err := pc.jsonMergePatch(); err != nil { + klog.Infof("fail to update patches for deployment: %v", err) + return err + } + break + } + } + } + return nil +} diff --git a/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_handler.go b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_handler.go new file mode 100644 index 00000000000..ffb4fad7d0c --- /dev/null +++ b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_handler.go @@ -0,0 +1,56 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + v1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" +) + +// SetupWebhookWithManager sets up Cluster webhooks. mutate path, validatepath, error +func (webhook *DeploymentRenderHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, string, error) { + // init + webhook.Client = mgr.GetClient() + webhook.Scheme = mgr.GetScheme() + + gvk, err := apiutil.GVKForObject(&v1.Deployment{}, mgr.GetScheme()) + if err != nil { + return "", "", err + } + return util.GenerateMutatePath(gvk), + util.GenerateValidatePath(gvk), + ctrl.NewWebhookManagedBy(mgr). + For(&v1.Deployment{}). + WithDefaulter(webhook). + Complete() +} + +// +kubebuilder:webhook:path=/mutate-apps-v1-deployment,mutating=true,failurePolicy=ignore,groups=apps,resources=deployments,verbs=create;update,versions=v1,name=mutate.apps.v1.deployment,sideEffects=None,admissionReviewVersions=v1 + +// Cluster implements a validating and defaulting webhook for Cluster. +type DeploymentRenderHandler struct { + Client client.Client + Scheme *runtime.Scheme +} + +var _ webhook.CustomDefaulter = &DeploymentRenderHandler{} diff --git a/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_webhook_test.go b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_webhook_test.go new file mode 100644 index 00000000000..1bd80a09cd8 --- /dev/null +++ b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/deploymentrender_webhook_test.go @@ -0,0 +1,326 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package v1alpha1 + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +var ( + replica int32 = 3 +) + +var defaultAppSet = &v1alpha1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "yurtappset-patch", + Namespace: "default", + }, + Spec: v1alpha1.YurtAppSetSpec{ + Topology: v1alpha1.Topology{ + Pools: []v1alpha1.Pool{{ + Name: "nodepool-test", + Replicas: &replica}}, + }, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "test"}}, + WorkloadTemplate: v1alpha1.WorkloadTemplate{ + DeploymentTemplate: &v1alpha1.DeploymentTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "test"}, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "test"}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "test"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "nginx", Image: "nginx"}, + }, + Volumes: []corev1.Volume{ + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "configMapSource-nodepool-test", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, +} + +var defaultAppDaemon = &v1alpha1.YurtAppDaemon{ + ObjectMeta: metav1.ObjectMeta{ + Name: "yurtappdaemon", + Namespace: "default", + }, + Spec: v1alpha1.YurtAppDaemonSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "test"}}, + WorkloadTemplate: v1alpha1.WorkloadTemplate{ + DeploymentTemplate: &v1alpha1.DeploymentTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "test"}, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "test"}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "test"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "nginx", Image: "nginx"}, + }, + }, + }, + }, + }, + }, + }, +} + +var defaultDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test1", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps.openyurt.io/v1alpha1", + Kind: "YurtAppSet", + Name: "yurtappset-patch", + }}, + Labels: map[string]string{ + "apps.openyurt.io/pool-name": "nodepool-test", + }, + }, + Status: appsv1.DeploymentStatus{}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replica, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, +} + +var daemonDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test2", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps.openyurt.io/v1alpha1", + Kind: "YurtAppDaemon", + Name: "yurtappdaemon", + }}, + Labels: map[string]string{ + "apps.openyurt.io/pool-name": "nodepool-test", + }, + }, + Status: appsv1.DeploymentStatus{}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replica, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, +} + +var overrider1 = &v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo", + Namespace: "default", + }, + Subject: v1alpha1.Subject{ + Name: "yurtappset-patch", + TypeMeta: metav1.TypeMeta{ + Kind: "YurtAppSet", + APIVersion: "apps.openyurt.io/v1alpha1", + }, + }, + Entries: []v1alpha1.Entry{ + { + Pools: []string{"nodepool-test"}, + Items: []v1alpha1.Item{ + { + Image: &v1alpha1.ImageItem{ + ContainerName: "nginx", + ImageClaim: "nginx:1.18", + }, + }, + }, + Patches: []v1alpha1.Patch{ + { + Operation: v1alpha1.REPLACE, + Path: "/spec/replicas", + Value: apiextensionsv1.JSON{ + Raw: []byte("3"), + }, + }, + }, + }, + }, +} + +var overrider2 = &v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo", + Namespace: "default", + }, + Subject: v1alpha1.Subject{ + Name: "yurtappset-patch", + TypeMeta: metav1.TypeMeta{ + Kind: "YurtAppSet", + APIVersion: "apps.openyurt.io/v1alpha1", + }, + }, + Entries: []v1alpha1.Entry{ + { + Pools: []string{"*"}, + Patches: []v1alpha1.Patch{ + { + Operation: v1alpha1.ADD, + Path: "/spec/template/spec/volumes/-", + Value: apiextensionsv1.JSON{ + Raw: []byte(`{"name":"configmap-{{nodepool}}","configMap":{"name":"demo","items":[{"key": "game.properities","path": "game.properities"}]}}`), + }, + }, + }, + }, + }, +} + +var overrider3 = &v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo", + Namespace: "default", + }, + Subject: v1alpha1.Subject{ + Name: "demo", + TypeMeta: metav1.TypeMeta{ + Kind: "test", + APIVersion: "apps.openyurt.io/v1alpha1", + }, + }, + Entries: []v1alpha1.Entry{ + { + Pools: []string{"*"}, + }, + }, +} + +var overrider4 = &v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo", + Namespace: "default", + }, + Subject: v1alpha1.Subject{ + Name: "yurtappdaemon", + TypeMeta: metav1.TypeMeta{ + Kind: "YurtAppDaemon", + APIVersion: "apps.openyurt.io/v1alpha1", + }, + }, + Entries: []v1alpha1.Entry{ + { + Pools: []string{"*", "-nodepool-test"}, + }, + }, +} + +func TestDeploymentRenderHandler_Default(t *testing.T) { + tcases := []struct { + overrider *v1alpha1.YurtAppOverrider + }{ + {overrider1}, + {overrider2}, + {overrider3}, + {overrider4}, + } + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Logf("failed to add yurt custom resource") + return + } + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Logf("failed to add kubernetes clint-go custom resource") + return + } + for _, tcase := range tcases { + t.Run("", func(t *testing.T) { + webhook := &DeploymentRenderHandler{ + Client: fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(defaultAppSet, daemonDeployment, defaultDeployment, defaultAppDaemon, tcase.overrider).Build(), + Scheme: scheme, + } + if err := webhook.Default(context.TODO(), defaultDeployment); err != nil { + t.Fatal(err) + } + if err := webhook.Default(context.TODO(), daemonDeployment); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control.go b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control.go new file mode 100644 index 00000000000..83e4fcaa862 --- /dev/null +++ b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control.go @@ -0,0 +1,43 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + v1 "k8s.io/api/apps/v1" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +func replaceItems(deployment *v1.Deployment, items []v1alpha1.Item) { + for _, item := range items { + switch { + case item.Replicas != nil: + deployment.Spec.Replicas = item.Replicas + case item.Image != nil: + for i := range deployment.Spec.Template.Spec.Containers { + if deployment.Spec.Template.Spec.Containers[i].Name == item.Image.ContainerName { + deployment.Spec.Template.Spec.Containers[i].Image = item.Image.ImageClaim + } + } + for i := range deployment.Spec.Template.Spec.InitContainers { + if deployment.Spec.Template.Spec.InitContainers[i].Name == item.Image.ContainerName { + deployment.Spec.Template.Spec.InitContainers[i].Image = item.Image.ImageClaim + } + } + } + } +} diff --git a/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control_test.go b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control_test.go new file mode 100644 index 00000000000..7fb5982876f --- /dev/null +++ b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/item_control_test.go @@ -0,0 +1,104 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +var ( + itemReplicas int32 = 3 +) + +var testItemDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Status: appsv1.DeploymentStatus{}, + Spec: appsv1.DeploymentSpec{ + Replicas: &itemReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: "initContainer", + Image: "initOld", + }, + }, + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + Volumes: []corev1.Volume{ + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "configMapSource", + }, + }, + }, + }, + }, + }, + }, + }, +} + +func TestReplaceItems(t *testing.T) { + items := []v1alpha1.Item{ + { + Image: &v1alpha1.ImageItem{ + ContainerName: "nginx", + ImageClaim: "nginx", + }, + }, + { + Image: &v1alpha1.ImageItem{ + ContainerName: "initOld", + ImageClaim: "initNew", + }, + }, + { + Replicas: &itemReplicas, + }, + } + replaceItems(testItemDeployment, items) +} diff --git a/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control.go b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control.go new file mode 100644 index 00000000000..9be0cee032a --- /dev/null +++ b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control.go @@ -0,0 +1,71 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "encoding/json" + + jsonpatch "github.com/evanphx/json-patch" + appsv1 "k8s.io/api/apps/v1" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +type PatchControl struct { + patches []v1alpha1.Patch + patchObject interface{} + // data structure + dataStruct interface{} +} + +type overrider struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value,omitempty"` +} + +// implement json patch +func (pc *PatchControl) jsonMergePatch() error { + // convert into json patch format + var patchOperations []overrider + for _, patch := range pc.patches { + single := overrider{ + Op: string(patch.Operation), + Path: patch.Path, + Value: patch.Value, + } + patchOperations = append(patchOperations, single) + } + patchBytes, err := json.Marshal(patchOperations) + if err != nil { + return err + } + patchedData, err := json.Marshal(pc.patchObject.(*appsv1.Deployment)) + if err != nil { + return err + } + // conduct json patch + patchObj, err := jsonpatch.DecodePatch(patchBytes) + if err != nil { + return err + } + patchedData, err = patchObj.Apply(patchedData) + if err != nil { + return err + } + return json.Unmarshal(patchedData, &pc.patchObject) +} diff --git a/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control_test.go b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control_test.go new file mode 100644 index 00000000000..845369ed137 --- /dev/null +++ b/pkg/yurtmanager/webhook/deploymentrender/v1alpha1/patch_control_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +var initialReplicas int32 = 2 + +var testPatchDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps.openyurt.io/v1alpha1", + Kind: "YurtAppSet", + Name: "yurtappset-patch", + }}, + }, + Status: appsv1.DeploymentStatus{}, + Spec: appsv1.DeploymentSpec{ + Replicas: &initialReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, +} + +var patchControl = PatchControl{ + patches: []v1alpha1.Patch{ + { + Operation: v1alpha1.REPLACE, + Path: "/spec/template/spec/containers/0/image", + Value: apiextensionsv1.JSON{ + Raw: []byte(`"tomcat:1.18"`), + }, + }, + { + Operation: v1alpha1.ADD, + Path: "/spec/replicas", + Value: apiextensionsv1.JSON{ + Raw: []byte("5"), + }, + }, + }, + patchObject: testPatchDeployment, + dataStruct: appsv1.Deployment{}, +} + +func TestJsonMergePatch(t *testing.T) { + if err := patchControl.jsonMergePatch(); err != nil { + t.Fatalf("fail to call jsonMergePatch") + } + t.Logf("image:%v", testPatchDeployment.Spec.Template.Spec.Containers[0].Name) +} diff --git a/pkg/yurtmanager/webhook/server.go b/pkg/yurtmanager/webhook/server.go index 4ed3ff08e5f..7b43a79c414 100644 --- a/pkg/yurtmanager/webhook/server.go +++ b/pkg/yurtmanager/webhook/server.go @@ -30,6 +30,7 @@ import ( "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" "github.com/openyurtio/openyurt/cmd/yurt-manager/names" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller" + v1alpha1deploymentrender "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/deploymentrender/v1alpha1" v1beta1gateway "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/gateway/v1beta1" v1node "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/node/v1" v1beta1nodepool "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/nodepool/v1beta1" @@ -39,6 +40,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" webhookcontroller "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util/controller" v1alpha1yurtappdaemon "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1" + v1alpha1yurtappoverrider "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1" v1alpha1yurtappset "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtappset/v1alpha1" v1alpha1yurtstaticset "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtstaticset/v1alpha1" ) @@ -76,6 +78,8 @@ func init() { addControllerWebhook(names.YurtAppDaemonController, &v1alpha1yurtappdaemon.YurtAppDaemonHandler{}) addControllerWebhook(names.PlatformAdminController, &v1alpha1platformadmin.PlatformAdminHandler{}) addControllerWebhook(names.PlatformAdminController, &v1alpha2platformadmin.PlatformAdminHandler{}) + addControllerWebhook(names.YurtAppOverriderController, &v1alpha1yurtappoverrider.YurtAppOverriderHandler{}) + addControllerWebhook(names.YurtAppOverriderController, &v1alpha1deploymentrender.DeploymentRenderHandler{}) independentWebhooks[v1pod.WebhookName] = &v1pod.PodHandler{} independentWebhooks[v1node.WebhookName] = &v1node.NodeHandler{} diff --git a/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_default.go b/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_default.go new file mode 100644 index 00000000000..3424b412670 --- /dev/null +++ b/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_default.go @@ -0,0 +1,36 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +// Default satisfies the defaulting webhook interface. +func (webhook *YurtAppOverriderHandler) Default(ctx context.Context, obj runtime.Object) error { + _, ok := obj.(*v1alpha1.YurtAppOverrider) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider but got a %T", obj)) + } + return nil +} diff --git a/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_handler.go b/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_handler.go new file mode 100644 index 00000000000..53df018c966 --- /dev/null +++ b/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_handler.go @@ -0,0 +1,56 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" +) + +// SetupWebhookWithManager sets up Cluster webhooks. mutate path, validatepath, error +func (webhook *YurtAppOverriderHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, string, error) { + // init + webhook.Client = mgr.GetClient() + + gvk, err := apiutil.GVKForObject(&v1alpha1.YurtAppOverrider{}, mgr.GetScheme()) + if err != nil { + return "", "", err + } + return util.GenerateMutatePath(gvk), + util.GenerateValidatePath(gvk), + ctrl.NewWebhookManagedBy(mgr). + For(&v1alpha1.YurtAppOverrider{}). + WithDefaulter(webhook). + WithValidator(webhook). + Complete() +} + +// +kubebuilder:webhook:path=/validate-apps-openyurt-io-v1alpha1-yurtappoverrider,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps.openyurt.io,resources=yurtappoverriders,verbs=create;update;delete,versions=v1alpha1,name=validate.apps.v1alpha1.yurtappoverrider.openyurt.io +// +kubebuilder:webhook:path=/mutate-apps-openyurt-io-v1alpha1-yurtappoverrider,mutating=true,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps.openyurt.io,resources=yurtappoverriders,verbs=create;update,versions=v1alpha1,name=mutate.apps.v1alpha1.yurtappoverrider.openyurt.io + +// Cluster implements a validating and defaulting webhook for Cluster. +type YurtAppOverriderHandler struct { + Client client.Client +} + +var _ webhook.CustomDefaulter = &YurtAppOverriderHandler{} +var _ webhook.CustomValidator = &YurtAppOverriderHandler{} diff --git a/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_validation.go b/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_validation.go new file mode 100644 index 00000000000..b319bf68fcb --- /dev/null +++ b/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_validation.go @@ -0,0 +1,111 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *YurtAppOverriderHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { + overrider, ok := obj.(*v1alpha1.YurtAppOverrider) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider but got a %T", obj)) + } + + // validate + if err := webhook.validateOneToOneBinding(ctx, overrider); err != nil { + return err + } + return nil +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *YurtAppOverriderHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { + oldOverrider, ok := oldObj.(*v1alpha1.YurtAppOverrider) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider but got a %T", newObj)) + } + newOverrider, ok := newObj.(*v1alpha1.YurtAppOverrider) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider} but got a %T", oldObj)) + } + if oldOverrider.Namespace != newOverrider.Namespace || newOverrider.Name != oldOverrider.Name { + return fmt.Errorf("unable to change metadata after %s is created", oldOverrider.Name) + } + if newOverrider.Subject.Kind != oldOverrider.Subject.Kind || newOverrider.Subject.Name != oldOverrider.Subject.Name { + return fmt.Errorf("unable to modify subject after %s is created", oldOverrider.Name) + } + // validate + if err := webhook.validateOneToOneBinding(ctx, newOverrider); err != nil { + return err + } + return nil +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *YurtAppOverriderHandler) ValidateDelete(ctx context.Context, obj runtime.Object) error { + overrider, ok := obj.(*v1alpha1.YurtAppOverrider) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider but got a %T", obj)) + } + switch overrider.Subject.Kind { + case "YurtAppSet": + appSet := &v1alpha1.YurtAppSet{} + err := webhook.Client.Get(ctx, client.ObjectKey{Name: overrider.Subject.Name, Namespace: overrider.Namespace}, appSet) + if err == nil { + return fmt.Errorf("namespace: %s, unable to delete YurtAppOverrider when subject resource exists: %s", overrider.Namespace, appSet.Name) + } + case "YurtAppDaemon": + appDaemon := &v1alpha1.YurtAppDaemon{} + err := webhook.Client.Get(ctx, client.ObjectKey{Name: overrider.Subject.Name, Namespace: overrider.Namespace}, appDaemon) + if err == nil { + return fmt.Errorf("namespace: %s, unable to delete YurtAppOverrider when subject resource exists: %s", overrider.Namespace, appDaemon.Name) + } + } + return nil +} + +// YurtAppOverrider and YurtAppSet are one-to-one relationship +func (webhook *YurtAppOverriderHandler) validateOneToOneBinding(ctx context.Context, app *v1alpha1.YurtAppOverrider) error { + var allOverriderList v1alpha1.YurtAppOverriderList + if err := webhook.Client.List(ctx, &allOverriderList, client.InNamespace(app.Namespace)); err != nil { + klog.Infof("could not list YurtAppOverrider, %v", err) + return err + } + duplicatedOverriders := make([]v1alpha1.YurtAppOverrider, 0) + for _, overrider := range allOverriderList.Items { + if overrider.Name == app.Name { + continue + } + if overrider.Subject.Kind == app.Subject.Kind && overrider.Subject.Name == app.Subject.Name { + duplicatedOverriders = append(duplicatedOverriders, overrider) + } + } + if len(duplicatedOverriders) > 0 { + return fmt.Errorf("unable to bind multiple yurtappoverriders to one subject resource %s in namespace %s, %s already exists", app.Subject.Name, app.Namespace, app.Name) + } + return nil +} diff --git a/test/e2e/yurt/yurtappoverrider.go b/test/e2e/yurt/yurtappoverrider.go new file mode 100644 index 00000000000..acccfd7ce2b --- /dev/null +++ b/test/e2e/yurt/yurtappoverrider.go @@ -0,0 +1,293 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package yurt + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/test/e2e/util" + ycfg "github.com/openyurtio/openyurt/test/e2e/yurtconfig" +) + +var _ = Describe("YurtAppOverrider Test", func() { + ctx := context.Background() + k8sClient := ycfg.YurtE2eCfg.RuntimeClient + var namespaceName string + timeout := 60 * time.Second + nodePoolName := "nodepool-test" + yurtAppSetName := "yurtappset-test" + yurtAppOverriderName := "yurtappoverrider-test" + var testReplicasOld int32 = 3 + var testReplicasNew int32 = 5 + createNameSpace := func() { + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + } + Eventually( + func() error { + return k8sClient.Delete(ctx, &ns, client.PropagationPolicy(metav1.DeletePropagationForeground)) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + By("make sure namespace are removed") + + res := &corev1.Namespace{} + Eventually(func() error { + return k8sClient.Get(ctx, client.ObjectKey{Name: namespaceName}, res) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(&util.NotFoundMatcher{}) + Eventually(func() error { + return k8sClient.Create(ctx, &ns) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + } + createNodePool := func() { + Eventually(func() error { + return k8sClient.Delete(ctx, &v1alpha1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodePoolName, + Namespace: namespaceName, + }, + }) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + testNodePool := v1alpha1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodePoolName, + Namespace: namespaceName, + }, + } + Eventually(func() error { + return k8sClient.Create(ctx, &testNodePool) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + } + createYurtAppSet := func() { + Eventually(func() error { + return k8sClient.Delete(ctx, &v1alpha1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: yurtAppSetName, + Namespace: namespaceName, + }, + }) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + testYurtAppSet := v1alpha1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: yurtAppSetName, + Namespace: namespaceName, + }, + Spec: v1alpha1.YurtAppSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + WorkloadTemplate: v1alpha1.WorkloadTemplate{ + DeploymentTemplate: &v1alpha1.DeploymentTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "test"}, + }, + Spec: v1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "test"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Image: "nginx-old", + Name: "nginx", + }}, + }, + }, + }, + }, + }, + Topology: v1alpha1.Topology{ + Pools: []v1alpha1.Pool{ + { + Name: nodePoolName, + Replicas: &testReplicasOld, + }, + }, + }, + }, + } + Eventually(func() error { + return k8sClient.Create(ctx, &testYurtAppSet) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + } + createYurtAppOverrider := func() { + Eventually(func() error { + return k8sClient.Delete(ctx, &v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: yurtAppOverriderName, + Namespace: namespaceName, + }, + }) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + testYurtAppOverrider := v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: yurtAppOverriderName, + Namespace: namespaceName, + }, + Subject: v1alpha1.Subject{ + Name: yurtAppSetName, + TypeMeta: metav1.TypeMeta{ + Kind: "YurtAppSet", + APIVersion: "apps.openyurt.io/v1alpha1", + }, + }, + Entries: []v1alpha1.Entry{ + { + Pools: []string{"nodepool-test"}, + Items: []v1alpha1.Item{ + { + Image: &v1alpha1.ImageItem{ + ContainerName: "nginx", + ImageClaim: "nginx-item", + }, + }, + { + Replicas: &testReplicasNew, + }, + }, + Patches: []v1alpha1.Patch{ + { + Operation: v1alpha1.REPLACE, + Path: "/spec/template/spec/containers/0/image", + Value: apiextensionsv1.JSON{ + Raw: []byte(`"nginx-patch"`), + }, + }, + }, + }, + }, + } + Eventually(func() error { + return k8sClient.Create(ctx, &testYurtAppOverrider) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + } + deleteNodePool := func() { + Eventually(func() error { + return k8sClient.Delete(ctx, &v1alpha1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodePoolName, + Namespace: namespaceName, + }, + }) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + } + deleteYurtAppSet := func() { + Eventually(func() error { + return k8sClient.Delete(ctx, &v1alpha1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: yurtAppSetName, + Namespace: namespaceName, + }, + }) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + } + deleteYurtAppOverrider := func() { + Eventually(func() error { + return k8sClient.Delete(ctx, &v1alpha1.YurtAppOverrider{ + ObjectMeta: metav1.ObjectMeta{ + Name: yurtAppOverriderName, + Namespace: namespaceName, + }, + }) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + } + + BeforeEach(func() { + By("Start to run YurtAppOverrider test, prepare resources") + namespaceName = "yurtappoverrider-e2e-test" + "-" + rand.String(4) + k8sClient = ycfg.YurtE2eCfg.RuntimeClient + createNameSpace() + + }) + AfterEach(func() { + By("Cleanup resources after test") + Expect(k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}}, client.PropagationPolicy(metav1.DeletePropagationBackground))).Should(Succeed()) + }) + + Describe("Test function of YurtAppOverrider", func() { + It("YurtAppOverrider should work after it is created", func() { + By("validate replicas and image of deployment") + Eventually(func() error { + deploymentList := &v1.DeploymentList{} + if err := k8sClient.List(ctx, deploymentList, client.InNamespace(namespaceName)); err != nil { + return err + } + for _, deployment := range deploymentList.Items { + if deployment.Labels["apps.openyurt.io/pool-name"] == nodePoolName { + if deployment.Spec.Template.Spec.Containers[0].Image != "nginx-patch" { + return fmt.Errorf("the image of nginx is not nginx-patch but %s", deployment.Spec.Template.Spec.Containers[0].Image) + } + if *deployment.Spec.Replicas != 5 { + return fmt.Errorf("the replicas of nginx is not 3 but %d", *deployment.Spec.Replicas) + } + } + } + return nil + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(Succeed()) + }) + It("YurtAppOverrider should refresh template after it is updated", func() { + By("Deployment will be returned to former when the YurtAppOverrider is deleted") + yurtAppOverrider := &v1alpha1.YurtAppOverrider{} + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: yurtAppOverriderName, Namespace: namespaceName}, yurtAppOverrider) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(Succeed()) + for _, entry := range yurtAppOverrider.Entries { + entry.Pools = []string{} + } + Expect(k8sClient.Update(ctx, yurtAppOverrider)).Should(Succeed()) + Eventually(func() error { + deploymentList := &v1.DeploymentList{} + if err := k8sClient.List(ctx, deploymentList, client.MatchingLabels{ + "apps.openyurt.io/pool-name": nodePoolName, + }); err != nil { + return err + } + for _, deployment := range deploymentList.Items { + if deployment.Spec.Template.Spec.Containers[0].Image != "nginx-old" { + return fmt.Errorf("the image of nginx is not nginx but %s", deployment.Spec.Template.Spec.Containers[0].Image) + } + if *deployment.Spec.Replicas != 3 { + return fmt.Errorf("the replicas of nginx is not 3 but %d", *deployment.Spec.Replicas) + } + } + return nil + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(Succeed()) + }) + BeforeEach(func() { + createNodePool() + createYurtAppSet() + createYurtAppOverrider() + }) + AfterEach(func() { + deleteNodePool() + deleteYurtAppSet() + deleteYurtAppOverrider() + }) + }) +})