From c3bfe1a95d0170f18fd64cdfd2e28dc9e93f172c Mon Sep 17 00:00:00 2001 From: cl382465 Date: Fri, 19 Jan 2024 09:45:51 +0800 Subject: [PATCH] Feature: add yurtappset v1beta1 --- .../crds/apps.openyurt.io_yurtappdaemons.yaml | 3 + .../apps.openyurt.io_yurtappoverriders.yaml | 3 + .../crds/apps.openyurt.io_yurtappsets.yaml | 875 +++++++++++------- .../yurt-manager-auto-generated.yaml | 10 +- hack/make-rules/kustomize_to_chart.sh | 2 +- pkg/apis/apps/v1alpha1/nodepool_types.go | 1 - pkg/apis/apps/v1alpha1/yurtappdaemon_types.go | 1 + .../apps/v1alpha1/yurtappoverrider_types.go | 1 + .../apps/v1alpha1/yurtappset_conversion.go | 195 +++- pkg/apis/apps/v1alpha1/yurtappset_types.go | 3 +- pkg/apis/apps/v1beta1/nodepool_types.go | 1 - .../apps/v1beta1/yurtappset_conversion.go | 26 + pkg/apis/apps/v1beta1/yurtappset_types.go | 242 +++++ .../apps/v1beta1/zz_generated.deepcopy.go | 305 ++++++ .../apps/well_known_labels_annotations.go | 3 + .../controller/util/refmanager/ref_manager.go | 4 + pkg/yurtmanager/controller/util/tools.go | 8 +- pkg/yurtmanager/controller/util/tools_test.go | 12 +- .../yurtappset/adapter/adapter_util_test.go | 19 +- .../adapter/{adapter.go => adpater.go} | 5 +- .../{adapter_util.go => adpater_util.go} | 26 +- .../yurtappset/adapter/deployment_adapter.go | 20 +- .../adapter/deployment_adapter_test.go | 14 +- .../yurtappset/adapter/statefulset_adapter.go | 267 ------ .../adapter/statefulset_adapter_test.go | 245 ----- .../controller/yurtappset/config/types.go | 2 +- pkg/yurtmanager/controller/yurtappset/pool.go | 68 -- .../controller/yurtappset/pool_control.go | 188 ---- .../yurtappset/pool_controller_test.go | 316 ------- .../controller/yurtappset/revision.go | 214 +++-- .../controller/yurtappset/revision_test.go | 786 +++++++++------- .../controller/yurtappset/utils.go | 70 ++ .../controller/yurtappset/utils_test.go | 84 ++ .../workloadmanager/deployment_manager.go | 186 ++++ .../deployment_manager_test.go | 138 +++ .../yurtappset/workloadmanager/interface.go | 39 + .../workloadmanager/statefulset_manager.go | 27 + .../yurtappset/workloadmanager/tweaks.go | 130 +++ .../yurtappset/workloadmanager/tweaks_test.go | 449 +++++++++ .../yurtappset/workloadmanager/util.go | 151 +++ .../yurtappset/workloadmanager/util_test.go | 388 ++++++++ .../yurtappset/yurtappset_controller.go | 541 +++++++---- .../yurtappset/yurtappset_controller_test.go | 526 ++++++----- .../yurtappset/yurtappset_controller_utils.go | 121 --- .../yurtappset/yurtappset_update.go | 198 ---- pkg/yurtmanager/webhook/server.go | 4 +- .../webhook/yurtappset/v1alpha1/validate.go | 322 ------- .../v1alpha1/yurtappset_validation.go | 69 -- .../yurtappset_default.go | 24 +- .../yurtappset_handler.go | 14 +- .../v1beta1/yurtappset_validation.go | 84 ++ .../yurtappset_webhook_test.go | 47 +- 52 files changed, 4337 insertions(+), 3140 deletions(-) create mode 100644 pkg/apis/apps/v1beta1/yurtappset_conversion.go create mode 100644 pkg/apis/apps/v1beta1/yurtappset_types.go rename pkg/yurtmanager/controller/yurtappset/adapter/{adapter.go => adpater.go} (97%) rename pkg/yurtmanager/controller/yurtappset/adapter/{adapter_util.go => adpater_util.go} (98%) delete mode 100644 pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter.go delete mode 100644 pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter_test.go delete mode 100644 pkg/yurtmanager/controller/yurtappset/pool.go delete mode 100644 pkg/yurtmanager/controller/yurtappset/pool_control.go delete mode 100644 pkg/yurtmanager/controller/yurtappset/pool_controller_test.go create mode 100644 pkg/yurtmanager/controller/yurtappset/utils.go create mode 100644 pkg/yurtmanager/controller/yurtappset/utils_test.go create mode 100644 pkg/yurtmanager/controller/yurtappset/workloadmanager/deployment_manager.go create mode 100644 pkg/yurtmanager/controller/yurtappset/workloadmanager/deployment_manager_test.go create mode 100644 pkg/yurtmanager/controller/yurtappset/workloadmanager/interface.go create mode 100644 pkg/yurtmanager/controller/yurtappset/workloadmanager/statefulset_manager.go create mode 100644 pkg/yurtmanager/controller/yurtappset/workloadmanager/tweaks.go create mode 100644 pkg/yurtmanager/controller/yurtappset/workloadmanager/tweaks_test.go create mode 100644 pkg/yurtmanager/controller/yurtappset/workloadmanager/util.go create mode 100644 pkg/yurtmanager/controller/yurtappset/workloadmanager/util_test.go delete mode 100644 pkg/yurtmanager/controller/yurtappset/yurtappset_controller_utils.go delete mode 100644 pkg/yurtmanager/controller/yurtappset/yurtappset_update.go delete mode 100644 pkg/yurtmanager/webhook/yurtappset/v1alpha1/validate.go delete mode 100644 pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_validation.go rename pkg/yurtmanager/webhook/yurtappset/{v1alpha1 => v1beta1}/yurtappset_default.go (65%) rename pkg/yurtmanager/webhook/yurtappset/{v1alpha1 => v1beta1}/yurtappset_handler.go (64%) create mode 100644 pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_validation.go rename pkg/yurtmanager/webhook/yurtappset/{v1alpha1 => v1beta1}/yurtappset_webhook_test.go (62%) diff --git a/charts/yurt-manager/crds/apps.openyurt.io_yurtappdaemons.yaml b/charts/yurt-manager/crds/apps.openyurt.io_yurtappdaemons.yaml index 23f898517e9..33a154422b0 100644 --- a/charts/yurt-manager/crds/apps.openyurt.io_yurtappdaemons.yaml +++ b/charts/yurt-manager/crds/apps.openyurt.io_yurtappdaemons.yaml @@ -34,6 +34,9 @@ spec: jsonPath: .status.overriderRef name: OverriderRef type: string + deprecated: true + deprecationWarning: apps.openyurt.io/v1alpha1 YurtAppDaemon is deprecated; use + apps.openyurt.io/v1beta1 YurtAppSet; name: v1alpha1 schema: openAPIV3Schema: diff --git a/charts/yurt-manager/crds/apps.openyurt.io_yurtappoverriders.yaml b/charts/yurt-manager/crds/apps.openyurt.io_yurtappoverriders.yaml index e8af4427ba3..c21075bd383 100644 --- a/charts/yurt-manager/crds/apps.openyurt.io_yurtappoverriders.yaml +++ b/charts/yurt-manager/crds/apps.openyurt.io_yurtappoverriders.yaml @@ -32,6 +32,9 @@ spec: jsonPath: .metadata.creationTimestamp name: AGE type: date + deprecated: true + deprecationWarning: apps.openyurt.io/v1alpha1 YurtAppOverrider is deprecated; + use apps.openyurt.io/v1beta1 YurtAppSet WorkloadTweaks; name: v1alpha1 schema: openAPIV3Schema: diff --git a/charts/yurt-manager/crds/apps.openyurt.io_yurtappsets.yaml b/charts/yurt-manager/crds/apps.openyurt.io_yurtappsets.yaml index 559f8967049..de14abb2e37 100644 --- a/charts/yurt-manager/crds/apps.openyurt.io_yurtappsets.yaml +++ b/charts/yurt-manager/crds/apps.openyurt.io_yurtappsets.yaml @@ -8,382 +8,557 @@ metadata: spec: group: apps.openyurt.io names: + categories: + - all kind: YurtAppSet listKind: YurtAppSetList plural: yurtappsets shortNames: - - yas + - yas singular: yurtappset scope: Namespaced versions: - - additionalPrinterColumns: - - description: The number of pods ready. - jsonPath: .status.readyReplicas - name: READY - type: integer - - description: The WorkloadTemplate Type. - jsonPath: .status.templateType - name: WorkloadTemplate - type: string - - description: CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before - order across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. - jsonPath: .metadata.creationTimestamp - name: AGE - type: date - - description: The name of overrider bound to this yurtappset - jsonPath: .status.overriderRef - name: OverriderRef - type: string - name: v1alpha1 - schema: - openAPIV3Schema: - description: YurtAppSet is the Schema for the yurtAppSets API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: YurtAppSetSpec defines the desired state of YurtAppSet. - properties: - revisionHistoryLimit: - description: Indicates the number of histories to be conserved. If - unspecified, defaults to 10. - format: int32 - type: integer - selector: - description: Selector is a label query over pods that should match - the replica count. It must match the pod template's labels. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic - merge patch. - items: + - additionalPrinterColumns: + - description: The number of pods ready. + jsonPath: .status.readyReplicas + name: READY + type: integer + - description: The WorkloadTemplate Type. + jsonPath: .status.templateType + name: WorkloadTemplate + type: string + - description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + jsonPath: .metadata.creationTimestamp + name: AGE + type: date + - description: The name of overrider bound to this yurtappset + jsonPath: .status.overriderRef + name: OverriderRef + type: string + deprecated: true + deprecationWarning: apps.openyurt.io/v1alpha1 YurtAppSet is deprecated; use apps.openyurt.io/v1beta1 YurtAppSet; v1alpha1 YurtAppSet.Status.WorkloadSummary should not be used + name: v1alpha1 + schema: + openAPIV3Schema: + description: YurtAppSet is the Schema for the yurtAppSets API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: YurtAppSetSpec defines the desired state of YurtAppSet. + properties: + revisionHistoryLimit: + description: Indicates the number of histories to be conserved. If unspecified, defaults to 10. + format: int32 + type: integer + selector: + description: Selector is a label query over pods that should match the replica count. It must match the pod template's labels. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. type: string - type: array - required: - - key - - operator + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - topology: - description: Topology describes the pods distribution detail between - each of pools. - properties: - pools: - description: Contains the details of each pool. Each element in - this array represents one pool which will be provisioned and - managed by YurtAppSet. - items: - description: Pool defines the detail of a pool. - properties: - name: - description: Indicates pool name as a DNS_LABEL, which will - be used to generate pool workload name prefix in the format - '--'. Name should be unique - between all of the pools under one YurtAppSet. Name is - NodePool Name - type: string - nodeSelectorTerm: - description: Indicates the node selector to form the pool. - Depending on the node selector, pods provisioned could - be distributed across multiple groups of nodes. A pool's - nodeSelectorTerm is not allowed to be updated. - properties: - matchExpressions: - description: A list of node selector requirements by - node's labels. - items: - description: A node selector requirement is a selector - that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: The label key that the selector applies - to. - type: string - operator: - description: Represents a key's relationship to - a set of values. Valid operators are In, NotIn, - Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the - operator is In or NotIn, the values array must - be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator - is Gt or Lt, the values array must have a single - element, which will be interpreted as an integer. - This array is replaced during a strategic merge - patch. - items: + type: object + topology: + description: Topology describes the pods distribution detail between each of pools. + properties: + pools: + description: Contains the details of each pool. Each element in this array represents one pool which will be provisioned and managed by YurtAppSet. + items: + description: Pool defines the detail of a pool. + properties: + name: + description: Indicates pool name as a DNS_LABEL, which will be used to generate pool workload name prefix in the format '--'. Name should be unique between all of the pools under one YurtAppSet. Name is NodePool Name + type: string + nodeSelectorTerm: + description: Indicates the node selector to form the pool. Depending on the node selector, pods provisioned could be distributed across multiple groups of nodes. A pool's nodeSelectorTerm is not allowed to be updated. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements by - node's fields. - items: - description: A node selector requirement is a selector - that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: The label key that the selector applies - to. - type: string - operator: - description: Represents a key's relationship to - a set of values. Valid operators are In, NotIn, - Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the - operator is In or NotIn, the values array must - be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator - is Gt or Lt, the values array must have a single - element, which will be interpreted as an integer. - This array is replaced during a strategic merge - patch. - items: + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - patch: - description: Indicates the patch for the templateSpec Now - support strategic merge path :https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch/#notes-on-the-strategic-merge-patch - Patch takes precedence over Replicas fields If the Patch - also modifies the Replicas, use the Replicas value in - the Patch - type: object - x-kubernetes-preserve-unknown-fields: true - replicas: - description: Indicates the number of the pod to be created - under this pool. - format: int32 - type: integer - tolerations: - description: Indicates the tolerations the pods under this - pool have. A pool's tolerations is not allowed to be updated. - items: - description: The pod this Toleration is attached to tolerates - any taint that matches the triple - using the matching operator . - properties: - effect: - description: Effect indicates the taint effect to - match. Empty means match all taint effects. When - specified, allowed values are NoSchedule, PreferNoSchedule - and NoExecute. - type: string - key: - description: Key is the taint key that the toleration - applies to. Empty means match all taint keys. If - the key is empty, operator must be Exists; this - combination means to match all values and all keys. - type: string - operator: - description: Operator represents a key's relationship - to the value. Valid operators are Exists and Equal. - Defaults to Equal. Exists is equivalent to wildcard - for value, so that a pod can tolerate all taints - of a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period - of time the toleration (which must be of effect - NoExecute, otherwise this field is ignored) tolerates - the taint. By default, it is not set, which means - tolerate the taint forever (do not evict). Zero - and negative values will be treated as 0 (evict - immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration - matches to. If the operator is Exists, the value - should be empty, otherwise just a regular string. - type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array type: object - type: array + patch: + description: Indicates the patch for the templateSpec Now support strategic merge path :https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch/#notes-on-the-strategic-merge-patch Patch takes precedence over Replicas fields If the Patch also modifies the Replicas, use the Replicas value in the Patch + type: object + x-kubernetes-preserve-unknown-fields: true + replicas: + description: Indicates the number of the pod to be created under this pool. + format: int32 + type: integer + tolerations: + description: Indicates the tolerations the pods under this pool have. A pool's tolerations is not allowed to be updated. + items: + description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + required: + - name + type: object + type: array + type: object + workloadTemplate: + description: WorkloadTemplate describes the pool that will be created. + properties: + deploymentTemplate: + description: Deployment template + properties: + metadata: + x-kubernetes-preserve-unknown-fields: true + spec: + x-kubernetes-preserve-unknown-fields: true required: - - name + - spec type: object - type: array - type: object - workloadTemplate: - description: WorkloadTemplate describes the pool that will be created. - properties: - deploymentTemplate: - description: Deployment template + statefulSetTemplate: + description: StatefulSet template + properties: + metadata: + x-kubernetes-preserve-unknown-fields: true + spec: + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + type: object + required: + - selector + type: object + status: + description: YurtAppSetStatus defines the observed state of YurtAppSet. + properties: + collisionCount: + description: Count of hash collisions for the YurtAppSet. The YurtAppSet controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ControllerRevision. + format: int32 + type: integer + conditions: + description: Represents the latest available observations of a YurtAppSet's current state. + items: + description: YurtAppSetCondition describes current state of a YurtAppSet. properties: - metadata: - x-kubernetes-preserve-unknown-fields: true - spec: - x-kubernetes-preserve-unknown-fields: true - required: - - spec + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: A human readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of in place set condition. + type: string type: object - statefulSetTemplate: - description: StatefulSet template + type: array + currentRevision: + description: CurrentRevision, if not empty, indicates the current version of the YurtAppSet. + type: string + observedGeneration: + description: ObservedGeneration is the most recent generation observed for this YurtAppSet. It corresponds to the YurtAppSet's generation, which is updated on mutation by the API Server. + format: int64 + type: integer + overriderRef: + type: string + poolReplicas: + additionalProperties: + format: int32 + type: integer + description: Records the topology detail information of the replicas of each pool. + type: object + readyReplicas: + description: The number of ready replicas. + format: int32 + type: integer + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + templateType: + description: TemplateType indicates the type of PoolTemplate + type: string + workloadSummary: + description: Records the topology detailed information of each workload. + items: properties: - metadata: - x-kubernetes-preserve-unknown-fields: true - spec: - x-kubernetes-preserve-unknown-fields: true + availableCondition: + type: string + readyReplicas: + format: int32 + type: integer + replicas: + format: int32 + type: integer + workloadName: + type: string required: - - spec + - availableCondition + - readyReplicas + - replicas + - workloadName type: object - type: object - required: - - selector - type: object - status: - description: YurtAppSetStatus defines the observed state of YurtAppSet. - properties: - collisionCount: - description: Count of hash collisions for the YurtAppSet. The YurtAppSet - controller uses this field as a collision avoidance mechanism when - it needs to create the name for the newest ControllerRevision. - format: int32 - type: integer - conditions: - description: Represents the latest available observations of a YurtAppSet's - current state. - items: - description: YurtAppSetCondition describes current state of a YurtAppSet. + type: array + required: + - currentRevision + - replicas + - templateType + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The total number of workloads. + jsonPath: .status.totalWorkloads + name: TOTAL + type: integer + - description: The number of workloads ready. + jsonPath: .status.readyWorkloads + name: READY + type: integer + - description: The number of workloads updated. + jsonPath: .status.updatedWorkloads + name: UPDATED + type: integer + - description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: YurtAppSet is the Schema for the YurtAppSets API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: YurtAppSetSpec defines the desired state of YurtAppSet. + properties: + nodepoolSelector: + description: NodePoolSelector is a label query over nodepool in which workloads should be deployed in. It must match the nodepool's labels. properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of in place set condition. - type: string + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object type: object - type: array - currentRevision: - description: CurrentRevision, if not empty, indicates the current - version of the YurtAppSet. - type: string - observedGeneration: - description: ObservedGeneration is the most recent generation observed - for this YurtAppSet. It corresponds to the YurtAppSet's generation, - which is updated on mutation by the API Server. - format: int64 - type: integer - overriderRef: - type: string - poolReplicas: - additionalProperties: + pools: + description: Pools is a list of selected nodepools specified with nodepool id in which workloads should be deployed in. It is primarily used for compatibility with v1alpha1 version and NodePoolSelector should be preferred to choose nodepools + items: + type: string + type: array + revisionHistoryLimit: + description: Indicates the number of histories to be conserved. If unspecified, defaults to 10. format: int32 type: integer - description: Records the topology detail information of the replicas - of each pool. - type: object - readyReplicas: - description: The number of ready replicas. - format: int32 - type: integer - replicas: - description: Replicas is the most recently observed number of replicas. - format: int32 - type: integer - templateType: - description: TemplateType indicates the type of PoolTemplate - type: string - workloadSummary: - description: Records the topology detailed information of each workload. - items: + workload: + description: Workload defines the workload to be deployed in the nodepools properties: - availableCondition: - type: string - readyReplicas: - format: int32 - type: integer - replicas: - format: int32 - type: integer - workloadName: - type: string + workloadTemplate: + description: WorkloadTemplate defines the pool template under the YurtAppSet. + properties: + deploymentTemplate: + description: Deployment template + properties: + metadata: + x-kubernetes-preserve-unknown-fields: true + spec: + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + statefulSetTemplate: + description: StatefulSet template + properties: + metadata: + x-kubernetes-preserve-unknown-fields: true + spec: + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + type: object + workloadTweaks: + description: WorkloadTemplate defines the customization that will be applied to certain workloads in specified nodepools. + items: + description: WorkloadTweak Describe detailed multi-region configuration of the subject BasicTweaks and AdvancedTweaks describe a set of nodepools and their shared or identical configurations + properties: + nodepoolSelector: + description: NodePoolSelector is a label query over nodepool in which workloads should be adjusted. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + pools: + description: Pools is a list of selected nodepools specified with nodepool id in which workloads should be adjusted. Pools is not recommended and NodePoolSelector should be preferred + items: + type: string + type: array + tweaks: + description: Tweaks is the adjustment can be applied to a certain workload in specified nodepools such as image and replicas + properties: + containerImages: + description: ContainerImages is a list of container images to be injected to a certain workload + items: + description: ContainerImage specifies the corresponding container and the target image + properties: + name: + description: Name represents name of the container in which the Image will be replaced + type: string + targetImage: + description: TargetImage represents the image name which is injected into the container above + type: string + required: + - name + - targetImage + type: object + type: array + patches: + description: Patches is a list of advanced tweaks to be applied to a certain workload It can add/remove/replace the field values of specified paths in the template. + items: + properties: + operation: + description: Operation represents the operation + enum: + - add + - remove + - replace + type: string + path: + description: Path represents the path in the json patch + type: string + value: + description: Indicates the value of json patch + x-kubernetes-preserve-unknown-fields: true + required: + - operation + - path + type: object + type: array + replicas: + description: Replicas overrides the replicas of the workload + format: int32 + type: integer + type: object + required: + - tweaks + type: object + type: array required: - - availableCondition - - readyReplicas - - replicas - - workloadName + - workloadTemplate type: object - type: array - required: - - currentRevision - - replicas - - templateType - type: object - type: object - served: true - storage: true - subresources: - status: {} + required: + - workload + type: object + status: + description: YurtAppSetStatus defines the observed state of YurtAppSet. + properties: + collisionCount: + description: Count of hash collisions for the YurtAppSet. The YurtAppSet controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ControllerRevision. + format: int32 + type: integer + conditions: + description: Represents the latest available observations of a YurtAppSet's current state. + items: + description: YurtAppSetCondition describes current state of a YurtAppSet. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: A human readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of in place set condition. + type: string + type: object + type: array + currentRevision: + description: CurrentRevision, if not empty, indicates the current version of the YurtAppSet. + type: string + observedGeneration: + description: ObservedGeneration is the most recent generation observed for this YurtAppSet. It corresponds to the YurtAppSet's generation, which is updated on mutation by the API Server. + format: int64 + type: integer + readyWorkloads: + description: The number of ready workloads. + format: int32 + type: integer + totalWorkloads: + description: TotalWorkloads is the most recently observed number of workloads. + format: int32 + type: integer + updatedWorkloads: + description: The number of updated workloads. + format: int32 + type: integer + required: + - currentRevision + - readyWorkloads + - totalWorkloads + type: object + type: object + served: true + storage: true + subresources: + status: {} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1alpha1 + clientConfig: + service: + namespace: kube-system + name: yurt-manager-webhook-service + path: /convert status: acceptedNames: kind: "" diff --git a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml index 42f391a1b8a..046a94d9cfb 100644 --- a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml +++ b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml @@ -675,18 +675,19 @@ webhooks: sideEffects: None - admissionReviewVersions: - v1 + - v1beta1 clientConfig: service: name: yurt-manager-webhook-service namespace: {{ .Release.Namespace }} - path: /mutate-apps-openyurt-io-v1alpha1-yurtappset + path: /mutate-apps-openyurt-io-v1beta1-yurtappset failurePolicy: Fail name: myurtappset.kb.io rules: - apiGroups: - apps.openyurt.io apiVersions: - - v1alpha1 + - v1beta1 operations: - CREATE - UPDATE @@ -848,18 +849,19 @@ webhooks: sideEffects: None - admissionReviewVersions: - v1 + - v1beta1 clientConfig: service: name: yurt-manager-webhook-service namespace: {{ .Release.Namespace }} - path: /validate-apps-openyurt-io-v1alpha1-yurtappset + path: /validate-apps-openyurt-io-v1beta1-yurtappset failurePolicy: Fail name: vyurtappset.kb.io rules: - apiGroups: - apps.openyurt.io apiVersions: - - v1alpha1 + - v1beta1 operations: - CREATE - UPDATE diff --git a/hack/make-rules/kustomize_to_chart.sh b/hack/make-rules/kustomize_to_chart.sh index 7beefbc1f4e..f1ae798a447 100755 --- a/hack/make-rules/kustomize_to_chart.sh +++ b/hack/make-rules/kustomize_to_chart.sh @@ -39,7 +39,7 @@ YURT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" SUFFIX="auto_generated" -Conversion_Files=("apps.openyurt.io_nodepools.yaml" "raven.openyurt.io_gateways.yaml") +Conversion_Files=("apps.openyurt.io_nodepools.yaml" "raven.openyurt.io_gateways.yaml" "apps.openyurt.io_yurtappsets.yaml") while [ $# -gt 0 ];do case $1 in diff --git a/pkg/apis/apps/v1alpha1/nodepool_types.go b/pkg/apis/apps/v1alpha1/nodepool_types.go index b19e0c8c984..54c1261b5bb 100644 --- a/pkg/apis/apps/v1alpha1/nodepool_types.go +++ b/pkg/apis/apps/v1alpha1/nodepool_types.go @@ -69,7 +69,6 @@ type NodePoolStatus struct { } // +genclient:nonNamespaced -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,path=nodepools,shortName=np,categories=all // +kubebuilder:subresource:status diff --git a/pkg/apis/apps/v1alpha1/yurtappdaemon_types.go b/pkg/apis/apps/v1alpha1/yurtappdaemon_types.go index a695d7c8d94..7cf5ede1c66 100644 --- a/pkg/apis/apps/v1alpha1/yurtappdaemon_types.go +++ b/pkg/apis/apps/v1alpha1/yurtappdaemon_types.go @@ -112,6 +112,7 @@ type YurtAppDaemonCondition struct { // +kubebuilder:printcolumn:name="WorkloadTemplate",type="string",JSONPath=".status.templateType",description="The WorkloadTemplate Type." // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC." // +kubebuilder:printcolumn:name="OverriderRef",type="string",JSONPath=".status.overriderRef",description="The name of overrider bound to this yurtappdaemon" +// +kubebuilder:deprecatedversion:warning="apps.openyurt.io/v1alpha1 YurtAppDaemon is deprecated; use apps.openyurt.io/v1beta1 YurtAppSet;" // YurtAppDaemon is the Schema for the samples API type YurtAppDaemon struct { diff --git a/pkg/apis/apps/v1alpha1/yurtappoverrider_types.go b/pkg/apis/apps/v1alpha1/yurtappoverrider_types.go index c7d658b7bb8..10f9f74c593 100644 --- a/pkg/apis/apps/v1alpha1/yurtappoverrider_types.go +++ b/pkg/apis/apps/v1alpha1/yurtappoverrider_types.go @@ -86,6 +86,7 @@ type Subject struct { // +kubebuilder:printcolumn:name="Subject",type="string",JSONPath=".subject.kind",description="The subject kind of this overrider." // +kubebuilder:printcolumn:name="Name",type="string",JSONPath=".subject.name",description="The subject name of this overrider." // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC." +// +kubebuilder:deprecatedversion:warning="apps.openyurt.io/v1alpha1 YurtAppOverrider is deprecated; use apps.openyurt.io/v1beta1 YurtAppSet WorkloadTweaks;" type YurtAppOverrider struct { metav1.TypeMeta `json:",inline"` diff --git a/pkg/apis/apps/v1alpha1/yurtappset_conversion.go b/pkg/apis/apps/v1alpha1/yurtappset_conversion.go index 843ba2f7cc2..5f1d81ef719 100644 --- a/pkg/apis/apps/v1alpha1/yurtappset_conversion.go +++ b/pkg/apis/apps/v1alpha1/yurtappset_conversion.go @@ -16,30 +16,183 @@ limitations under the License. package v1alpha1 -/* -Implementing the hub method is pretty easy -- we just have to add an empty -method called Hub() to serve as a -[marker](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/conversion?tab=doc#Hub). -*/ +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/conversion" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" + "github.com/openyurtio/openyurt/pkg/projectinfo" +) + +func (src *YurtAppSet) ConvertTo(dstRaw conversion.Hub) error { + dst := dstRaw.(*v1beta1.YurtAppSet) + + dst.ObjectMeta = src.ObjectMeta + + // convert spec + if src.Spec.WorkloadTemplate.DeploymentTemplate != nil { + if dst.Spec.Workload.WorkloadTemplate.DeploymentTemplate == nil { + dst.Spec.Workload.WorkloadTemplate.DeploymentTemplate = &v1beta1.DeploymentTemplateSpec{} + } + src.Spec.WorkloadTemplate.DeploymentTemplate.ObjectMeta.DeepCopyInto(&dst.Spec.Workload.WorkloadTemplate.DeploymentTemplate.ObjectMeta) + src.Spec.WorkloadTemplate.DeploymentTemplate.Spec.DeepCopyInto(&dst.Spec.Workload.WorkloadTemplate.DeploymentTemplate.Spec) + } + + if src.Spec.WorkloadTemplate.StatefulSetTemplate != nil { + if dst.Spec.Workload.WorkloadTemplate.StatefulSetTemplate == nil { + dst.Spec.Workload.WorkloadTemplate.StatefulSetTemplate = &v1beta1.StatefulSetTemplateSpec{} + } + src.Spec.WorkloadTemplate.StatefulSetTemplate.ObjectMeta.DeepCopyInto(&dst.Spec.Workload.WorkloadTemplate.StatefulSetTemplate.ObjectMeta) + src.Spec.WorkloadTemplate.StatefulSetTemplate.Spec.DeepCopyInto(&dst.Spec.Workload.WorkloadTemplate.StatefulSetTemplate.Spec) + } + + pools := make([]string, 0) + tweaks := make([]v1beta1.WorkloadTweak, 0) + for _, pool := range src.Spec.Topology.Pools { + if len(pool.NodeSelectorTerm.MatchExpressions) == 1 && pool.NodeSelectorTerm.MatchExpressions[0].Key == projectinfo.GetNodePoolLabel() || + pool.NodeSelectorTerm.MatchExpressions[0].Operator == corev1.NodeSelectorOpIn { + pools = append(pools, pool.NodeSelectorTerm.MatchExpressions[0].Values...) + if pool.Replicas != nil { + tweaks = append(tweaks, v1beta1.WorkloadTweak{ + Pools: pool.NodeSelectorTerm.MatchExpressions[0].Values, + Tweaks: v1beta1.Tweaks{ + Replicas: pool.Replicas, + }, + }) + } + } + } + dst.Spec.Pools = pools + dst.Spec.Workload.WorkloadTweaks = tweaks + dst.Spec.RevisionHistoryLimit = src.Spec.RevisionHistoryLimit + + // convert status + dst.Status.ObservedGeneration = src.Status.ObservedGeneration + dst.Status.CollisionCount = src.Status.CollisionCount + dst.Status.TotalWorkloads = int32(len(src.Status.PoolReplicas)) + dst.Status.CurrentRevision = src.Status.CurrentRevision + dst.Status.Conditions = make([]v1beta1.YurtAppSetCondition, 0) + for _, condition := range src.Status.Conditions { + dst.Status.Conditions = append(dst.Status.Conditions, + v1beta1.YurtAppSetCondition{ + Type: v1beta1.YurtAppSetConditionType(condition.Type), + Status: condition.Status, + LastTransitionTime: condition.LastTransitionTime, + Reason: condition.Reason, + Message: condition.Message, + }) + } + readyWorkloads := 0 + for _, workload := range src.Status.WorkloadSummaries { + if workload.Replicas != workload.ReadyReplicas { + readyWorkloads++ + } + } + dst.Status.ReadyWorkloads = int32(readyWorkloads) + + klog.V(4).Infof("convert from v1alpha1 to v1beta1 for yurtappset %s", src.Name) + return nil +} + +func (dst *YurtAppSet) ConvertFrom(srcRaw conversion.Hub) error { + src := srcRaw.(*v1beta1.YurtAppSet) + var defaultReplicas int32 + + dst.ObjectMeta = src.ObjectMeta + + // convert spec + dst.Spec.Selector = &metav1.LabelSelector{} + if src.Spec.Workload.WorkloadTemplate.DeploymentTemplate != nil { + if dst.Spec.WorkloadTemplate.DeploymentTemplate == nil { + dst.Spec.WorkloadTemplate.DeploymentTemplate = &DeploymentTemplateSpec{} + } + src.Spec.Workload.WorkloadTemplate.DeploymentTemplate.ObjectMeta.DeepCopyInto(&dst.Spec.WorkloadTemplate.DeploymentTemplate.ObjectMeta) + src.Spec.Workload.WorkloadTemplate.DeploymentTemplate.Spec.DeepCopyInto(&dst.Spec.WorkloadTemplate.DeploymentTemplate.Spec) + if src.Spec.Workload.WorkloadTemplate.DeploymentTemplate.Spec.Replicas != nil { + defaultReplicas = *src.Spec.Workload.WorkloadTemplate.DeploymentTemplate.Spec.Replicas + } else { + defaultReplicas = 0 + } + + if src.Spec.Workload.WorkloadTemplate.DeploymentTemplate.Spec.Selector != nil { + dst.Spec.WorkloadTemplate.DeploymentTemplate.Spec.Selector.DeepCopyInto(dst.Spec.Selector) + } + dst.Status.TemplateType = DeploymentTemplateType + } + + if src.Spec.Workload.WorkloadTemplate.StatefulSetTemplate != nil { + if dst.Spec.WorkloadTemplate.StatefulSetTemplate == nil { + dst.Spec.WorkloadTemplate.StatefulSetTemplate = &StatefulSetTemplateSpec{} + } + src.Spec.Workload.WorkloadTemplate.StatefulSetTemplate.ObjectMeta.DeepCopyInto(&dst.Spec.WorkloadTemplate.StatefulSetTemplate.ObjectMeta) + src.Spec.Workload.WorkloadTemplate.StatefulSetTemplate.Spec.DeepCopyInto(&dst.Spec.WorkloadTemplate.StatefulSetTemplate.Spec) + if src.Spec.Workload.WorkloadTemplate.StatefulSetTemplate.Spec.Replicas != nil { + defaultReplicas = *src.Spec.Workload.WorkloadTemplate.StatefulSetTemplate.Spec.Replicas + } else { + defaultReplicas = 0 + } + + if src.Spec.Workload.WorkloadTemplate.StatefulSetTemplate.Spec.Selector != nil { + dst.Spec.WorkloadTemplate.StatefulSetTemplate.Spec.Selector.DeepCopyInto(dst.Spec.Selector) + } + dst.Status.TemplateType = StatefulSetTemplateType + } + + poolTweaks := make(map[string]*Pool, 0) + for _, poolName := range src.Spec.Pools { + poolTweaks[poolName] = &Pool{ + Name: poolName, + NodeSelectorTerm: corev1.NodeSelectorTerm{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: projectinfo.GetNodePoolLabel(), + Operator: corev1.NodeSelectorOpIn, + Values: []string{poolName}, + }, + }, + }, + Replicas: &defaultReplicas, + } + } -// NOTE !!!!!! @kadisi -// If this version is storageversion, you only need to uncommand this method + for _, tweak := range src.Spec.Workload.WorkloadTweaks { + for _, poolName := range tweak.Pools { + if _, ok := poolTweaks[poolName]; ok && tweak.Tweaks.Replicas != nil { + poolTweaks[poolName].Replicas = tweak.Tweaks.Replicas + } + } + } -// Hub marks this type as a conversion hub. -//func (*YurtAppSet) Hub() {} + for _, tweak := range poolTweaks { + dst.Spec.Topology.Pools = append(dst.Spec.Topology.Pools, *tweak) + } -// NOTE !!!!!!! @kadisi -// If this version is not storageversion, you need to implement the ConvertTo and ConvertFrom methods + // convert status + dst.Status.ObservedGeneration = src.Status.ObservedGeneration + dst.Status.CollisionCount = src.Status.CollisionCount + dst.Status.CurrentRevision = src.Status.CurrentRevision + dst.Status.Conditions = make([]YurtAppSetCondition, 0) + // this is just an estimate, because the real value can not be obtained from v1beta1 status + dst.Status.ReadyReplicas = src.Status.ReadyWorkloads * defaultReplicas + dst.Status.Replicas = src.Status.TotalWorkloads * defaultReplicas -// need import "sigs.k8s.io/controller-runtime/pkg/conversion" -//func (src *YurtAppSet) ConvertTo(dstRaw conversion.Hub) error { -// return nil -//} + dst.Status.PoolReplicas = make(map[string]int32) + for _, pool := range src.Spec.Pools { + dst.Status.PoolReplicas[pool] = defaultReplicas + } -// NOTE !!!!!!! @kadisi -// If this version is not storageversion, you need to implement the ConvertTo and ConvertFrom methods + for _, condition := range src.Status.Conditions { + dst.Status.Conditions = append(dst.Status.Conditions, YurtAppSetCondition{ + Type: YurtAppSetConditionType(condition.Type), + Status: condition.Status, + LastTransitionTime: condition.LastTransitionTime, + Reason: condition.Reason, + Message: condition.Message, + }) + } -// need import "sigs.k8s.io/controller-runtime/pkg/conversion" -//func (dst *YurtAppSet) ConvertFrom(srcRaw conversion.Hub) error { -// return nil -//} + klog.V(4).Infof("convert from v1beta1 to v1alpha1 for yurtappset %s", src.Name) + return nil +} diff --git a/pkg/apis/apps/v1alpha1/yurtappset_types.go b/pkg/apis/apps/v1alpha1/yurtappset_types.go index 4f333d203d7..7f1a7293702 100644 --- a/pkg/apis/apps/v1alpha1/yurtappset_types.go +++ b/pkg/apis/apps/v1alpha1/yurtappset_types.go @@ -211,12 +211,13 @@ type YurtAppSetCondition struct { // +genclient // +kubebuilder:object:root=true +// +kubebuilder:resource:shortName=yas,categories=all // +kubebuilder:subresource:status -// +kubebuilder:resource:shortName=yas // +kubebuilder:printcolumn:name="READY",type="integer",JSONPath=".status.readyReplicas",description="The number of pods ready." // +kubebuilder:printcolumn:name="WorkloadTemplate",type="string",JSONPath=".status.templateType",description="The WorkloadTemplate Type." // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC." // +kubebuilder:printcolumn:name="OverriderRef",type="string",JSONPath=".status.overriderRef",description="The name of overrider bound to this yurtappset" +// +kubebuilder:deprecatedversion:warning="apps.openyurt.io/v1alpha1 YurtAppSet is deprecated; use apps.openyurt.io/v1beta1 YurtAppSet; v1alpha1 YurtAppSet.Status.WorkloadSummary should not be used" // YurtAppSet is the Schema for the yurtAppSets API type YurtAppSet struct { diff --git a/pkg/apis/apps/v1beta1/nodepool_types.go b/pkg/apis/apps/v1beta1/nodepool_types.go index 3a28b1d8f89..34eaaa51c81 100644 --- a/pkg/apis/apps/v1beta1/nodepool_types.go +++ b/pkg/apis/apps/v1beta1/nodepool_types.go @@ -70,7 +70,6 @@ type NodePoolStatus struct { Nodes []string `json:"nodes,omitempty"` } -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +genclient // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,path=nodepools,shortName=np,categories=all diff --git a/pkg/apis/apps/v1beta1/yurtappset_conversion.go b/pkg/apis/apps/v1beta1/yurtappset_conversion.go new file mode 100644 index 00000000000..b7aa656c561 --- /dev/null +++ b/pkg/apis/apps/v1beta1/yurtappset_conversion.go @@ -0,0 +1,26 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +/* +Implementing the hub method is pretty easy -- we just have to add an empty +method called `Hub()` to serve as a +[marker](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/conversion?tab=doc#Hub). +*/ + +// Hub marks this type as a conversion hub. +func (*YurtAppSet) Hub() {} diff --git a/pkg/apis/apps/v1beta1/yurtappset_types.go b/pkg/apis/apps/v1beta1/yurtappset_types.go new file mode 100644 index 00000000000..844a5bc15a2 --- /dev/null +++ b/pkg/apis/apps/v1beta1/yurtappset_types.go @@ -0,0 +1,242 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// YurtAppSetSpec defines the desired state of YurtAppSet. +type YurtAppSetSpec struct { + // Workload defines the workload to be deployed in the nodepools + Workload `json:"workload"` + + // NodePoolSelector is a label query over nodepool in which workloads should be deployed in. + // It must match the nodepool's labels. + // +optional + NodePoolSelector *metav1.LabelSelector `json:"nodepoolSelector,omitempty"` + + // Pools is a list of selected nodepools specified with nodepool id in which workloads should be deployed in. + // It is primarily used for compatibility with v1alpha1 version and NodePoolSelector should be preferred to choose nodepools + // +optional + Pools []string `json:"pools,omitempty"` + + // Indicates the number of histories to be conserved. + // If unspecified, defaults to 10. + // +optional + RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` +} + +// Workload defines the workload to be deployed in the nodepools +type Workload struct { + // WorkloadTemplate defines the pool template under the YurtAppSet. + WorkloadTemplate `json:"workloadTemplate"` + // WorkloadTemplate defines the customization that will be applied to certain workloads in specified nodepools. + // +optional + WorkloadTweaks []WorkloadTweak `json:"workloadTweaks,omitempty"` +} + +// WorkloadTemplate defines the pool template under the YurtAppSet. +// YurtAppSet will provision every pool based on one workload templates in WorkloadTemplate. +// WorkloadTemplate now support statefulset and deployment +// Only one of its members may be specified. +type WorkloadTemplate struct { + // StatefulSet template + // +optional + StatefulSetTemplate *StatefulSetTemplateSpec `json:"statefulSetTemplate,omitempty"` + + // Deployment template + // +optional + DeploymentTemplate *DeploymentTemplateSpec `json:"deploymentTemplate,omitempty"` +} + +// StatefulSetTemplateSpec defines the pool template of StatefulSet. +type StatefulSetTemplateSpec struct { + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + metav1.ObjectMeta `json:"metadata,omitempty"` + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + Spec appsv1.StatefulSetSpec `json:"spec"` +} + +// DeploymentTemplateSpec defines the pool template of Deployment. +type DeploymentTemplateSpec struct { + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + metav1.ObjectMeta `json:"metadata,omitempty"` + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + Spec appsv1.DeploymentSpec `json:"spec"` +} + +// WorkloadTweak Describe detailed multi-region configuration of the subject +// BasicTweaks and AdvancedTweaks describe a set of nodepools and their shared or identical configurations +type WorkloadTweak struct { + // NodePoolSelector is a label query over nodepool in which workloads should be adjusted. + // +optional + NodePoolSelector *metav1.LabelSelector `json:"nodepoolSelector,omitempty"` + // Pools is a list of selected nodepools specified with nodepool id in which workloads should be adjusted. + // Pools is not recommended and NodePoolSelector should be preferred + // +optional + Pools []string `json:"pools,omitempty"` + // Tweaks is the adjustment can be applied to a certain workload in specified nodepools such as image and replicas + Tweaks `json:"tweaks"` +} + +// Tweaks represents configuration to be injected. +// Only one of its members may be specified. +type Tweaks struct { + // +optional + // Replicas overrides the replicas of the workload + Replicas *int32 `json:"replicas,omitempty"` + // +optional + // ContainerImages is a list of container images to be injected to a certain workload + ContainerImages []ContainerImage `json:"containerImages,omitempty"` + // +optional + // Patches is a list of advanced tweaks to be applied to a certain workload + // It can add/remove/replace the field values of specified paths in the template. + Patches []Patch `json:"patches,omitempty"` +} + +// ContainerImage specifies the corresponding container and the target image +type ContainerImage struct { + // Name represents name of the container in which the Image will be replaced + Name string `json:"name"` + // TargetImage represents the image name which is injected into the container above + TargetImage string `json:"targetImage"` +} + +type Operation string + +const ( + ADD Operation = "add" // json patch + REMOVE Operation = "remove" // json patch + REPLACE Operation = "replace" // json patch +) + +type Patch struct { + // Path represents the path in the json patch + Path string `json:"path"` + // Operation represents the operation + // +kubebuilder:validation:Enum=add;remove;replace + Operation Operation `json:"operation"` + // Indicates the value of json patch + // +optional + Value apiextensionsv1.JSON `json:"value,omitempty"` +} + +// YurtAppSetStatus defines the observed state of YurtAppSet. +type YurtAppSetStatus struct { + // ObservedGeneration is the most recent generation observed for this YurtAppSet. It corresponds to the + // YurtAppSet's generation, which is updated on mutation by the API Server. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Count of hash collisions for the YurtAppSet. The YurtAppSet controller + // uses this field as a collision avoidance mechanism when it needs to + // create the name for the newest ControllerRevision. + // +optional + CollisionCount *int32 `json:"collisionCount,omitempty"` + + // CurrentRevision, if not empty, indicates the current version of the YurtAppSet. + CurrentRevision string `json:"currentRevision"` + + // Represents the latest available observations of a YurtAppSet's current state. + // +optional + Conditions []YurtAppSetCondition `json:"conditions,omitempty"` + + // The number of ready workloads. + ReadyWorkloads int32 `json:"readyWorkloads"` + + // The number of updated workloads. + // +optional + UpdatedWorkloads int32 `json:"updatedWorkloads"` + + // TotalWorkloads is the most recently observed number of workloads. + TotalWorkloads int32 `json:"totalWorkloads"` +} + +// YurtAppSetConditionType indicates valid conditions type of a YurtAppSet. +type YurtAppSetConditionType string + +const ( + // AppDispatched means all the expected workloads are created successfully. + AppSetAppDispatchced YurtAppSetConditionType = "AppDispatched" + // AppDeleted means all the unexpected workloads are deleted successfully. + AppSetAppDeleted YurtAppSetConditionType = "AppDeleted" + // AppUpdated means all expected workloads are updated successfully. + AppSetAppUpdated YurtAppSetConditionType = "AppUpdated" + // AppUpdated means all workloads are ready + AppSetAppReady YurtAppSetConditionType = "AppReady" + // PoolFound is added to a YurtAppSet when all specified nodepools are found + // if no nodepools meets the nodepoolselector or pools of yurtappset, PoolFound condition is set to false + AppSetPoolFound YurtAppSetConditionType = "PoolFound" +) + +// YurtAppSetCondition describes current state of a YurtAppSet. +type YurtAppSetCondition struct { + // Type of in place set condition. + Type YurtAppSetConditionType `json:"type,omitempty"` + + // Status of the condition, one of True, False, Unknown. + Status corev1.ConditionStatus `json:"status,omitempty"` + + // Last time the condition transitioned from one status to another. + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + + // The reason for the condition's last transition. + Reason string `json:"reason,omitempty"` + + // A human readable message indicating details about the transition. + Message string `json:"message,omitempty"` +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:resource:shortName=yas,categories=all +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="TOTAL",type="integer",JSONPath=".status.totalWorkloads",description="The total number of workloads." +// +kubebuilder:printcolumn:name="READY",type="integer",JSONPath=".status.readyWorkloads",description="The number of workloads ready." +// +kubebuilder:printcolumn:name="UPDATED",type="integer",JSONPath=".status.updatedWorkloads",description="The number of workloads updated." +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC." +// +kubebuilder:storageversion + +// YurtAppSet is the Schema for the YurtAppSets API +type YurtAppSet struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec YurtAppSetSpec `json:"spec,omitempty"` + Status YurtAppSetStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// YurtAppSetList contains a list of YurtAppSet +type YurtAppSetList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []YurtAppSet `json:"items"` +} + +func init() { + SchemeBuilder.Register(&YurtAppSet{}, &YurtAppSetList{}) +} diff --git a/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go b/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go index aed2b7ae309..bfef4f9854e 100644 --- a/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go @@ -23,9 +23,42 @@ package v1beta1 import ( "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContainerImage) DeepCopyInto(out *ContainerImage) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerImage. +func (in *ContainerImage) DeepCopy() *ContainerImage { + if in == nil { + return nil + } + out := new(ContainerImage) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentTemplateSpec) DeepCopyInto(out *DeploymentTemplateSpec) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentTemplateSpec. +func (in *DeploymentTemplateSpec) DeepCopy() *DeploymentTemplateSpec { + if in == nil { + return nil + } + out := new(DeploymentTemplateSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodePool) DeepCopyInto(out *NodePool) { *out = *in @@ -140,3 +173,275 @@ func (in *NodePoolStatus) DeepCopy() *NodePoolStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Patch) DeepCopyInto(out *Patch) { + *out = *in + in.Value.DeepCopyInto(&out.Value) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Patch. +func (in *Patch) DeepCopy() *Patch { + if in == nil { + return nil + } + out := new(Patch) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StatefulSetTemplateSpec) DeepCopyInto(out *StatefulSetTemplateSpec) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetTemplateSpec. +func (in *StatefulSetTemplateSpec) DeepCopy() *StatefulSetTemplateSpec { + if in == nil { + return nil + } + out := new(StatefulSetTemplateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Tweaks) DeepCopyInto(out *Tweaks) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + if in.ContainerImages != nil { + in, out := &in.ContainerImages, &out.ContainerImages + *out = make([]ContainerImage, len(*in)) + copy(*out, *in) + } + if in.Patches != nil { + in, out := &in.Patches, &out.Patches + *out = make([]Patch, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Tweaks. +func (in *Tweaks) DeepCopy() *Tweaks { + if in == nil { + return nil + } + out := new(Tweaks) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Workload) DeepCopyInto(out *Workload) { + *out = *in + in.WorkloadTemplate.DeepCopyInto(&out.WorkloadTemplate) + if in.WorkloadTweaks != nil { + in, out := &in.WorkloadTweaks, &out.WorkloadTweaks + *out = make([]WorkloadTweak, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Workload. +func (in *Workload) DeepCopy() *Workload { + if in == nil { + return nil + } + out := new(Workload) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadTemplate) DeepCopyInto(out *WorkloadTemplate) { + *out = *in + if in.StatefulSetTemplate != nil { + in, out := &in.StatefulSetTemplate, &out.StatefulSetTemplate + *out = new(StatefulSetTemplateSpec) + (*in).DeepCopyInto(*out) + } + if in.DeploymentTemplate != nil { + in, out := &in.DeploymentTemplate, &out.DeploymentTemplate + *out = new(DeploymentTemplateSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadTemplate. +func (in *WorkloadTemplate) DeepCopy() *WorkloadTemplate { + if in == nil { + return nil + } + out := new(WorkloadTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadTweak) DeepCopyInto(out *WorkloadTweak) { + *out = *in + if in.NodePoolSelector != nil { + in, out := &in.NodePoolSelector, &out.NodePoolSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.Pools != nil { + in, out := &in.Pools, &out.Pools + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Tweaks.DeepCopyInto(&out.Tweaks) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadTweak. +func (in *WorkloadTweak) DeepCopy() *WorkloadTweak { + if in == nil { + return nil + } + out := new(WorkloadTweak) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *YurtAppSet) DeepCopyInto(out *YurtAppSet) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new YurtAppSet. +func (in *YurtAppSet) DeepCopy() *YurtAppSet { + if in == nil { + return nil + } + out := new(YurtAppSet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *YurtAppSet) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *YurtAppSetCondition) DeepCopyInto(out *YurtAppSetCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new YurtAppSetCondition. +func (in *YurtAppSetCondition) DeepCopy() *YurtAppSetCondition { + if in == nil { + return nil + } + out := new(YurtAppSetCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *YurtAppSetList) DeepCopyInto(out *YurtAppSetList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]YurtAppSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new YurtAppSetList. +func (in *YurtAppSetList) DeepCopy() *YurtAppSetList { + if in == nil { + return nil + } + out := new(YurtAppSetList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *YurtAppSetList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *YurtAppSetSpec) DeepCopyInto(out *YurtAppSetSpec) { + *out = *in + in.Workload.DeepCopyInto(&out.Workload) + if in.NodePoolSelector != nil { + in, out := &in.NodePoolSelector, &out.NodePoolSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.Pools != nil { + in, out := &in.Pools, &out.Pools + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.RevisionHistoryLimit != nil { + in, out := &in.RevisionHistoryLimit, &out.RevisionHistoryLimit + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new YurtAppSetSpec. +func (in *YurtAppSetSpec) DeepCopy() *YurtAppSetSpec { + if in == nil { + return nil + } + out := new(YurtAppSetSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *YurtAppSetStatus) DeepCopyInto(out *YurtAppSetStatus) { + *out = *in + if in.CollisionCount != nil { + in, out := &in.CollisionCount, &out.CollisionCount + *out = new(int32) + **out = **in + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]YurtAppSetCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new YurtAppSetStatus. +func (in *YurtAppSetStatus) DeepCopy() *YurtAppSetStatus { + if in == nil { + return nil + } + out := new(YurtAppSetStatus) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/apis/apps/well_known_labels_annotations.go b/pkg/apis/apps/well_known_labels_annotations.go index 2fe3e5d4059..082ef47ee3f 100644 --- a/pkg/apis/apps/well_known_labels_annotations.go +++ b/pkg/apis/apps/well_known_labels_annotations.go @@ -22,6 +22,9 @@ package apps // YurtAppSet & YurtAppDaemon related labels and annotations const ( + // YurtAppSetOwnerLabelKey is used to record which yas owns this deployment + YurtAppSetOwnerLabelKey = "apps.openyurt.io/ref-yurtappset" + // ControllerRevisionHashLabelKey is used to record the controller revision of current resource. ControllerRevisionHashLabelKey = "apps.openyurt.io/controller-revision-hash" diff --git a/pkg/yurtmanager/controller/util/refmanager/ref_manager.go b/pkg/yurtmanager/controller/util/refmanager/ref_manager.go index dbe2cdc2bcb..2ee86b2c3d0 100644 --- a/pkg/yurtmanager/controller/util/refmanager/ref_manager.go +++ b/pkg/yurtmanager/controller/util/refmanager/ref_manager.go @@ -250,6 +250,10 @@ func (mgr *RefManager) claimObject(obj metav1.Object, match func(metav1.Object) // Ignore if the object is being deleted return false, nil } + if len(mgr.owner.GetNamespace()) > 0 && mgr.owner.GetNamespace() != obj.GetNamespace() { + // Ignore if the object is in a different namespace + return false, nil + } // Selector matches. Try to adopt. if err := mgr.adopt(obj); err != nil { // If the pod no longer exists, ignore the error. diff --git a/pkg/yurtmanager/controller/util/tools.go b/pkg/yurtmanager/controller/util/tools.go index 2f3d9302a00..85a44b46a37 100644 --- a/pkg/yurtmanager/controller/util/tools.go +++ b/pkg/yurtmanager/controller/util/tools.go @@ -23,6 +23,7 @@ import ( "sync" "time" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/util/workqueue" "k8s.io/utils/integer" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -62,8 +63,13 @@ func SlowStartBatch(count int, initialBatchSize int, fn func(index int) error) ( wg.Wait() curSuccesses := batchSize - len(errCh) successes += curSuccesses + close(errCh) if len(errCh) > 0 { - return successes, <-errCh + errs := make([]error, 0) + for err := range errCh { + errs = append(errs, err) + } + return successes, utilerrors.NewAggregate(errs) } remaining -= batchSize } diff --git a/pkg/yurtmanager/controller/util/tools_test.go b/pkg/yurtmanager/controller/util/tools_test.go index c1c94d89115..870c327f463 100644 --- a/pkg/yurtmanager/controller/util/tools_test.go +++ b/pkg/yurtmanager/controller/util/tools_test.go @@ -77,6 +77,16 @@ func TestSlowStartBatch(t *testing.T) { }, } + errEqual := func(err1, err2 error) bool { + if err1 == nil && err2 == nil { + return true + } + if err1 == nil || err2 == nil { + return false + } + return err1.Error() == err2.Error() + } + for _, test := range tests { callCnt = 0 callLimit = test.callLimit @@ -84,7 +94,7 @@ func TestSlowStartBatch(t *testing.T) { if successes != test.expectedSuccesses { t.Errorf("%s: unexpected processed batch size, expected %d, got %d", test.name, test.expectedSuccesses, successes) } - if err != test.expectedErr { + if !errEqual(err, test.expectedErr) { t.Errorf("%s: unexpected processed batch size, expected %v, got %v", test.name, test.expectedErr, err) } // verify that slowStartBatch stops trying more calls after a batch fails diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util_test.go b/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util_test.go index 614ed00ebe3..a601e23d627 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util_test.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util_test.go @@ -1,12 +1,11 @@ /* Copyright 2021 The OpenYurt Authors. Copyright 2019 The Kruise Authors. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -14,7 +13,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - package adapter import ( @@ -26,7 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps" + "github.com/openyurtio/openyurt/pkg/apis/apps" ) func TestGetCurrentPartitionForStrategyOnDelete(t *testing.T) { @@ -34,38 +32,31 @@ func TestGetCurrentPartitionForStrategyOnDelete(t *testing.T) { if partition := getCurrentPartition(currentPods, "v2"); *partition != 1 { t.Fatalf("expected partition 1, got %d", *partition) } - currentPods = buildPodList([]int{0, 1, 2}, []string{"v1", "v1", "v2"}, t) if partition := getCurrentPartition(currentPods, "v2"); *partition != 2 { t.Fatalf("expected partition 2, got %d", *partition) } - currentPods = buildPodList([]int{0, 1, 2, 3}, []string{"v2", "v1", "v2", "v2"}, t) if partition := getCurrentPartition(currentPods, "v2"); *partition != 1 { t.Fatalf("expected partition 1, got %d", *partition) } - currentPods = buildPodList([]int{1, 2, 3}, []string{"v1", "v2", "v2"}, t) if partition := getCurrentPartition(currentPods, "v2"); *partition != 1 { t.Fatalf("expected partition 1, got %d", *partition) } - currentPods = buildPodList([]int{0, 1, 3}, []string{"v2", "v1", "v2"}, t) if partition := getCurrentPartition(currentPods, "v2"); *partition != 1 { t.Fatalf("expected partition 1, got %d", *partition) } - currentPods = buildPodList([]int{0, 1, 2}, []string{"v1", "v1", "v1"}, t) if partition := getCurrentPartition(currentPods, "v2"); *partition != 3 { t.Fatalf("expected partition 3, got %d", *partition) } - currentPods = buildPodList([]int{0, 1, 2, 4}, []string{"v1", "", "v2", "v3"}, t) if partition := getCurrentPartition(currentPods, "v2"); *partition != 3 { t.Fatalf("expected partition 3, got %d", *partition) } } - func buildPodList(ordinals []int, revisions []string, t *testing.T) []*corev1.Pod { if len(ordinals) != len(revisions) { t.Fatalf("ordinals count should equals to revision count") @@ -80,15 +71,13 @@ func buildPodList(ordinals []int, revisions []string, t *testing.T) []*corev1.Po } if revisions[i] != "" { pod.Labels = map[string]string{ - unitv1alpha1.ControllerRevisionHashLabelKey: revisions[i], + apps.ControllerRevisionHashLabelKey: revisions[i], } } pods = append(pods, pod) } - return pods } - func TestCreateNewPatchedObject(t *testing.T) { cases := []struct { Name string @@ -146,7 +135,6 @@ func TestCreateNewPatchedObject(t *testing.T) { if !ok { return false } - image1, ok := containerMap["nginx111"] if !ok { return false @@ -155,7 +143,6 @@ func TestCreateNewPatchedObject(t *testing.T) { }, }, } - for _, c := range cases { t.Run(c.Name, func(t *testing.T) { newObj := &appsv1.Deployment{} diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/adapter.go b/pkg/yurtmanager/controller/yurtappset/adapter/adpater.go similarity index 97% rename from pkg/yurtmanager/controller/yurtappset/adapter/adapter.go rename to pkg/yurtmanager/controller/yurtappset/adapter/adpater.go index 9ff1110c16a..48949da1c01 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/adapter.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/adpater.go @@ -1,12 +1,11 @@ /* Copyright 2020 The OpenYurt Authors. Copyright 2019 The Kruise Authors. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -17,7 +16,6 @@ limitations under the License. OpenYurt Authors: change Adapter interface */ - package adapter import ( @@ -49,7 +47,6 @@ type Adapter interface { // PostUpdate does some works after pool updated PostUpdate(yas *alpha1.YurtAppSet, pool runtime.Object, revision string) error } - type ReplicasInfo struct { Replicas int32 ReadyReplicas int32 diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util.go b/pkg/yurtmanager/controller/yurtappset/adapter/adpater_util.go similarity index 98% rename from pkg/yurtmanager/controller/yurtappset/adapter/adapter_util.go rename to pkg/yurtmanager/controller/yurtappset/adapter/adpater_util.go index e62094596c4..83d1003b162 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/adpater_util.go @@ -1,12 +1,11 @@ /* Copyright 2021 The OpenYurt Authors. Copyright 2019 The Kruise Authors. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -14,7 +13,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - package adapter import ( @@ -39,33 +37,26 @@ func getPoolPrefix(controllerName, poolName string) string { } return prefix } - func attachNodeAffinityAndTolerations(podSpec *corev1.PodSpec, pool *appsv1alpha1.Pool) { attachNodeAffinity(podSpec, pool) attachTolerations(podSpec, pool) } - func attachNodeAffinity(podSpec *corev1.PodSpec, pool *appsv1alpha1.Pool) { if podSpec.Affinity == nil { podSpec.Affinity = &corev1.Affinity{} } - if podSpec.Affinity.NodeAffinity == nil { podSpec.Affinity.NodeAffinity = &corev1.NodeAffinity{} } - if podSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { podSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{} } - if podSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms == nil { podSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = []corev1.NodeSelectorTerm{} } - if len(podSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms) == 0 { podSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(podSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, corev1.NodeSelectorTerm{}) } - for _, matchExpression := range pool.NodeSelectorTerm.MatchExpressions { for i, term := range podSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { term.MatchExpressions = append(term.MatchExpressions, matchExpression) @@ -73,22 +64,16 @@ func attachNodeAffinity(podSpec *corev1.PodSpec, pool *appsv1alpha1.Pool) { } } } - func attachTolerations(podSpec *corev1.PodSpec, poolConfig *appsv1alpha1.Pool) { - if poolConfig.Tolerations == nil { return } - if podSpec.Tolerations == nil { podSpec.Tolerations = []corev1.Toleration{} } - podSpec.Tolerations = append(podSpec.Tolerations, poolConfig.Tolerations...) - return } - func getRevision(objMeta metav1.Object) string { if objMeta.GetLabels() == nil { return "" @@ -104,36 +89,30 @@ func getCurrentPartition(pods []*corev1.Pod, revision string) *int32 { partition++ } } - return &partition } - func StrategicMergeByPatches(oldobj interface{}, patch *runtime.RawExtension, newPatched interface{}) error { patchMap := make(map[string]interface{}) if err := json.Unmarshal(patch.Raw, &patchMap); err != nil { klog.Errorf("Unmarshal pool patch error %v, patch Raw %v", err, string(patch.Raw)) return err } - originalObjMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(oldobj) if err != nil { klog.Errorf("ToUnstructured error %v", err) return err } - patchedObjMap, err := strategicpatch.StrategicMergeMapPatch(originalObjMap, patchMap, newPatched) if err != nil { klog.Errorf("StartegicMergeMapPatch error %v", err) return err } - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(patchedObjMap, newPatched); err != nil { klog.Errorf("FromUnstructured error %v", err) return err } return nil } - func PoolHasPatch(poolConfig *appsv1alpha1.Pool, set metav1.Object) bool { if poolConfig.Patch == nil { // If No Patches, Must Set patches annotation to "" @@ -144,13 +123,10 @@ func PoolHasPatch(poolConfig *appsv1alpha1.Pool, set metav1.Object) bool { } return true } - func CreateNewPatchedObject(patchInfo *runtime.RawExtension, set metav1.Object, newPatched metav1.Object) error { - if err := StrategicMergeByPatches(set, patchInfo, newPatched); err != nil { return err } - if anno := newPatched.GetAnnotations(); anno == nil { newPatched.SetAnnotations(map[string]string{ apps.AnnotationPatchKey: string(patchInfo.Raw), diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter.go b/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter.go index 7b6c55f1e0f..bc4c21e2046 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter.go @@ -1,11 +1,10 @@ /* Copyright 2021 The OpenYurt Authors. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -13,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - package adapter import ( @@ -33,7 +31,6 @@ import ( type DeploymentAdapter struct { client.Client - Scheme *runtime.Scheme } @@ -57,7 +54,6 @@ func (a *DeploymentAdapter) GetStatusObservedGeneration(obj metav1.Object) int64 // GetDetails returns the replicas detail the pool needs. func (a *DeploymentAdapter) GetDetails(obj metav1.Object) (ReplicasInfo, error) { set := obj.(*appsv1.Deployment) - var specReplicas int32 if set.Spec.Replicas != nil { specReplicas = *set.Spec.Replicas @@ -90,7 +86,6 @@ func (a *DeploymentAdapter) GetPoolFailure() *string { func (a *DeploymentAdapter) ApplyPoolTemplate(yas *alpha1.YurtAppSet, poolName, revision string, replicas int32, obj runtime.Object) error { set := obj.(*appsv1.Deployment) - var poolConfig *alpha1.Pool for i, pool := range yas.Spec.Topology.Pools { if pool.Name == poolName { @@ -101,9 +96,7 @@ func (a *DeploymentAdapter) ApplyPoolTemplate(yas *alpha1.YurtAppSet, poolName, if poolConfig == nil { return fmt.Errorf("could not find pool config %s", poolName) } - set.Namespace = yas.Namespace - if set.Labels == nil { set.Labels = map[string]string{} } @@ -116,26 +109,20 @@ func (a *DeploymentAdapter) ApplyPoolTemplate(yas *alpha1.YurtAppSet, poolName, set.Labels[apps.ControllerRevisionHashLabelKey] = revision // record the pool name as a label set.Labels[apps.PoolNameLabelKey] = poolName - if set.Annotations == nil { set.Annotations = map[string]string{} } for k, v := range yas.Spec.WorkloadTemplate.DeploymentTemplate.Annotations { set.Annotations[k] = v } - set.GenerateName = getPoolPrefix(yas.Name, poolName) - selectors := yas.Spec.Selector.DeepCopy() selectors.MatchLabels[apps.PoolNameLabelKey] = poolName - if err := controllerutil.SetControllerReference(yas, set, a.Scheme); err != nil { return err } - set.Spec.Selector = selectors set.Spec.Replicas = &replicas - set.Spec.Strategy = *yas.Spec.WorkloadTemplate.DeploymentTemplate.Spec.Strategy.DeepCopy() set.Spec.Template = *yas.Spec.WorkloadTemplate.DeploymentTemplate.Spec.Template.DeepCopy() if set.Spec.Template.Labels == nil { @@ -143,20 +130,16 @@ func (a *DeploymentAdapter) ApplyPoolTemplate(yas *alpha1.YurtAppSet, poolName, } set.Spec.Template.Labels[apps.PoolNameLabelKey] = poolName set.Spec.Template.Labels[apps.ControllerRevisionHashLabelKey] = revision - set.Spec.RevisionHistoryLimit = yas.Spec.RevisionHistoryLimit set.Spec.MinReadySeconds = yas.Spec.WorkloadTemplate.DeploymentTemplate.Spec.MinReadySeconds set.Spec.Paused = yas.Spec.WorkloadTemplate.DeploymentTemplate.Spec.Paused set.Spec.ProgressDeadlineSeconds = yas.Spec.WorkloadTemplate.DeploymentTemplate.Spec.ProgressDeadlineSeconds - attachNodeAffinityAndTolerations(&set.Spec.Template.Spec, poolConfig) - if !PoolHasPatch(poolConfig, set) { klog.Infof("Deployment[%s/%s-] has no patches, do not need strategicmerge", set.Namespace, set.GenerateName) return nil } - patched := &appsv1.Deployment{} if err := CreateNewPatchedObject(poolConfig.Patch, set, patched); err != nil { klog.Errorf("Deployment[%s/%s-] strategic merge by patch %s error %v", set.Namespace, @@ -164,7 +147,6 @@ func (a *DeploymentAdapter) ApplyPoolTemplate(yas *alpha1.YurtAppSet, poolName, return err } patched.DeepCopyInto(set) - klog.Infof("Deployment [%s/%s-] has patches configure successfully:%v", set.Namespace, set.GenerateName, string(poolConfig.Patch.Raw)) return nil diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter_test.go b/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter_test.go index c0f41234cc7..03ce66e245c 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter_test.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter_test.go @@ -1,11 +1,10 @@ /* Copyright 2022 The OpenYurt Authors. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -13,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - package adapter import ( @@ -33,7 +31,6 @@ import ( func TestDeploymentAdapter_ApplyPoolTemplate(t *testing.T) { var one int32 = 1 - cases := []struct { name string yas *appsv1alpha1.YurtAppSet @@ -109,7 +106,6 @@ func TestDeploymentAdapter_ApplyPoolTemplate(t *testing.T) { revision: "1", replicas: one, obj: &appsv1.Deployment{}, - wantDeploy: &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", @@ -180,23 +176,19 @@ func TestDeploymentAdapter_ApplyPoolTemplate(t *testing.T) { return } fc := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects().Build() - da := DeploymentAdapter{Client: fc, Scheme: scheme} - for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { err := da.ApplyPoolTemplate(tt.yas, tt.poolName, tt.revision, tt.replicas, tt.obj) if err != nil { t.Logf("failed to appply pool template") } - if err = controllerutil.SetControllerReference(tt.yas, tt.wantDeploy, scheme); err != nil { panic(err) } }) } } - func TestDeploymentAdapter_GetDetails(t *testing.T) { var one int32 = 1 cases := []struct { @@ -220,7 +212,6 @@ func TestDeploymentAdapter_GetDetails(t *testing.T) { }, }, } - scheme := runtime.NewScheme() if err := appsv1alpha1.AddToScheme(scheme); err != nil { t.Logf("failed to add yurt custom resource") @@ -231,12 +222,9 @@ func TestDeploymentAdapter_GetDetails(t *testing.T) { return } fc := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects().Build() - da := DeploymentAdapter{Client: fc, Scheme: scheme} - for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { - got, err := da.GetDetails(tt.obj) if err != nil || got.Replicas != tt.wantReplicasInfo.Replicas { t.Logf("failed to get details") diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter.go b/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter.go deleted file mode 100644 index 7d48cbea2dd..00000000000 --- a/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter.go +++ /dev/null @@ -1,267 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. -Copyright 2019 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -@CHANGELOG -OpenYurt Authors: -change statefulset adapter -*/ - -package adapter - -import ( - "fmt" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - "github.com/openyurtio/openyurt/pkg/apis/apps" - alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" -) - -type StatefulSetAdapter struct { - client.Client - - Scheme *runtime.Scheme -} - -var _ Adapter = &StatefulSetAdapter{} - -// NewResourceObject creates a empty StatefulSet object. -func (a *StatefulSetAdapter) NewResourceObject() runtime.Object { - return &appsv1.StatefulSet{} -} - -// NewResourceListObject creates a empty StatefulSetList object. -func (a *StatefulSetAdapter) NewResourceListObject() runtime.Object { - return &appsv1.StatefulSetList{} -} - -// GetStatusObservedGeneration returns the observed generation of the pool. -func (a *StatefulSetAdapter) GetStatusObservedGeneration(obj metav1.Object) int64 { - return obj.(*appsv1.StatefulSet).Status.ObservedGeneration -} - -// GetDetails returns the replicas detail the pool needs. -func (a *StatefulSetAdapter) GetDetails(obj metav1.Object) (ReplicasInfo, error) { - set := obj.(*appsv1.StatefulSet) - - var specReplicas int32 - if set.Spec.Replicas != nil { - specReplicas = *set.Spec.Replicas - } - replicasInfo := ReplicasInfo{ - Replicas: specReplicas, - ReadyReplicas: set.Status.ReadyReplicas, - } - - return replicasInfo, nil -} - -// GetAvailableStatus returns the available condition status of the workload -func (a *StatefulSetAdapter) GetAvailableStatus(obj metav1.Object) (conditionStatus corev1.ConditionStatus, err error) { - set := obj.(*appsv1.StatefulSet) - - if set.Status.AvailableReplicas != set.Status.Replicas { - return corev1.ConditionFalse, nil - } - return corev1.ConditionTrue, nil -} - -// GetPoolFailure returns the failure information of the pool. -// StatefulSet has no condition. -func (a *StatefulSetAdapter) GetPoolFailure() *string { - return nil -} - -// ApplyPoolTemplate updates the pool to the latest revision, depending on the StatefulSetTemplate. -func (a *StatefulSetAdapter) ApplyPoolTemplate(yas *alpha1.YurtAppSet, poolName, revision string, - replicas int32, obj runtime.Object) error { - set := obj.(*appsv1.StatefulSet) - - var poolConfig *alpha1.Pool - for i, pool := range yas.Spec.Topology.Pools { - if pool.Name == poolName { - poolConfig = &(yas.Spec.Topology.Pools[i]) - break - } - } - if poolConfig == nil { - return fmt.Errorf("could not find pool config %s", poolName) - } - - set.Namespace = yas.Namespace - - if set.Labels == nil { - set.Labels = map[string]string{} - } - for k, v := range yas.Spec.WorkloadTemplate.StatefulSetTemplate.Labels { - set.Labels[k] = v - } - for k, v := range yas.Spec.Selector.MatchLabels { - set.Labels[k] = v - } - set.Labels[apps.ControllerRevisionHashLabelKey] = revision - // record the pool name as a label - set.Labels[apps.PoolNameLabelKey] = poolName - - if set.Annotations == nil { - set.Annotations = map[string]string{} - } - for k, v := range yas.Spec.WorkloadTemplate.StatefulSetTemplate.Annotations { - set.Annotations[k] = v - } - - set.GenerateName = getPoolPrefix(yas.Name, poolName) - - selectors := yas.Spec.Selector.DeepCopy() - selectors.MatchLabels[apps.PoolNameLabelKey] = poolName - - if err := controllerutil.SetControllerReference(yas, set, a.Scheme); err != nil { - return err - } - - set.Spec.Selector = selectors - set.Spec.Replicas = &replicas - - set.Spec.UpdateStrategy = *yas.Spec.WorkloadTemplate.StatefulSetTemplate.Spec.UpdateStrategy.DeepCopy() - set.Spec.Template = *yas.Spec.WorkloadTemplate.StatefulSetTemplate.Spec.Template.DeepCopy() - if set.Spec.Template.Labels == nil { - set.Spec.Template.Labels = map[string]string{} - } - set.Spec.Template.Labels[apps.PoolNameLabelKey] = poolName - set.Spec.Template.Labels[apps.ControllerRevisionHashLabelKey] = revision - - set.Spec.RevisionHistoryLimit = yas.Spec.RevisionHistoryLimit - set.Spec.PodManagementPolicy = yas.Spec.WorkloadTemplate.StatefulSetTemplate.Spec.PodManagementPolicy - set.Spec.ServiceName = yas.Spec.WorkloadTemplate.StatefulSetTemplate.Spec.ServiceName - set.Spec.VolumeClaimTemplates = yas.Spec.WorkloadTemplate.StatefulSetTemplate.Spec.VolumeClaimTemplates - - attachNodeAffinityAndTolerations(&set.Spec.Template.Spec, poolConfig) - - if !PoolHasPatch(poolConfig, set) { - klog.Infof("StatefulSet[%s/%s-] has no patches, do not need strategicmerge", set.Namespace, - set.GenerateName) - return nil - } - - patched := &appsv1.StatefulSet{} - if err := CreateNewPatchedObject(poolConfig.Patch, set, patched); err != nil { - klog.Errorf("StatefulSet[%s/%s-] strategic merge by patch %s error %v", set.Namespace, - set.GenerateName, string(poolConfig.Patch.Raw), err) - return err - } - patched.DeepCopyInto(set) - klog.Infof("Statefulset [%s/%s-] has patches configure successfully:%v", set.Namespace, - set.GenerateName, string(poolConfig.Patch.Raw)) - return nil -} - -// PostUpdate does some works after pool updated. StatefulSet will implement this method to clean stuck pods. -func (a *StatefulSetAdapter) PostUpdate(yas *alpha1.YurtAppSet, obj runtime.Object, revision string) error { - /* - if strategy == nil { - return nil - } - set := obj.(*appsv1.StatefulSet) - if set.Spec.UpdateStrategy.Type == appsv1.OnDeleteStatefulSetStrategyType { - return nil - } - - // If RollingUpdate, work around for issue https://github.com/kubernetes/kubernetes/issues/67250 - return a.deleteStuckPods(set, revision, strategy.GetPartition()) - */ - return nil -} - -// IsExpected checks the pool is the expected revision or not. -// The revision label can tell the current pool revision. -func (a *StatefulSetAdapter) IsExpected(obj metav1.Object, revision string) bool { - return obj.GetLabels()[apps.ControllerRevisionHashLabelKey] != revision -} - -/* -func (a *StatefulSetAdapter) getStatefulSetPods(set *appsv1.StatefulSet) ([]*corev1.Pod, error) { - selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) - if err != nil { - return nil, err - } - podList := &corev1.PodList{} - err = a.Client.List(context.TODO(), podList, &client.ListOptions{LabelSelector: selector}) - if err != nil { - return nil, err - } - - manager, err := refmanager.New(a.Client, set.Spec.Selector, set, a.Scheme) - if err != nil { - return nil, err - } - selected := make([]metav1.Object, len(podList.Items)) - for i, pod := range podList.Items { - selected[i] = pod.DeepCopy() - } - claimed, err := manager.ClaimOwnedObjects(selected) - if err != nil { - return nil, err - } - - claimedPods := make([]*corev1.Pod, len(claimed)) - for i, pod := range claimed { - claimedPods[i] = pod.(*corev1.Pod) - } - return claimedPods, nil -} - -// deleteStucckPods tries to work around the blocking issue https://github.com/kubernetes/kubernetes/issues/67250 -func (a *StatefulSetAdapter) deleteStuckPods(set *appsv1.StatefulSet, revision string, partition int32) error { - pods, err := a.getStatefulSetPods(set) - if err != nil { - return err - } - - for i := range pods { - pod := pods[i] - // If the pod is considered as stuck, delete it. - if isPodStuckForRollingUpdate(pod, revision, partition) { - klog.V(2).Infof("Delete pod %s/%s at stuck state", pod.Namespace, pod.Name) - err = a.Delete(context.TODO(), pod, client.PropagationPolicy(metav1.DeletePropagationBackground)) - if err != nil { - return err - } - } - } - return nil -} - -// isPodStuckForRollingUpdate checks whether the pod is stuck under strategy RollingUpdate. -// If a pod needs to upgrade (pod_ordinal >= partition && pod_revision != sts_revision) -// and its readiness is false, or worse status like Pending, ImagePullBackOff, it will be blocked. -func isPodStuckForRollingUpdate(pod *corev1.Pod, revision string, partition int32) bool { - if yurtctlutil.GetOrdinal(pod) < partition { - return false - } - - if getRevision(pod) == revision { - return false - } - - return !podutil.IsPodReadyConditionTrue(pod.Status) -} -*/ diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter_test.go b/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter_test.go deleted file mode 100644 index 2eb70775a33..00000000000 --- a/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter_test.go +++ /dev/null @@ -1,245 +0,0 @@ -/* -Copyright 2022 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package adapter - -import ( - "testing" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - fakeclint "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - "github.com/openyurtio/openyurt/pkg/apis/apps" - appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" -) - -func TestStatefulSetAdapter_ApplyPoolTemplate(t *testing.T) { - var one int32 = 1 - cases := []struct { - name string - yas *appsv1alpha1.YurtAppSet - poolName string - revision string - replicas int32 - obj runtime.Object - wantSts *appsv1.StatefulSet - }{ - { - name: "apply pool template", - yas: &appsv1alpha1.YurtAppSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "default", - }, - Spec: appsv1alpha1.YurtAppSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "name": "foo", - }, - }, - WorkloadTemplate: appsv1alpha1.WorkloadTemplate{ - StatefulSetTemplate: &appsv1alpha1.StatefulSetTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - apps.AnnotationPatchKey: "annotation-v", - }, - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: appsv1.StatefulSetSpec{ - Replicas: &one, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "container-a", - Image: "nginx:1.0", - }, - }, - }, - }, - }, - }, - }, - Topology: appsv1alpha1.Topology{ - Pools: []appsv1alpha1.Pool{ - { - Name: "hangzhou", - NodeSelectorTerm: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "node-name", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"nodeA"}, - }, - }, - }, - }, - }, - }, - RevisionHistoryLimit: &one, - }, - }, - poolName: "hangzhou", - revision: "1", - replicas: one, - obj: &appsv1.StatefulSet{}, - - wantSts: &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "default", - Labels: map[string]string{ - "name": "foo", - apps.ControllerRevisionHashLabelKey: "1", - apps.PoolNameLabelKey: "hangzhou", - }, - Annotations: map[string]string{ - apps.AnnotationPatchKey: "", - }, - GenerateName: "foo-hangzhou-", - }, - Spec: appsv1.StatefulSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "name": "foo", - apps.PoolNameLabelKey: "hangzhou", - }, - }, - Replicas: &one, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "name": "foo", - apps.ControllerRevisionHashLabelKey: "1", - apps.PoolNameLabelKey: "hangzhou", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "container-a", - Image: "nginx:1.0", - }, - }, - Affinity: &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "node-name", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"nodeA"}, - }, - }, - }, - }, - }, - }, - }, - }, - }, - RevisionHistoryLimit: &one, - }, - }, - }, - } - - scheme := runtime.NewScheme() - if err := appsv1alpha1.AddToScheme(scheme); err != nil { - t.Logf("failed to add yurt custom resource") - return - } - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Logf("failed to add kubernetes clint-go custom resource") - return - } - fc := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects().Build() - - sa := StatefulSetAdapter{Client: fc, Scheme: scheme} - - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - err := sa.ApplyPoolTemplate(tt.yas, tt.poolName, tt.revision, tt.replicas, tt.obj) - if err != nil { - t.Logf("failed to appply pool template") - } - if err = controllerutil.SetControllerReference(tt.yas, tt.wantSts, sa.Scheme); err != nil { - panic(err) - } - }) - } -} - -func TestStatefulSetAdapter_GetDetails(t *testing.T) { - var one int32 = 1 - tests := []struct { - name string - obj metav1.Object - wantReplicasInfo ReplicasInfo - }{ - { - name: "get statefulsetAdapter details", - obj: &appsv1.StatefulSet{ - Spec: appsv1.StatefulSetSpec{ - Replicas: &one, - }, - Status: appsv1.StatefulSetStatus{ - ReadyReplicas: one, - }, - }, - wantReplicasInfo: ReplicasInfo{ - Replicas: one, - ReadyReplicas: one, - }, - }, - } - - scheme := runtime.NewScheme() - if err := appsv1alpha1.AddToScheme(scheme); err != nil { - t.Logf("failed to add yurt custom resource") - return - } - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Logf("failed to add kubernetes clint-go custom resource") - return - } - fc := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects().Build() - - sa := StatefulSetAdapter{Client: fc, Scheme: scheme} - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - - got, err := sa.GetDetails(tt.obj) - if err != nil || got.Replicas != tt.wantReplicasInfo.Replicas { - t.Logf("failed to get details") - } - }) - } -} diff --git a/pkg/yurtmanager/controller/yurtappset/config/types.go b/pkg/yurtmanager/controller/yurtappset/config/types.go index dafc880d736..638058c95d3 100644 --- a/pkg/yurtmanager/controller/yurtappset/config/types.go +++ b/pkg/yurtmanager/controller/yurtappset/config/types.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The OpenYurt Authors. +Copyright 2024 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/yurtmanager/controller/yurtappset/pool.go b/pkg/yurtmanager/controller/yurtappset/pool.go deleted file mode 100644 index 5960d8afab4..00000000000 --- a/pkg/yurtmanager/controller/yurtappset/pool.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. -Copyright 2019 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package yurtappset - -import ( - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/adapter" -) - -// Pool stores the details of a pool resource owned by one YurtAppSet. -type Pool struct { - Name string - Namespace string - Spec PoolSpec - Status PoolStatus -} - -// PoolSpec stores the spec details of the Pool -type PoolSpec struct { - PoolRef metav1.Object -} - -// PoolStatus stores the observed state of the Pool. -type PoolStatus struct { - ObservedGeneration int64 - adapter.ReplicasInfo - PatchInfo string - AvailableCondition v1.ConditionStatus -} - -// ResourceRef stores the Pool resource it represents. -type ResourceRef struct { - Resources []metav1.Object -} - -// ControlInterface defines the interface that YurtAppSet uses to list, create, update, and delete Pools. -type ControlInterface interface { - // GetAllPools returns the pools which are managed by the YurtAppSet. - GetAllPools(yas *unitv1alpha1.YurtAppSet) ([]*Pool, error) - // CreatePool creates the pool depending on the inputs. - CreatePool(yas *unitv1alpha1.YurtAppSet, unit string, revision string, replicas int32) error - // UpdatePool updates the target pool with the input information. - UpdatePool(pool *Pool, yas *unitv1alpha1.YurtAppSet, revision string, replicas int32) error - // DeletePool is used to delete the input pool. - DeletePool(*Pool) error - // GetPoolFailure extracts the pool failure message to expose on YurtAppSet status. - GetPoolFailure(*Pool) *string - // IsExpected check the pool is the expected revision - IsExpected(pool *Pool, revision string) bool -} diff --git a/pkg/yurtmanager/controller/yurtappset/pool_control.go b/pkg/yurtmanager/controller/yurtappset/pool_control.go deleted file mode 100644 index d72801d4e2b..00000000000 --- a/pkg/yurtmanager/controller/yurtappset/pool_control.go +++ /dev/null @@ -1,188 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. -Copyright 2019 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package yurtappset - -import ( - "context" - "errors" - "reflect" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/openyurtio/openyurt/pkg/apis/apps" - alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/refmanager" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/adapter" -) - -// PoolControl provides pool operations of MutableSet. -type PoolControl struct { - client.Client - - scheme *runtime.Scheme - adapter adapter.Adapter -} - -// GetAllPools returns all of pools owned by the YurtAppSet. -func (m *PoolControl) GetAllPools(yas *alpha1.YurtAppSet) (pools []*Pool, err error) { - selector, err := metav1.LabelSelectorAsSelector(yas.Spec.Selector) - if err != nil { - return nil, err - } - - setList := m.adapter.NewResourceListObject() - cliSetList, ok := setList.(client.ObjectList) - if !ok { - return nil, errors.New("could not convert runtime object to client.ObjectList") - } - err = m.Client.List(context.TODO(), cliSetList, &client.ListOptions{LabelSelector: selector}) - if err != nil { - return nil, err - } - - manager, err := refmanager.New(m.Client, yas.Spec.Selector, yas, m.scheme) - if err != nil { - return nil, err - } - - v := reflect.ValueOf(setList).Elem().FieldByName("Items") - selected := make([]metav1.Object, v.Len()) - for i := 0; i < v.Len(); i++ { - selected[i] = v.Index(i).Addr().Interface().(metav1.Object) - } - claimedSets, err := manager.ClaimOwnedObjects(selected) - if err != nil { - return nil, err - } - - for _, claimedSet := range claimedSets { - pool, err := m.convertToPool(claimedSet) - if err != nil { - return nil, err - } - pools = append(pools, pool) - } - return pools, nil -} - -// CreatePool creates the Pool depending on the inputs. -func (m *PoolControl) CreatePool(yas *alpha1.YurtAppSet, poolName string, revision string, - replicas int32) error { - - set := m.adapter.NewResourceObject() - m.adapter.ApplyPoolTemplate(yas, poolName, revision, replicas, set) - - klog.V(4).Infof("Have %d replicas when creating Pool for YurtAppSet %s/%s", replicas, yas.Namespace, yas.Name) - cliSet, ok := set.(client.Object) - if !ok { - return errors.New("could not convert runtime.Object to client.Object") - } - return m.Create(context.TODO(), cliSet) -} - -// UpdatePool is used to update the pool. The target Pool workload can be found with the input pool. -func (m *PoolControl) UpdatePool(pool *Pool, yas *alpha1.YurtAppSet, revision string, replicas int32) error { - set := m.adapter.NewResourceObject() - cliSet, ok := set.(client.Object) - if !ok { - return errors.New("could not convert runtime.Object to client.Object") - } - var updateError error - for i := 0; i < updateRetries; i++ { - getError := m.Client.Get(context.TODO(), m.objectKey(pool), cliSet) - if getError != nil { - return getError - } - - if err := m.adapter.ApplyPoolTemplate(yas, pool.Name, revision, replicas, set); err != nil { - return err - } - updateError = m.Client.Update(context.TODO(), cliSet) - if updateError == nil { - break - } - } - - if updateError != nil { - return updateError - } - - return m.adapter.PostUpdate(yas, set, revision) -} - -// DeletePool is called to delete the pool. The target Pool workload can be found with the input pool. -func (m *PoolControl) DeletePool(pool *Pool) error { - set := pool.Spec.PoolRef.(runtime.Object) - cliSet, ok := set.(client.Object) - if !ok { - return errors.New("could not convert runtime.Object to client.Object") - } - return m.Delete(context.TODO(), cliSet, client.PropagationPolicy(metav1.DeletePropagationBackground)) -} - -// GetPoolFailure return the error message extracted form Pool workload status conditions. -func (m *PoolControl) GetPoolFailure(pool *Pool) *string { - return m.adapter.GetPoolFailure() -} - -// IsExpected checks the pool is expected revision or not. -func (m *PoolControl) IsExpected(pool *Pool, revision string) bool { - return m.adapter.IsExpected(pool.Spec.PoolRef, revision) -} - -func (m *PoolControl) convertToPool(set metav1.Object) (*Pool, error) { - poolName, err := getPoolNameFrom(set) - if err != nil { - return nil, err - } - specReplicas, err := m.adapter.GetDetails(set) - if err != nil { - return nil, err - } - conditionStatus, err := m.adapter.GetAvailableStatus(set) - if err != nil { - return nil, err - } - pool := &Pool{ - Name: poolName, - Namespace: set.GetNamespace(), - Spec: PoolSpec{ - PoolRef: set, - }, - Status: PoolStatus{ - ObservedGeneration: m.adapter.GetStatusObservedGeneration(set), - ReplicasInfo: specReplicas, - AvailableCondition: conditionStatus, - }, - } - if data, ok := set.GetAnnotations()[apps.AnnotationPatchKey]; ok { - pool.Status.PatchInfo = data - } - return pool, nil -} - -func (m *PoolControl) objectKey(pool *Pool) client.ObjectKey { - return types.NamespacedName{ - Namespace: pool.Namespace, - Name: pool.Spec.PoolRef.GetName(), - } -} diff --git a/pkg/yurtmanager/controller/yurtappset/pool_controller_test.go b/pkg/yurtmanager/controller/yurtappset/pool_controller_test.go deleted file mode 100644 index da2c8e0ac5a..00000000000 --- a/pkg/yurtmanager/controller/yurtappset/pool_controller_test.go +++ /dev/null @@ -1,316 +0,0 @@ -/* -Copyright 2022 The OpenYurt Authors. -Copyright 2019 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package yurtappset - -import ( - "strconv" - "testing" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - fakeclint "sigs.k8s.io/controller-runtime/pkg/client/fake" - - appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - adpt "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/adapter" -) - -var ( - one int32 = 1 - two int32 = 2 -) - -func TestPoolControl_GetAllPools(t *testing.T) { - - instance := &appsv1alpha1.YurtAppSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "foo-ns", - }, - Spec: appsv1alpha1.YurtAppSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "foo", - }, - }, - WorkloadTemplate: appsv1alpha1.WorkloadTemplate{ - DeploymentTemplate: &appsv1alpha1.DeploymentTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &two, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "container", - Image: "nginx:1.0", - }, - }, - }, - }, - }, - }, - }, - Topology: appsv1alpha1.Topology{ - Pools: []appsv1alpha1.Pool{ - { - Name: "foo-0", - Replicas: &one, - NodeSelectorTerm: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "app.openyurt.io/nodepool", - Operator: corev1.NodeSelectorOpIn, - Values: []string{ - "foo-0", - }, - }, - }, - }, - }, - { - Name: "foo-1", - Replicas: &two, - NodeSelectorTerm: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "app.openyurt.io/nodepool", - Operator: corev1.NodeSelectorOpIn, - Values: []string{ - "foo-1", - }, - }, - }, - }, - }, - }, - }, - }, - } - - scheme := runtime.NewScheme() - if err := appsv1alpha1.AddToScheme(scheme); err != nil { - t.Logf("failed to add yurt custom resource") - return - } - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Logf("failed to add kubernetes clint-go custom resource") - return - } - fc := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance).Build() - pc := PoolControl{ - Client: fc, - scheme: scheme, - adapter: &adpt.DeploymentAdapter{Client: fc, Scheme: scheme}, - } - for i := 0; i < 2; i++ { - tf := pc.CreatePool(instance, "foo-"+strconv.FormatInt(int64(i), 10), "v0.1.0", two) - if tf != nil { - t.Logf("failed create node pool resource") - } - } - pools, err := pc.GetAllPools(instance) - if err != nil && len(pools) != 2 { - t.Logf("failed to get the pools of yurtappset") - } -} - -func TestPoolControl_UpdatePool(t *testing.T) { - - instance := &appsv1alpha1.YurtAppSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "foo-ns", - }, - Spec: appsv1alpha1.YurtAppSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "foo", - }, - }, - WorkloadTemplate: appsv1alpha1.WorkloadTemplate{ - DeploymentTemplate: &appsv1alpha1.DeploymentTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &two, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "container", - Image: "nginx:1.0", - }, - }, - }, - }, - }, - }, - }, - Topology: appsv1alpha1.Topology{ - Pools: []appsv1alpha1.Pool{ - { - Name: "foo", - Replicas: &one, - NodeSelectorTerm: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "app.openyurt.io/nodepool", - Operator: corev1.NodeSelectorOpIn, - Values: []string{ - "foo-0", - }, - }, - }, - }, - }, - }, - }, - }, - } - - scheme := runtime.NewScheme() - if err := appsv1alpha1.AddToScheme(scheme); err != nil { - t.Logf("failed to add yurt custom resource") - return - } - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Logf("failed to add kubernetes clint-go custom resource") - return - } - - fc := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance).Build() - pc := PoolControl{ - Client: fc, - scheme: scheme, - adapter: &adpt.DeploymentAdapter{Client: fc, Scheme: scheme}, - } - tf := pc.CreatePool(instance, "foo", "v0.1.0", two) - if tf != nil { - t.Logf("failed create node pool resource") - } - pools, err := pc.GetAllPools(instance) - if err != nil && len(pools) != 2 { - t.Logf("failed to get the pools of yurtappset") - } - tf = pc.UpdatePool(pools[0], instance, "v2", one) - if tf != nil { - t.Logf("failed update node pool resource") - } -} - -func TestPoolControl_DeletePool(t *testing.T) { - instance := &appsv1alpha1.YurtAppSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "foo-ns", - }, - Spec: appsv1alpha1.YurtAppSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "foo", - }, - }, - WorkloadTemplate: appsv1alpha1.WorkloadTemplate{ - DeploymentTemplate: &appsv1alpha1.DeploymentTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &two, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "container", - Image: "nginx:1.0", - }, - }, - }, - }, - }, - }, - }, - Topology: appsv1alpha1.Topology{ - Pools: []appsv1alpha1.Pool{ - { - Name: "foo", - Replicas: &one, - }, - }, - }, - }, - } - scheme := runtime.NewScheme() - if err := appsv1alpha1.AddToScheme(scheme); err != nil { - t.Logf("failed to add yurt custom resource") - return - } - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Logf("failed to add kubernetes clint-go custom resource") - return - } - - fc := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance).Build() - pc := PoolControl{ - Client: fc, - scheme: scheme, - adapter: &adpt.DeploymentAdapter{Client: fc, Scheme: scheme}, - } - tf := pc.CreatePool(instance, "foo", "v0.1.0", two) - if tf != nil { - t.Logf("failed create node pool resource") - } - pools, err := pc.GetAllPools(instance) - if err != nil && len(pools) != 2 { - t.Logf("failed to get the pools of yurtappset") - } - tf = pc.DeletePool(pools[0]) - if tf != nil { - t.Logf("failed update node pool resource") - } -} diff --git a/pkg/yurtmanager/controller/yurtappset/revision.go b/pkg/yurtmanager/controller/yurtappset/revision.go index 987e72e8231..7fd8a11fcef 100644 --- a/pkg/yurtmanager/controller/yurtappset/revision.go +++ b/pkg/yurtmanager/controller/yurtappset/revision.go @@ -1,7 +1,5 @@ /* -Copyright 2020 The OpenYurt Authors. -Copyright 2019 The Kruise Authors. -Copyright 2017 The Kubernetes Authors. +Copyright 2024 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -27,42 +25,41 @@ import ( apps "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/controller/history" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - appsalphav1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + appsbetav1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/refmanager" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/workloadmanager" ) -// ControllerRevisionHashLabel is the label used to indicate the hash value of a ControllerRevision's Data. -const ControllerRevisionHashLabel = "controller.kubernetes.io/hash" +// get all histories controlled by the YurtAppSet +func controlledHistories(cli client.Client, scheme *runtime.Scheme, yas *appsbetav1.YurtAppSet) ([]*apps.ControllerRevision, error) { -func (r *ReconcileYurtAppSet) controlledHistories(yas *appsalphav1.YurtAppSet) ([]*apps.ControllerRevision, error) { // List all histories to include those that don't match the selector anymore - // but have a ControllerRef pointing to the controller. - selector, err := metav1.LabelSelectorAsSelector(yas.Spec.Selector) - if err != nil { - return nil, err - } histories := &apps.ControllerRevisionList{} - err = r.Client.List(context.TODO(), histories, &client.ListOptions{LabelSelector: selector}) + err := cli.List(context.TODO(), histories, &client.ListOptions{LabelSelector: labels.Everything(), Namespace: yas.GetNamespace()}) if err != nil { return nil, err } - klog.V(1).Infof("List controller revision of YurtAppSet %s/%s: count %d\n", yas.Namespace, yas.Name, len(histories.Items)) // Use ControllerRefManager to adopt/orphan as needed. - cm, err := refmanager.New(r.Client, yas.Spec.Selector, yas, r.scheme) + yasSelector, err := workloadmanager.NewLabelSelectorForYurtAppSet(yas) + if err != nil { + return nil, err + } + cm, err := refmanager.New(cli, yasSelector, yas, scheme) if err != nil { return nil, err } mts := make([]metav1.Object, len(histories.Items)) - for i, history := range histories.Items { - mts[i] = history.DeepCopy() + for i := range histories.Items { + mts[i] = &histories.Items[i] } claims, err := cm.ClaimOwnedObjects(mts) if err != nil { @@ -74,113 +71,115 @@ func (r *ReconcileYurtAppSet) controlledHistories(yas *appsalphav1.YurtAppSet) ( claimHistories[i] = mt.(*apps.ControllerRevision) } + klog.V(4).Infof("List controller revision of YurtAppSet %s/%s: %d\n", yas.Namespace, yas.Name, len(claimHistories)) + return claimHistories, nil } -func (r *ReconcileYurtAppSet) constructYurtAppSetRevisions(yas *appsalphav1.YurtAppSet) (*apps.ControllerRevision, *apps.ControllerRevision, int32, error) { - var currentRevision, updateRevision *apps.ControllerRevision - revisions, err := r.controlledHistories(yas) - if err != nil { - if yas.Status.CollisionCount == nil { - return currentRevision, updateRevision, 0, err - } - return currentRevision, updateRevision, *yas.Status.CollisionCount, err - } +func (r *ReconcileYurtAppSet) constructYurtAppSetRevisions(yas *appsbetav1.YurtAppSet) (allRevisions []*apps.ControllerRevision, updateRevision *apps.ControllerRevision, collisionCount int32, err error) { - history.SortControllerRevisions(revisions) - cleanedRevision, err := r.cleanExpiredRevision(yas, &revisions) + allRevisions, err = controlledHistories(r.Client, r.scheme, yas) if err != nil { if yas.Status.CollisionCount == nil { - return currentRevision, updateRevision, 0, err + return nil, nil, 0, err } - return currentRevision, updateRevision, *yas.Status.CollisionCount, err + return nil, nil, *yas.Status.CollisionCount, err } - revisions = *cleanedRevision // Use a local copy of set.Status.CollisionCount to avoid modifying set.Status directly. // This copy is returned so the value gets carried over to set.Status in updateStatefulSet. - var collisionCount int32 if yas.Status.CollisionCount != nil { collisionCount = *yas.Status.CollisionCount } // create a new revision from the current set - updateRevision, err = r.newRevision(yas, nextRevision(revisions), &collisionCount) + history.SortControllerRevisions(allRevisions) + updateRevision, err = newRevision(yas, nextRevision(allRevisions), &collisionCount, r.scheme) if err != nil { return nil, nil, collisionCount, err } // find any equivalent revisions - equalRevisions := history.FindEqualRevisions(revisions, updateRevision) + equalRevisions := history.FindEqualRevisions(allRevisions, updateRevision) equalCount := len(equalRevisions) - revisionCount := len(revisions) + revisionCount := len(allRevisions) - if equalCount > 0 && history.EqualRevision(revisions[revisionCount-1], equalRevisions[equalCount-1]) { + if equalCount > 0 && history.EqualRevision(allRevisions[revisionCount-1], equalRevisions[equalCount-1]) { // if the equivalent revision is immediately prior the update revision has not changed - updateRevision = revisions[revisionCount-1] + updateRevision = allRevisions[revisionCount-1] } else if equalCount > 0 { - // if the equivalent revision is not immediately prior we will roll back by incrementing the - // Revision of the equivalent revision - equalRevisions[equalCount-1].Revision = updateRevision.Revision - err := r.Client.Update(context.TODO(), equalRevisions[equalCount-1]) - if err != nil { - return nil, nil, collisionCount, err + if isRevisionInvalid(equalRevisions[0]) { + // if equal revision is invalid, just reuse it + updateRevision = equalRevisions[0] + } else { + // if the equivalent revision is valid and not immediately prior, we will roll back by incrementing the + // Revision of the equivalent revision + equalRevisions[equalCount-1].Revision = updateRevision.Revision + err := r.Client.Update(context.TODO(), equalRevisions[equalCount-1]) + if err != nil { + return nil, nil, collisionCount, err + } + updateRevision = equalRevisions[equalCount-1] } - updateRevision = equalRevisions[equalCount-1] } else { //if there is no equivalent revision we create a new one - updateRevision, err = r.createControllerRevision(yas, updateRevision, &collisionCount) + updateRevision, err = createControllerRevision(r.Client, yas, updateRevision, &collisionCount) if err != nil { return nil, nil, collisionCount, err } + allRevisions = append(allRevisions, updateRevision) } - // attempt to find the revision that corresponds to the current revision - for i := range revisions { - if revisions[i].Name == yas.Status.CurrentRevision { - currentRevision = revisions[i] - } - } - - // if the current revision is nil we initialize the history by setting it to the update revision - if currentRevision == nil { - currentRevision = updateRevision - } - - return currentRevision, updateRevision, collisionCount, nil + klog.V(4).Infof("YurtAppSet [%s/%s] get expectRevision %s collisionCount %v", yas.GetNamespace(), yas.GetName(), updateRevision.Name, collisionCount) + return allRevisions, updateRevision, collisionCount, nil } -func (r *ReconcileYurtAppSet) cleanExpiredRevision(yas *appsalphav1.YurtAppSet, - sortedRevisions *[]*apps.ControllerRevision) (*[]*apps.ControllerRevision, error) { +// clean expired and invalid revisions +func cleanRevisions(cli client.Client, yas *appsbetav1.YurtAppSet, revisions []*apps.ControllerRevision) error { - exceedNum := len(*sortedRevisions) - int(*yas.Spec.RevisionHistoryLimit) - if exceedNum <= 0 { - return sortedRevisions, nil - } - - live := map[string]bool{} - live[yas.Status.CurrentRevision] = true - - for i, revision := range *sortedRevisions { - if _, exist := live[revision.Name]; exist { - continue + // clean invalid revisions + validRevisions := make([]*apps.ControllerRevision, 0) + for _, revision := range revisions { + if isRevisionInvalid(revision) { + if err := cli.Delete(context.TODO(), revision); err != nil { + klog.Errorf("YurtAppSet [%s/%s] delete invalid revision %s error: %v") + return err + } + klog.Infof("YurtAppSet [%s/%s] delete invalid revision %s", yas.GetNamespace(), yas.GetName(), revision.Name) + } else { + validRevisions = append(validRevisions, revision) } + } - if i >= exceedNum { - break - } + // clean expired revisions + var revisionLimit int + if yas.Spec.RevisionHistoryLimit != nil { + revisionLimit = int(*(yas.Spec.RevisionHistoryLimit)) + } else { + klog.Warningf("YurtAppSet [%s/%s] revisionHistoryLimit is nil, default to 10", yas.Namespace, yas.Name) + revisionLimit = 10 + } - if err := r.Client.Delete(context.TODO(), revision); err != nil { - return sortedRevisions, err + if len(validRevisions) > revisionLimit { + klog.V(4).Info("YurtAppSet [%s/%s] clean expired revisions", yas.GetNamespace(), yas.GetName()) + for i := 0; i < len(validRevisions)-revisionLimit; i++ { + if validRevisions[i].GetName() == yas.Status.CurrentRevision { + klog.Warningf("YurtAppSet [%s/%s] current revision %s is expired, skip") + continue + } + if err := cli.Delete(context.TODO(), validRevisions[i]); err != nil { + klog.Errorf("YurtAppSet [%s/%s] delete expired revision %s error: %v") + return err + } + klog.Infof("YurtAppSet [%s/%s] delete expired revision %s", yas.GetNamespace(), yas.GetName(), validRevisions[i].Name) } } - cleanedRevisions := (*sortedRevisions)[exceedNum:] - return &cleanedRevisions, nil + return nil } -// createControllerRevision creates the controller revision owned by the parent. -func (r *ReconcileYurtAppSet) createControllerRevision(parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) { +// createControllerRevision creates the revision owned by the parent and update the collisionCount +func createControllerRevision(cli client.Client, parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) { if collisionCount == nil { return nil, fmt.Errorf("collisionCount should not be nil") } @@ -194,19 +193,24 @@ func (r *ReconcileYurtAppSet) createControllerRevision(parent metav1.Object, rev hash := history.HashControllerRevision(revision, collisionCount) // Update the revisions name clone.Name = history.ControllerRevisionName(parent.GetName(), hash) - err = r.Client.Create(context.TODO(), clone) + err = cli.Create(context.TODO(), clone) if errors.IsAlreadyExists(err) { + klog.V(4).Infof("YurtAppSet [%s/%s] createControllerRevision %s error: name already exist", parent.GetNamespace(), parent.GetName(), clone.GetName()) exists := &apps.ControllerRevision{} - err := r.Client.Get(context.TODO(), client.ObjectKey{Namespace: parent.GetNamespace(), Name: clone.Name}, exists) + err := cli.Get(context.TODO(), client.ObjectKey{Namespace: parent.GetNamespace(), Name: clone.Name}, exists) if err != nil { + klog.V(4).Infof("YurtAppSet [%s/%s] createControllerRevision %s: get failed: %v ", parent.GetNamespace(), parent.GetName(), clone.GetName(), err) return nil, err } if bytes.Equal(exists.Data.Raw, clone.Data.Raw) { + klog.V(4).Infof("YurtAppSet [%s/%s] createControllerRevision %s: contents are the same with cr already exists", parent.GetNamespace(), parent.GetName(), clone.GetName()) return exists, nil } + klog.Info("YurtAppSet [%s/%s] createControllerRevision collision exists, collision count increased %d->%d", parent.GetNamespace(), parent.GetName(), *collisionCount, *collisionCount+1) *collisionCount++ continue } + klog.Infof("YurtAppSet [%s/%s] createControllerRevision %s success", parent.GetNamespace(), parent.GetName(), clone.GetName()) return clone, err } } @@ -215,26 +219,21 @@ func (r *ReconcileYurtAppSet) createControllerRevision(parent metav1.Object, rev // The Revision of the returned ControllerRevision is set to revision. If the returned error is nil, the returned // ControllerRevision is valid. StatefulSet revisions are stored as patches that re-apply the current state of set // to a new StatefulSet using a strategic merge patch to replace the saved state of the new StatefulSet. -func (r *ReconcileYurtAppSet) newRevision(yas *appsalphav1.YurtAppSet, revision int64, collisionCount *int32) (*apps.ControllerRevision, error) { +func newRevision(yas *appsbetav1.YurtAppSet, revision int64, collisionCount *int32, scheme *runtime.Scheme) (*apps.ControllerRevision, error) { patch, err := getYurtAppSetPatch(yas) if err != nil { return nil, err } - gvk, err := apiutil.GVKForObject(yas, r.scheme) + gvk, err := apiutil.GVKForObject(yas, scheme) if err != nil { return nil, err } - var selectedLabels map[string]string - switch { - case yas.Spec.WorkloadTemplate.StatefulSetTemplate != nil: - selectedLabels = yas.Spec.WorkloadTemplate.StatefulSetTemplate.Labels - case yas.Spec.WorkloadTemplate.DeploymentTemplate != nil: - selectedLabels = yas.Spec.WorkloadTemplate.DeploymentTemplate.Labels - default: - klog.Errorf("YurtAppSet(%s/%s) need specific WorkloadTemplate", yas.GetNamespace(), yas.GetName()) - return nil, fmt.Errorf("YurtAppSet(%s/%s) need specific WorkloadTemplate", yas.GetNamespace(), yas.GetName()) + selectedLabels := yas.GetLabels() + labelSelector, err := workloadmanager.NewLabelSelectorForYurtAppSet(yas) + if err == nil { + selectedLabels = workloadmanager.CombineMaps(selectedLabels, labelSelector.MatchLabels) } cr, err := history.NewControllerRevision(yas, @@ -262,7 +261,13 @@ func nextRevision(revisions []*apps.ControllerRevision) int64 { return revisions[count-1].Revision + 1 } -func getYurtAppSetPatch(yas *appsalphav1.YurtAppSet) ([]byte, error) { +// getYurtAppSetPatch creates a patch of the YurtAppSet that replaces spec.template +// it only contains spec.workload, which means only workload field change will be recorded in this patch +func getYurtAppSetPatch(yas *appsbetav1.YurtAppSet) ([]byte, error) { + if yas == nil { + return nil, fmt.Errorf("yurtAppSet is nil") + } + dsBytes, err := json.Marshal(yas) if err != nil { return nil, err @@ -277,10 +282,25 @@ func getYurtAppSetPatch(yas *appsalphav1.YurtAppSet) ([]byte, error) { // Create a patch of the YurtAppSet that replaces spec.template spec := raw["spec"].(map[string]interface{}) - template := spec["workloadTemplate"].(map[string]interface{}) - specCopy["workloadTemplate"] = template - template["$patch"] = "replace" + + if template, ok := spec["workload"].(map[string]interface{}); ok { + template["$patch"] = "replace" + specCopy["workload"] = template + } + objCopy["spec"] = specCopy patch, err := json.Marshal(objCopy) return patch, err } + +func isRevisionInvalid(revision *apps.ControllerRevision) bool { + // set Revision to 0 to indicate invalid, because the revision number is increased from 1 + return revision.Revision == 0 +} + +func setRevisionInvalid(cli client.Client, revision *apps.ControllerRevision) { + revision.Revision = 0 + if err := cli.Update(context.TODO(), revision); err != nil { + klog.Warningf("YurtAppSet [%s/%s] set revision invalid error: %v", revision.Namespace, revision.Name, err) + } +} diff --git a/pkg/yurtmanager/controller/yurtappset/revision_test.go b/pkg/yurtmanager/controller/yurtappset/revision_test.go index 2e056a0aa14..e3be16498f6 100644 --- a/pkg/yurtmanager/controller/yurtappset/revision_test.go +++ b/pkg/yurtmanager/controller/yurtappset/revision_test.go @@ -1,7 +1,5 @@ /* -Copyright 2020 The OpenYurt Authors. -Copyright 2019 The Kruise Authors. -Copyright 2017 The Kubernetes Authors. +Copyright 2024 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,411 +17,543 @@ limitations under the License. package yurtappset import ( + "reflect" "testing" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" + "github.com/stretchr/testify/assert" + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - fakeclint "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/apis" + yurtapps "github.com/openyurtio/openyurt/pkg/apis/apps" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" + beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" ) -func TestReconcileyurtAppSet_ControlledHistories(t *testing.T) { +const ( + failed = "\u2717" + succeed = "\u2713" +) - instance := struct { - yas *appsv1alpha1.YurtAppSet - ctr []*appsv1.ControllerRevision +func TestNextRevision(t *testing.T) { + tests := []struct { + name string + revisions []*apps.ControllerRevision + expect int64 }{ - yas: &appsv1alpha1.YurtAppSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "default", - }, - Spec: appsv1alpha1.YurtAppSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "name": "foo", - }, + { + "zero", + []*apps.ControllerRevision{}, + 1, + }, + { + "normal", + []*apps.ControllerRevision{ + { + Revision: 1, }, - WorkloadTemplate: appsv1alpha1.WorkloadTemplate{ - DeploymentTemplate: &appsv1alpha1.DeploymentTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "name": "foo", + }, + 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + get := nextRevision(tt.revisions) + + if !reflect.DeepEqual(get, tt.expect) { + t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) + } + t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) + }) + } +} + +func TestGetYurtAppSetPatch(t *testing.T) { + + type test struct { + name string + yas *beta1.YurtAppSet + err bool + } + + var tests = []test{ + { + name: "Yas is nil", + yas: nil, + err: true, + }, + { + name: "No 'spec' field in original data", + yas: &beta1.YurtAppSet{}, + err: false, + }, + { + name: "No 'spec.workloadTemplate' & 'spec.workloadTweaks' field in original data", + yas: &beta1.YurtAppSet{ + Spec: beta1.YurtAppSetSpec{}, + }, + err: false, + }, + { + name: "No 'spec.workloadTemplate' field in original data", + yas: &beta1.YurtAppSet{ + Spec: beta1.YurtAppSetSpec{ + Workload: beta1.Workload{ + WorkloadTweaks: []beta1.WorkloadTweak{ + { + Pools: []string{"pool1"}, }, }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ + }, + }, + }, + err: false, + }, + { + name: "No 'spec.workloadTweaks' field in original data", + yas: &beta1.YurtAppSet{ + Spec: beta1.YurtAppSetSpec{ + Workload: beta1.Workload{ + WorkloadTemplate: beta1.WorkloadTemplate{ + DeploymentTemplate: &beta1.DeploymentTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "name": "foo", + "a": "a", }, }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "container-a", - Image: "nginx:1.0", + Spec: apps.DeploymentSpec{ + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{}, + }, + }, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + yurtapps.PoolNameLabelKey: "a", }, }, }, }, - }, - }, + }}, }, - Topology: appsv1alpha1.Topology{ - Pools: []appsv1alpha1.Pool{ - { - Name: "foo-0", - Replicas: &one, - NodeSelectorTerm: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "app.openyurt.io/nodepool", - Operator: corev1.NodeSelectorOpIn, - Values: []string{ - "foo-0", - }, + }, + err: false, + }, + { + name: "All fields exist", + yas: &beta1.YurtAppSet{ + Spec: beta1.YurtAppSetSpec{ + Workload: beta1.Workload{ + WorkloadTemplate: beta1.WorkloadTemplate{ + DeploymentTemplate: &beta1.DeploymentTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "a": "a", }, }, - }, - }, - { - Name: "foo-1", - Replicas: &two, - NodeSelectorTerm: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "app.openyurt.io/nodepool", - Operator: corev1.NodeSelectorOpIn, - Values: []string{ - "foo-1", + Spec: apps.DeploymentSpec{ + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{}, + }, + }, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + yurtapps.PoolNameLabelKey: "a", }, }, }, }, }, + WorkloadTweaks: []beta1.WorkloadTweak{ + { + Pools: []string{"pool1"}, + }, + }, }, }, - RevisionHistoryLimit: &two, }, + err: false, }, } - scheme := runtime.NewScheme() - if err := appsv1alpha1.AddToScheme(scheme); err != nil { - t.Logf("failed to add yurt custom resource") - return - } - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Logf("failed to add kubernetes clint-go custom resource") - return - } - if err := appsv1.AddToScheme(scheme); err != nil { - t.Logf("failed to add appsv1 custom resource") - return - } - - fc := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance.yas).Build() - ryas := ReconcileYurtAppSet{ - Client: fc, - scheme: scheme, - } - histories, err := ryas.controlledHistories(instance.yas) - if err != nil || len(histories) != 0 { - t.Logf("failed to get controlled histories") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := getYurtAppSetPatch(tt.yas) + if (err != nil) != tt.err { + t.Errorf("getYurtAppSetPatch() error = %v, wantErr %v", err, tt.err) + return + } + }) } +} +func getFakeScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + apis.AddToScheme(scheme) + apps.AddToScheme(scheme) + return scheme } -func TestReconcileYurtAppSet_CreateControllerRevision(t *testing.T) { - instance := struct { - yas *appsv1alpha1.YurtAppSet - ctr *appsv1.ControllerRevision +var fakeScheme = getFakeScheme() + +func TestNewRevision(t *testing.T) { + + type args struct { + yas *beta1.YurtAppSet + revision int64 + collision *int32 + expectedErr bool + } + tests := []struct { + name string + args args }{ - yas: &appsv1alpha1.YurtAppSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "default", - }, - Spec: appsv1alpha1.YurtAppSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "name": "foo", - }, - }, - WorkloadTemplate: appsv1alpha1.WorkloadTemplate{ - DeploymentTemplate: &appsv1alpha1.DeploymentTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "container-a", - Image: "nginx:1.0", - }, - }, - }, - }, - }, - }, - }, - Topology: appsv1alpha1.Topology{ - Pools: []appsv1alpha1.Pool{ - { - Name: "pool-a", - NodeSelectorTerm: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "node-name", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"nodeA"}, - }, - }, - }, + { + name: "Valid YurtAppSet", + args: args{ + yas: &beta1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset", + Namespace: "default", + Labels: map[string]string{ + "a": "a", }, }, + Spec: beta1.YurtAppSetSpec{}, }, - RevisionHistoryLimit: &two, + revision: 1, + collision: nil, + expectedErr: false, }, }, - ctr: &appsv1.ControllerRevision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-0", - Namespace: "foo-ns", + { + name: "Error in getting YurtAppSet patch", + args: args{ + yas: nil, + revision: 1, + collision: nil, + expectedErr: true, }, - Revision: 1, }, } - scheme := runtime.NewScheme() - if err := appsv1alpha1.AddToScheme(scheme); err != nil { - t.Logf("failed to add yurt custom resource") - return + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cr, err := newRevision(tt.args.yas, tt.args.revision, tt.args.collision, fakeScheme) + if tt.args.expectedErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Subset(t, cr.Labels, tt.args.yas.Labels) + } + }) } - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Logf("failed to add kubernetes clint-go custom resource") - return +} + +var ( + yas = beta1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset", + Namespace: "default", + }, } - if err := appsv1.AddToScheme(scheme); err != nil { - t.Logf("failed to add appsv1 custom resource") - return + + cr1 = apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset-", + Namespace: "default", + Labels: map[string]string{ + yurtapps.YurtAppSetOwnerLabelKey: "test-yurtappset", + }, + }, + Data: runtime.RawExtension{ + Raw: []byte(`{"a":"a"}`), + }, + Revision: 2, } - fc := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance.yas).Build() - ryas := ReconcileYurtAppSet{ - Client: fc, - scheme: scheme, + cr2 = apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset-7fcd4f8557", + Namespace: "default", + }, + Data: runtime.RawExtension{ + Raw: []byte(`{"a":"a"}`), + }, + Revision: 0, } - o, err := ryas.createControllerRevision(instance.yas, instance.ctr, &two) - if err != nil && o.Revision != 1 { - t.Logf("failed to create controller revision") + + cr3 = apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset-7fcd4f8557", + Namespace: "default", + }, + Data: runtime.RawExtension{ + Raw: []byte(`{"a":"b"}`), + }, } -} -func TestReconcileYurtAppSet_newRevision(t *testing.T) { - instance := &appsv1alpha1.YurtAppSet{ + cr4 = apps.ControllerRevision{ ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "foo-ns", + Name: "test-yurtappset-694bbcc68", + Namespace: "default", + Labels: map[string]string{ + yurtapps.YurtAppSetOwnerLabelKey: "test-yurtappset", + }, }, - Spec: appsv1alpha1.YurtAppSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "foo", + Data: runtime.RawExtension{ + Raw: []byte("{\"spec\":{\"workload\":{\"$patch\":\"replace\",\"workloadTemplate\":{}}}}"), + }, + Revision: 1, + } + + cr5 = apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset-694bbcc68", + Namespace: "default", + Labels: map[string]string{ + yurtapps.YurtAppSetOwnerLabelKey: "test-yurtappset", + }, + }, + Data: runtime.RawExtension{ + Raw: []byte("{\"spec\":{\"workload\":{\"$patch\":\"replace\",\"workloadTemplate\":{}}}}"), + }, + Revision: 0, + } +) + +func TestCreateControllerRevision(t *testing.T) { + + collisionCount := int32(0) + + tests := []struct { + name string + cli client.Client + yas *beta1.YurtAppSet + revision *apps.ControllerRevision + collision *int32 + err bool + }{ + { + name: "CollisionCount is nil", + yas: &beta1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset", + Namespace: "default", }, }, - WorkloadTemplate: appsv1alpha1.WorkloadTemplate{ - StatefulSetTemplate: &appsv1alpha1.StatefulSetTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: appsv1.StatefulSetSpec{ - Replicas: &two, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "nginx", - Image: "nginx:1.19", - }, - }, - }, - }, - }, + revision: nil, + collision: nil, + err: true, + }, + { + name: "create success without error", + yas: &beta1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset", + Namespace: "default", }, }, - Topology: appsv1alpha1.Topology{ - Pools: []appsv1alpha1.Pool{ - { - Name: "foo-0", - Replicas: &one, - NodeSelectorTerm: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "app.openyurt.io/nodepool", - Operator: corev1.NodeSelectorOpIn, - Values: []string{ - "foo-0", - }, - }, - }, - }, - }, - { - Name: "foo-1", - Replicas: &two, - NodeSelectorTerm: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "app.openyurt.io/nodepool", - Operator: corev1.NodeSelectorOpIn, - Values: []string{ - "foo-1", - }, - }, - }, - }, - }, + revision: cr1.DeepCopy(), + collision: &collisionCount, + err: false, + cli: fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects().Build(), + }, + { + name: "create success with already exist", + yas: &beta1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset", + Namespace: "default", }, }, + revision: cr1.DeepCopy(), + collision: &collisionCount, + err: false, + cli: fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(cr2.DeepCopy()).Build(), + }, + { + name: "create success with already exist and collison occurs", + yas: &beta1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset", + Namespace: "default", + }, + }, + revision: cr1.DeepCopy(), + collision: &collisionCount, + err: false, + cli: fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(cr3.DeepCopy()).Build(), }, - } - scheme := runtime.NewScheme() - if err := appsv1alpha1.AddToScheme(scheme); err != nil { - t.Logf("failed to add yurt custom resource") - return - } - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Logf("failed to add kubernetes clint-go custom resource") - return - } - if err := appsv1.AddToScheme(scheme); err != nil { - t.Logf("failed to add appsv1 custom resource") - return } - fc := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects().Build() - ryas := ReconcileYurtAppSet{ - Client: fc, - scheme: scheme, - } - cr, err := ryas.newRevision(instance, 2, &two) - if err != nil && cr.Namespace != instance.Namespace { - t.Logf("failed to new revision for yurtappset") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := createControllerRevision(tt.cli, tt.yas, tt.revision, tt.collision) + if !tt.err { + assert.NoError(t, err) + } + }) } + } -func TestReconcileYurtAppSet_ConstructYurtAppSetRevisions(t *testing.T) { - instance := struct { - yas *appsv1alpha1.YurtAppSet - ctr *appsv1.ControllerRevision +func TestCleanRevisions(t *testing.T) { + + itemRevisionHistoryLimit := int32(0) + + tests := []struct { + name string + cli client.Client + yas *beta1.YurtAppSet + revisions []*apps.ControllerRevision + err bool }{ - yas: &appsv1alpha1.YurtAppSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "foo-ns", - }, - Spec: appsv1alpha1.YurtAppSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "name": "foo", - }, - }, - WorkloadTemplate: appsv1alpha1.WorkloadTemplate{ - DeploymentTemplate: &appsv1alpha1.DeploymentTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "container-a", - Image: "nginx:1.0", - }, - }, - }, - }, - }, - }, + { + name: "clean success and clear success", + yas: &beta1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset", + Namespace: "default", }, - Topology: appsv1alpha1.Topology{ - Pools: []appsv1alpha1.Pool{ - { - Name: "pool-a", - NodeSelectorTerm: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "node-name", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"nodeA"}, - }, - }, - }, - }, - }, + Spec: v1beta1.YurtAppSetSpec{ + RevisionHistoryLimit: &itemRevisionHistoryLimit, }, - RevisionHistoryLimit: &two, }, + revisions: []*apps.ControllerRevision{ + cr1.DeepCopy(), cr2.DeepCopy(), + }, + err: false, + cli: fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects( + cr1.DeepCopy(), + cr2.DeepCopy(), + ).Build(), }, - ctr: &appsv1.ControllerRevision{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-0", - Namespace: "foo-ns", + { + name: "clean success with yas revisionHistoryLimit is nil", + yas: &beta1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset", + Namespace: "default", + }, + }, + revisions: []*apps.ControllerRevision{ + cr1.DeepCopy(), cr2.DeepCopy(), }, - Revision: 1, + err: false, + cli: fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects( + cr1.DeepCopy(), + cr2.DeepCopy(), + ).Build(), }, } - scheme := runtime.NewScheme() - if err := appsv1alpha1.AddToScheme(scheme); err != nil { - t.Logf("failed to add yurt custom resource") - return + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := cleanRevisions(tt.cli, tt.yas, tt.revisions) + if !tt.err { + assert.NoError(t, err) + } + }) } - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Logf("failed to add kubernetes clint-go custom resource") - return +} + +func TestControlledHistories(t *testing.T) { + // cr_should_adopt is a cr has owner label but doesnot has owner reference + cr_should_adopt := apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset-1", + Namespace: "default", + Labels: map[string]string{yurtapps.YurtAppSetOwnerLabelKey: "test-yurtappset"}, + }, + Data: runtime.RawExtension{ + Raw: []byte(`{"a":"a"}`), + }, } - if err := appsv1.AddToScheme(scheme); err != nil { - t.Logf("failed to add appsv1 custom resource") - return + // cr_should_release is a cr not has owner label but has owner reference + cr_should_release := apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset-2", + Namespace: "default", + }, + Data: runtime.RawExtension{ + Raw: []byte(`{"a":"a"}`), + }, + Revision: 0, } + controllerutil.SetControllerReference(&yas, &cr_should_release, fakeScheme) + + crs, err := controlledHistories(fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects( + &cr_should_adopt, + &cr_should_release, + &yas, + ).Build(), fakeScheme, &yas) + assert.NoError(t, err) - fc := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance.yas).Build() - ryas := ReconcileYurtAppSet{ - Client: fc, - scheme: scheme, + assert.Len(t, crs, 1) + assert.Equal(t, cr_should_adopt.Name, crs[0].Name) +} + +func TestConstructYurtAppSetRevisions(t *testing.T) { + tests := []struct { + name string + yas *beta1.YurtAppSet + cli client.Client + err bool + }{ + { + name: "construct success and create new revision", + yas: &yas, + cli: fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(cr1.DeepCopy(), cr2.DeepCopy(), &yas).Build(), + err: false, + }, + { + name: "construct success and update old revision", + yas: &yas, + cli: fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(cr1.DeepCopy(), cr4.DeepCopy(), &yas).Build(), + err: false, + }, + { + name: "construct success and reuse invalid revision", + yas: &yas, + cli: fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(cr1.DeepCopy(), cr5.DeepCopy(), &yas).Build(), + err: false, + }, + { + name: "construct success and no need to update old revision", + yas: &yas, + cli: fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(cr2.DeepCopy(), cr4.DeepCopy(), &yas).Build(), + err: false, + }, } - _, _, _, err := ryas.constructYurtAppSetRevisions(instance.yas) - if err != nil { - t.Logf("failed to construct yurtappset revisions") + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testReconcile := &ReconcileYurtAppSet{ + Client: tt.cli, + scheme: fakeScheme, + } + _, _, _, err := testReconcile.constructYurtAppSetRevisions(tt.yas) + if !tt.err { + assert.NoError(t, err) + } + }) } } diff --git a/pkg/yurtmanager/controller/yurtappset/utils.go b/pkg/yurtmanager/controller/yurtappset/utils.go new file mode 100644 index 00000000000..a9515e62dee --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappset/utils.go @@ -0,0 +1,70 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yurtappset + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + + unitv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" +) + +// NewYurtAppSetCondition creates a new YurtAppSet condition. +func NewYurtAppSetCondition(condType unitv1beta1.YurtAppSetConditionType, status corev1.ConditionStatus, reason, message string) *unitv1beta1.YurtAppSetCondition { + return &unitv1beta1.YurtAppSetCondition{ + Type: condType, + Status: status, + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: message, + } +} + +// SetYurtAppSetCondition updates the YurtAppSet to include the provided condition. If the condition that +// we are about to add already exists and has the same status, reason then we are not going to update. +func SetYurtAppSetCondition(status *unitv1beta1.YurtAppSetStatus, condition *unitv1beta1.YurtAppSetCondition) { + originalCondition, newConditions := filterOutCondition(status.Conditions, condition.Type) + if originalCondition != nil && + originalCondition.Type == condition.Type && + originalCondition.Status == condition.Status && + originalCondition.Reason == condition.Reason && + originalCondition.Message == condition.Message { + klog.V(5).Infof("Not updating condition %s status to %s because it is already set to %s", condition.Type, condition.Status, originalCondition.Status) + } else { + klog.V(4).Infof("Updating condition %s status to %s: %s", condition.Type, condition.Status, condition.Message) + status.Conditions = append(newConditions, *condition) + } +} + +// RemoveYurtAppSetCondition removes the YurtAppSet condition with the provided type. +func RemoveYurtAppSetCondition(status *unitv1beta1.YurtAppSetStatus, condType unitv1beta1.YurtAppSetConditionType) { + _, status.Conditions = filterOutCondition(status.Conditions, condType) +} + +// filterOutCondition returns a tuple containing the first matching condition and a new slice of conditions without conditions with the provided type +func filterOutCondition(conditions []unitv1beta1.YurtAppSetCondition, condType unitv1beta1.YurtAppSetConditionType) (outCondition *unitv1beta1.YurtAppSetCondition, newConditions []unitv1beta1.YurtAppSetCondition) { + newConditions = []unitv1beta1.YurtAppSetCondition{} + for i, c := range conditions { + if c.Type == condType { + outCondition = &conditions[i] + } else { + newConditions = append(newConditions, c) + } + } + return outCondition, newConditions +} diff --git a/pkg/yurtmanager/controller/yurtappset/utils_test.go b/pkg/yurtmanager/controller/yurtappset/utils_test.go new file mode 100644 index 00000000000..564484b33c8 --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappset/utils_test.go @@ -0,0 +1,84 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yurtappset + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + unitv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" +) + +func TestFilterOutCondition(t *testing.T) { + // 测试用例1: 空条件列表 + conditions := []unitv1beta1.YurtAppSetCondition{} + expectedNewConditions := []unitv1beta1.YurtAppSetCondition{} + outCondition, actualConditions := filterOutCondition(conditions, unitv1beta1.AppSetAppDispatchced) + assert.Equal(t, expectedNewConditions, actualConditions) + assert.Nil(t, outCondition) + + // 测试用例2: 条件列表中没有指定类型 + conditions = []unitv1beta1.YurtAppSetCondition{ + {Type: unitv1beta1.AppSetPoolFound}, + {Type: unitv1beta1.AppSetAppDispatchced}, + } + expectedNewConditions = []unitv1beta1.YurtAppSetCondition{ + {Type: unitv1beta1.AppSetPoolFound}, + {Type: unitv1beta1.AppSetAppDispatchced}, + } + outCondition, actualConditions = filterOutCondition(conditions, unitv1beta1.AppSetAppDeleted) + assert.Equal(t, expectedNewConditions, actualConditions) + assert.Nil(t, outCondition) + + // 测试用例3: 条件列表中有指定类型 + matchedCondition := &unitv1beta1.YurtAppSetCondition{Type: unitv1beta1.AppSetAppDeleted} + conditions = []unitv1beta1.YurtAppSetCondition{ + {Type: unitv1beta1.AppSetPoolFound}, + *matchedCondition, + {Type: unitv1beta1.AppSetAppDispatchced}, + } + expectedNewConditions = []unitv1beta1.YurtAppSetCondition{ + {Type: unitv1beta1.AppSetPoolFound}, + {Type: unitv1beta1.AppSetAppDispatchced}, + } + outCondition, actualConditions = filterOutCondition(conditions, unitv1beta1.AppSetAppDeleted) + assert.Equal(t, expectedNewConditions, actualConditions) + assert.Equal(t, outCondition, matchedCondition) +} + +func TestSetYurtAppSetCondition(t *testing.T) { + // 测试用例1: 当条件已经存在且状态和原因相同，则更新 + var status unitv1beta1.YurtAppSetStatus = unitv1beta1.YurtAppSetStatus{ + Conditions: []unitv1beta1.YurtAppSetCondition{ + {Type: unitv1beta1.AppSetPoolFound, Status: "True", Reason: "TestReason1", Message: "TestMessage2"}, + }, + } + condition1 := NewYurtAppSetCondition(unitv1beta1.AppSetPoolFound, "True", "TestReason1", "TestMessage1") + SetYurtAppSetCondition(&status, condition1) + assert.Equal(t, status.Conditions[0].Message, "TestMessage1") + + // 测试用例2: 当条件不存在则更新 + condition2 := NewYurtAppSetCondition(unitv1beta1.AppSetAppDeleted, "False", "TestReason2", "TestMessage2") + SetYurtAppSetCondition(&status, condition2) + assert.Equal(t, len(status.Conditions), 2) + + // 测试用例3: 当条件已经存在且状态和原因不同，则更新 + condition3 := NewYurtAppSetCondition(unitv1beta1.AppSetAppDeleted, "True", "TestReason3", "TestMessage3") + SetYurtAppSetCondition(&status, condition3) + assert.Equal(t, string(status.Conditions[1].Status), "True") +} diff --git a/pkg/yurtmanager/controller/yurtappset/workloadmanager/deployment_manager.go b/pkg/yurtmanager/controller/yurtappset/workloadmanager/deployment_manager.go new file mode 100644 index 00000000000..89f88841d2a --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappset/workloadmanager/deployment_manager.go @@ -0,0 +1,186 @@ +/* +Copyright 2024 The Openyurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workloadmanager + +import ( + "context" + "errors" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/openyurtio/openyurt/pkg/apis/apps" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/refmanager" +) + +const updateRetries = 5 + +type DeploymentManager struct { + client.Client + Scheme *runtime.Scheme +} + +func (d *DeploymentManager) GetTemplateType() TemplateType { + return DeploymentTemplateType +} + +func (d *DeploymentManager) Delete(yas *v1beta1.YurtAppSet, workload metav1.Object) error { + klog.V(4).Infof("YurtAppSet[%s/%s] prepare delete [Deployment/%s/%s]", yas.GetNamespace(), + yas.GetName(), workload.GetNamespace(), workload.GetName()) + + workloadObj, ok := workload.(client.Object) + if !ok { + return errors.New("could not convert metav1.Object to client.Object") + } + return d.Client.Delete(context.TODO(), workloadObj, client.PropagationPolicy(metav1.DeletePropagationBackground)) +} + +// ApplyTemplate updates the object to the latest revision, depending on the YurtAppSet. +func (d *DeploymentManager) applyTemplate(yas *v1beta1.YurtAppSet, nodepoolName, revision string, workload *appsv1.Deployment) error { + + deployTemplate := yas.Spec.Workload.WorkloadTemplate.DeploymentTemplate + if deployTemplate == nil { + return errors.New("no deployment template in workloadTemplate") + } + + // deployment meta data + workload.Labels = CombineMaps(workload.Labels, deployTemplate.Labels, map[string]string{ + apps.PoolNameLabelKey: nodepoolName, + apps.ControllerRevisionHashLabelKey: revision, + apps.YurtAppSetOwnerLabelKey: yas.Name, + }) + workload.Annotations = CombineMaps(workload.Annotations, deployTemplate.Annotations, map[string]string{ + apps.AnnotationRefNodePool: nodepoolName, + }) + + workload.Namespace = yas.GetNamespace() + workload.GenerateName = getWorkloadPrefix(yas.GetName(), nodepoolName) + if err := controllerutil.SetControllerReference(yas, workload, d.Scheme); err != nil { + return err + } + + // deployment spec data + workload.Spec = *deployTemplate.Spec.DeepCopy() + if workload.Spec.Selector == nil { + // if no selector, create one + // add this check, because we have no yas webhook check to ensure deployment template is valid + workload.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: make(map[string]string, 0), + } + } + workload.Spec.Selector.MatchLabels[apps.PoolNameLabelKey] = nodepoolName + workload.Spec.Template.Labels = CombineMaps(workload.Spec.Template.Labels, map[string]string{ + apps.PoolNameLabelKey: nodepoolName, + apps.ControllerRevisionHashLabelKey: revision, + }) + workload.Spec.Template.Spec.NodeSelector = CombineMaps(workload.Spec.Template.Spec.NodeSelector, CreateNodeSelectorByNodepoolName(nodepoolName)) + + // apply tweaks + tweaks, err := GetNodePoolTweaksFromYurtAppSet(d.Client, nodepoolName, yas) + if err != nil { + return err + } + + if err = ApplyTweaksToDeployment(workload, tweaks); err != nil { + return err + } + + return nil +} + +func (d *DeploymentManager) Update(yas *v1beta1.YurtAppSet, workload metav1.Object, nodepoolName, revision string) error { + klog.V(4).Infof("YurtAppSet[%s/%s] prepare update [Deployment/%s/%s]", yas.GetNamespace(), + yas.GetName(), workload.GetNamespace(), workload.GetName()) + + if nodepoolName == "" { + klog.Warningf("Deployment[%s/%s] to be updated's nodepool name is empty.", workload.GetNamespace(), workload.GetName()) + } + + deploy := &appsv1.Deployment{} + var updateError error + for i := 0; i < updateRetries; i++ { + getError := d.Client.Get(context.TODO(), types.NamespacedName{Namespace: workload.GetNamespace(), Name: workload.GetName()}, deploy) + if getError != nil { + return getError + } + + if err := d.applyTemplate(yas, nodepoolName, revision, deploy); err != nil { + return err + } + updateError = d.Client.Update(context.TODO(), deploy) + if updateError == nil { + break + } + klog.V(4).Info("update deployment failed, retry") + } + + return updateError +} + +func (d *DeploymentManager) Create(yas *v1beta1.YurtAppSet, nodepoolName, revision string) error { + klog.V(4).Infof("YurtAppSet[%s/%s] prepare create new deployment for nodepool %s ", yas.GetNamespace(), yas.GetName(), nodepoolName) + + deploy := appsv1.Deployment{} + if err := d.applyTemplate(yas, nodepoolName, revision, &deploy); err != nil { + klog.Errorf("YurtAppSet[%s/%s] could not apply template, when create deployment: %v", yas.GetNamespace(), + yas.GetName(), err) + return err + } + return d.Client.Create(context.TODO(), &deploy) +} + +func (d *DeploymentManager) List(yas *v1beta1.YurtAppSet) ([]metav1.Object, error) { + + // get yas selector from yas name + yasSelector, err := NewLabelSelectorForYurtAppSet(yas) + if err != nil { + return nil, err + } + + // List all Deployment to include those that don't match the selector anymore but + // have a ControllerRef pointing to this controller. + allDeployments := appsv1.DeploymentList{} + if err := d.Client.List(context.TODO(), &allDeployments); err != nil { + return nil, err + } + + manager, err := refmanager.New(d.Client, yasSelector, yas, d.Scheme) + if err != nil { + return nil, err + } + + selected := make([]metav1.Object, 0, len(allDeployments.Items)) + for i := 0; i < len(allDeployments.Items); i++ { + t := allDeployments.Items[i] + selected = append(selected, &t) + } + + objs, err := manager.ClaimOwnedObjects(selected) + if err != nil { + return nil, err + } + + return objs, nil +} + +var _ WorkloadManager = &DeploymentManager{} diff --git a/pkg/yurtmanager/controller/yurtappset/workloadmanager/deployment_manager_test.go b/pkg/yurtmanager/controller/yurtappset/workloadmanager/deployment_manager_test.go new file mode 100644 index 00000000000..1946999087e --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappset/workloadmanager/deployment_manager_test.go @@ -0,0 +1,138 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package workloadmanager + +import ( + "testing" + + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/openyurtio/openyurt/pkg/apis/apps" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" +) + +var testYAS = &v1beta1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yas", + }, + Spec: v1beta1.YurtAppSetSpec{ + Pools: []string{"test-nodepool"}, + Workload: v1beta1.Workload{ + WorkloadTemplate: v1beta1.WorkloadTemplate{ + DeploymentTemplate: &v1beta1.DeploymentTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &itemReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: "initContainer", + Image: "initOld", + }, + }, + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + Volumes: []corev1.Volume{ + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "configMapSource", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, +} + +var testNp = &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-nodepool", + }, + Spec: v1beta1.NodePoolSpec{ + HostNetwork: false, + }, +} + +func TestDeploymentManager(t *testing.T) { + var fakeScheme = newOpenYurtScheme() + var fakeClient = fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(testYAS, testNp).Build() + + dm := &DeploymentManager{ + Client: fakeClient, + Scheme: fakeScheme, + } + + // test create + err := dm.Create(testYAS, "test-nodepool", "test-revision") + assert.Nil(t, err) + + // test list + deploys, err := dm.List(testYAS) + assert.Nil(t, err) + assert.Equal(t, len(deploys), 1) + assert.Equal(t, GetWorkloadRefNodePool(deploys[0]), "test-nodepool") + + // test update + err = dm.Update(testYAS, deploys[0], "test-nodepool", "test-revision-1") + assert.Nil(t, err) + + deploys, err = dm.List(testYAS) + assert.Nil(t, err) + assert.Equal(t, len(deploys), 1) + assert.Equal(t, deploys[0].GetLabels()[apps.ControllerRevisionHashLabelKey], "test-revision-1") + + // test delete + err = dm.Delete(testYAS, deploys[0]) + assert.Nil(t, err) + + deploys, err = dm.List(testYAS) + assert.Nil(t, err) + assert.Equal(t, len(deploys), 0) + +} diff --git a/pkg/yurtmanager/controller/yurtappset/workloadmanager/interface.go b/pkg/yurtmanager/controller/yurtappset/workloadmanager/interface.go new file mode 100644 index 00000000000..5df95263abb --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappset/workloadmanager/interface.go @@ -0,0 +1,39 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workloadmanager + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" +) + +type TemplateType string + +const ( + StatefulSetTemplateType TemplateType = "StatefulSet" + DeploymentTemplateType TemplateType = "Deployment" +) + +type WorkloadManager interface { + GetTemplateType() TemplateType + + List(yas *v1beta1.YurtAppSet) ([]metav1.Object, error) + Create(yas *v1beta1.YurtAppSet, nodepoolName, revision string) error + Update(yas *v1beta1.YurtAppSet, workload metav1.Object, nodepoolName, revision string) error + Delete(yas *v1beta1.YurtAppSet, workload metav1.Object) error +} diff --git a/pkg/yurtmanager/controller/yurtappset/workloadmanager/statefulset_manager.go b/pkg/yurtmanager/controller/yurtappset/workloadmanager/statefulset_manager.go new file mode 100644 index 00000000000..bbfcd436bc6 --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappset/workloadmanager/statefulset_manager.go @@ -0,0 +1,27 @@ +/* +Copyright 2024 The Openyurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workloadmanager + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type StatefulSetManager struct { + client.Client +} + +// var _ WorkloadController = &StatefulSetControllor{} diff --git a/pkg/yurtmanager/controller/yurtappset/workloadmanager/tweaks.go b/pkg/yurtmanager/controller/yurtappset/workloadmanager/tweaks.go new file mode 100644 index 00000000000..ba356d3a585 --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappset/workloadmanager/tweaks.go @@ -0,0 +1,130 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workloadmanager + +import ( + "context" + "encoding/json" + + jsonpatch "github.com/evanphx/json-patch" + v1 "k8s.io/api/apps/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" +) + +func GetNodePoolTweaksFromYurtAppSet(cli client.Client, nodepoolName string, yas *v1beta1.YurtAppSet) (tweaksList []*v1beta1.Tweaks, err error) { + tweaksList = []*v1beta1.Tweaks{} + + np := v1beta1.NodePool{} + if err = cli.Get(context.TODO(), client.ObjectKey{Name: nodepoolName}, &np); err != nil { + return + } + + for _, yasTweak := range yas.Spec.Workload.WorkloadTweaks { + if isNodePoolRelated(&np, yasTweak.Pools, yasTweak.NodePoolSelector) { + klog.V(4).Infof("nodepool %s is related to yurtappset %s/%s, add tweaks", nodepoolName, yas.Namespace, yas.Name) + tweaksList = append(tweaksList, &yasTweak.Tweaks) + } + } + + return +} + +func ApplyTweaksToDeployment(deployment *v1.Deployment, tweaks []*v1beta1.Tweaks) error { + if len(tweaks) > 0 { + applyBasicTweaksToDeployment(deployment, tweaks) + if err := applyAdvancedTweaksToDeployment(deployment, tweaks); err != nil { + return err + } + } + return nil +} + +func applyBasicTweaksToDeployment(deployment *v1.Deployment, basicTweaks []*v1beta1.Tweaks) { + for _, item := range basicTweaks { + if item.Replicas != nil { + klog.V(4).Infof("Apply BasicTweaks successfully: overwrite replicas to %d in deployment %s/%s", *item.Replicas, deployment.Name, deployment.Namespace) + deployment.Spec.Replicas = item.Replicas + } + + for _, item := range item.ContainerImages { + for i := range deployment.Spec.Template.Spec.Containers { + if deployment.Spec.Template.Spec.Containers[i].Name == item.Name { + klog.V(5).Infof("Apply BasicTweaks successfully: overwrite container %s 's image to %s in deployment %s/%s", item.Name, item.TargetImage, deployment.Name, deployment.Namespace) + deployment.Spec.Template.Spec.Containers[i].Image = item.TargetImage + } + } + for i := range deployment.Spec.Template.Spec.InitContainers { + if deployment.Spec.Template.Spec.InitContainers[i].Name == item.Name { + klog.V(5).Infof("Apply BasicTweaks successfully: overwrite init container %s 's image to %s in deployment %s/%s", item.Name, item.TargetImage, deployment.Name, deployment.Namespace) + deployment.Spec.Template.Spec.InitContainers[i].Image = item.TargetImage + } + } + } + + } + +} + +type patchOperation struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value,omitempty"` +} + +func applyAdvancedTweaksToDeployment(deployment *v1.Deployment, tweaks []*v1beta1.Tweaks) error { + // convert into json patch format + var patchOperations []patchOperation + for _, tweak := range tweaks { + for _, patch := range tweak.Patches { + patchOperations = append(patchOperations, patchOperation{ + Op: string(patch.Operation), + Path: patch.Path, + Value: patch.Value, + }) + } + } + + if len(patchOperations) == 0 { + return nil + } + + patchBytes, err := json.Marshal(patchOperations) + if err != nil { + return err + } + patchedData, err := json.Marshal(deployment) + if err != nil { + return err + } + + // conduct json patch + patchObj, err := jsonpatch.DecodePatch(patchBytes) + if err != nil { + return err + } + patchedData, err = patchObj.Apply(patchedData) + if err != nil { + return err + } + json.Unmarshal(patchedData, deployment) + + klog.V(5).Infof("Apply AdvancedTweaks %v successfully: patched deployment %+v", patchOperations, deployment) + return nil +} diff --git a/pkg/yurtmanager/controller/yurtappset/workloadmanager/tweaks_test.go b/pkg/yurtmanager/controller/yurtappset/workloadmanager/tweaks_test.go new file mode 100644 index 00000000000..ae65083f206 --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappset/workloadmanager/tweaks_test.go @@ -0,0 +1,449 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workloadmanager + +import ( + "testing" + + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/scale/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/openyurtio/openyurt/pkg/apis" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" +) + +var ( + itemReplicas int32 = 3 + fakeScheme = newOpenYurtScheme() +) + +func newOpenYurtScheme() *runtime.Scheme { + myScheme := runtime.NewScheme() + + apis.AddToScheme(myScheme) + scheme.AddToScheme(myScheme) + appsv1.AddToScheme(myScheme) + + return myScheme +} + +var testDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &itemReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: "initContainer", + Image: "initOld", + }, + }, + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + Volumes: []corev1.Volume{ + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "configMapSource", + }, + }, + }, + }, + }, + }, + }, + }, +} + +func TestGetNodePoolTweaksFromYurtAppSet(t *testing.T) { + type args struct { + cli client.Client + nodepoolName string + yas *v1beta1.YurtAppSet + } + tests := []struct { + name string + args args + want []*v1beta1.Tweaks + wantErr bool + }{ + // Test case 1 + { + name: "nodepool matches yurtappset", + args: args{ + cli: fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects( + &v1beta1.NodePool{ObjectMeta: metav1.ObjectMeta{ + Name: "test-nodepool", + }}, + &v1beta1.YurtAppSet{ + Spec: v1beta1.YurtAppSetSpec{ + Workload: v1beta1.Workload{ + WorkloadTweaks: []v1beta1.WorkloadTweak{ + { + Pools: []string{"test-nodepool"}, + Tweaks: v1beta1.Tweaks{ + Replicas: &itemReplicas, + }, + }, + }, + }, + }}, + ).Build(), + nodepoolName: "test-nodepool", + yas: &v1beta1.YurtAppSet{ + Spec: v1beta1.YurtAppSetSpec{ + Workload: v1beta1.Workload{ + WorkloadTweaks: []v1beta1.WorkloadTweak{ + { + Pools: []string{"test-nodepool"}, + Tweaks: v1beta1.Tweaks{ + Replicas: &itemReplicas, + }, + }, + }, + }, + }, + }, + }, + want: []*v1beta1.Tweaks{ + {Replicas: &itemReplicas}, + }, + wantErr: false, + }, + // Test case 2 + { + name: "no nodepool selector or pools specified", + args: args{ + cli: fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects( + &v1beta1.NodePool{ObjectMeta: metav1.ObjectMeta{ + Name: "test-nodepool", + }}, + &v1beta1.YurtAppSet{ + Spec: v1beta1.YurtAppSetSpec{}, + }, + ).Build(), + nodepoolName: "test-nodepool", + yas: &v1beta1.YurtAppSet{ + Spec: v1beta1.YurtAppSetSpec{}, + }, + }, + want: []*v1beta1.Tweaks{}, + wantErr: false, + }, + // Test case 3 + { + name: "nodepool selector match", + args: args{ + cli: fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects( + &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-nodepool", + Labels: map[string]string{ + "app": "test-nodepool", + }, + }, + }, + &v1beta1.YurtAppSet{ + Spec: v1beta1.YurtAppSetSpec{ + Workload: v1beta1.Workload{ + WorkloadTweaks: []v1beta1.WorkloadTweak{ + { + NodePoolSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test-nodepool", + }, + }, + Tweaks: v1beta1.Tweaks{ + Replicas: &itemReplicas, + }, + }, + }, + }, + }, + }, + ).Build(), + nodepoolName: "test-nodepool", + yas: &v1beta1.YurtAppSet{ + Spec: v1beta1.YurtAppSetSpec{ + Workload: v1beta1.Workload{ + WorkloadTweaks: []v1beta1.WorkloadTweak{ + { + NodePoolSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test-nodepool", + }, + }, + Tweaks: v1beta1.Tweaks{ + Replicas: &itemReplicas, + }, + }, + }, + }, + }, + }, + }, + want: []*v1beta1.Tweaks{ + {Replicas: &itemReplicas}, + }, + wantErr: false, + }, + // Test case 4 + { + name: "no nodepools", + args: args{ + cli: fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects( + &v1beta1.YurtAppSet{ + Spec: v1beta1.YurtAppSetSpec{}, + }, + ).Build(), + nodepoolName: "test-nodepool", + yas: &v1beta1.YurtAppSet{ + Spec: v1beta1.YurtAppSetSpec{}, + }, + }, + want: []*v1beta1.Tweaks{}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetNodePoolTweaksFromYurtAppSet(tt.args.cli, tt.args.nodepoolName, tt.args.yas) + if (err != nil) != tt.wantErr { + t.Errorf("GetNodePoolTweaksFromYurtAppSet() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !assert.Equal(t, tt.want, got) { + t.Errorf("GetNodePoolTweaksFromYurtAppSet() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestApplyBasicTweaksToDeployment(t *testing.T) { + var targetReplicas int32 = 2 + items := []*v1beta1.Tweaks{ + { + ContainerImages: []v1beta1.ContainerImage{ + { + Name: "nginx", + TargetImage: "nginx-test", + }, + { + Name: "initContainer", + TargetImage: "initNew", + }, + }, + Replicas: &targetReplicas, + }, + } + applyBasicTweaksToDeployment(testDeployment, items) + assert.Equal(t, "nginx-test", testDeployment.Spec.Template.Spec.Containers[0].Image) + assert.Equal(t, "initNew", testDeployment.Spec.Template.Spec.InitContainers[0].Image) + assert.Equal(t, targetReplicas, *testDeployment.Spec.Replicas) +} + +func TestApplyAdavancedTweaksToDeployment(t *testing.T) { + patches := []v1beta1.Patch{ + { + Operation: v1beta1.REPLACE, + Path: "/spec/template/spec/containers/0/image", + Value: apiextensionsv1.JSON{ + Raw: []byte(`"tomcat:1.18"`), + }, + }, + { + Operation: v1beta1.ADD, + Path: "/spec/replicas", + Value: apiextensionsv1.JSON{ + Raw: []byte("5"), + }, + }, + } + + var initialReplicas int32 = 2 + var testPatchDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps.openyurt.io/v1alpha1", + Kind: "YurtAppSet", + Name: "yurtappset-patch", + }}, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &initialReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, + } + + // apply no patches + testPatchDeploymentCopy := testPatchDeployment.DeepCopy() + applyAdvancedTweaksToDeployment(testPatchDeploymentCopy, []*v1beta1.Tweaks{}) + assert.Equal(t, testPatchDeploymentCopy, testPatchDeployment) + + // apply patches + applyAdvancedTweaksToDeployment(testPatchDeployment, []*v1beta1.Tweaks{ + {Patches: patches}, + }) + assert.Equal(t, "tomcat:1.18", testPatchDeployment.Spec.Template.Spec.Containers[0].Image) +} + +func TestApplyTweaksToDeployment(t *testing.T) { + type args struct { + deployment *appsv1.Deployment + tweaks []*v1beta1.Tweaks + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "Test case 1: BasicTweaks is not empty, AdvancedTweaks is empty.", + args: args{ + deployment: &appsv1.Deployment{}, + tweaks: []*v1beta1.Tweaks{ + { + ContainerImages: []v1beta1.ContainerImage{ + { + Name: "nginx", + TargetImage: "nginx-test", + }, + { + Name: "initContainer", + TargetImage: "initNew", + }, + }, + }, + { + Replicas: &itemReplicas, + }, + }, + }, + wantErr: false, + }, + { + name: "Test case 2: Both BasicTweaks and AdvancedTweaks are not empty.", + args: args{ + deployment: &appsv1.Deployment{}, + tweaks: []*v1beta1.Tweaks{ + { + ContainerImages: []v1beta1.ContainerImage{ + { + Name: "nginx", + TargetImage: "nginx-test", + }, + { + Name: "initContainer", + TargetImage: "initNew", + }, + }, + }, + { + Replicas: &itemReplicas, + }, + { + Patches: []v1beta1.Patch{ + { + Operation: v1beta1.REPLACE, + Path: "/spec/template/spec/containers/0/image", + Value: apiextensionsv1.JSON{ + Raw: []byte(`"tomcat:1.18"`), + }, + }, + { + Operation: v1beta1.ADD, + Path: "/spec/replicas", + Value: apiextensionsv1.JSON{ + Raw: []byte("5"), + }, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "Test case 3: Both BasicTweaks and AdvancedTweaks are empty.", + args: args{ + deployment: &appsv1.Deployment{}, + tweaks: []*v1beta1.Tweaks{}, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ApplyTweaksToDeployment(tt.args.deployment, tt.args.tweaks) + if (err != nil) != tt.wantErr { + t.Errorf("ApplyTweaksToDeployment() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/yurtmanager/controller/yurtappset/workloadmanager/util.go b/pkg/yurtmanager/controller/yurtappset/workloadmanager/util.go new file mode 100644 index 00000000000..638ac774781 --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappset/workloadmanager/util.go @@ -0,0 +1,151 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workloadmanager + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/pkg/apis/apps" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" + "github.com/openyurtio/openyurt/pkg/projectinfo" +) + +func getWorkloadPrefix(controllerName, nodepoolName string) string { + prefix := fmt.Sprintf("%s-%s-", controllerName, nodepoolName) + if len(validation.NameIsDNSSubdomain(prefix, true)) != 0 { + prefix = fmt.Sprintf("%s-", controllerName) + } + return prefix +} + +func CreateNodeSelectorByNodepoolName(nodepool string) map[string]string { + return map[string]string{ + projectinfo.GetNodePoolLabel(): nodepool, + } +} + +// Get label selector from yurtappset generated from yas name +func NewLabelSelectorForYurtAppSet(yas *v1beta1.YurtAppSet) (*metav1.LabelSelector, error) { + if yas == nil { + return nil, fmt.Errorf("yurtappset is nil") + } else if yas.Name == "" { + return nil, fmt.Errorf("yurtappset name is empty") + } + + selector := labels.Set{ + apps.YurtAppSetOwnerLabelKey: yas.Name, + } + return &metav1.LabelSelector{ + MatchLabels: selector, + }, nil +} + +// Get selecetd NodePools from YurtAppSet +// return sets for deduplication of NodePools +func GetNodePoolsFromYurtAppSet(cli client.Client, yas *v1beta1.YurtAppSet) (npNames sets.String, err error) { + return getSelectedNodepools(cli, yas.Spec.Pools, yas.Spec.NodePoolSelector) +} + +// Get NodePools selected by pools and npSelector +// If specified pool does not exist, it will skip +func getSelectedNodepools(cli client.Client, pools []string, npSelector *metav1.LabelSelector) (selectedNps sets.String, err error) { + selectedNps = sets.NewString() + + // get all nodepools + allNps := v1beta1.NodePoolList{} + err = cli.List(context.TODO(), &allNps) + if err != nil { + return nil, err + } + + // use selector to get selector matched nps + var selector labels.Selector + if selector, err = metav1.LabelSelectorAsSelector(npSelector); err != nil { + return nil, err + } + + expectedNps := sets.NewString(pools...) + + for _, np := range allNps.Items { + if selector != nil && selector.Matches(labels.Set(np.GetLabels())) { + selectedNps.Insert(np.Name) + } + if expectedNps.Has(np.Name) { + selectedNps.Insert(np.Name) + } + } + + return selectedNps, nil +} + +func IsNodePoolRelatedToYurtAppSet(nodePool client.Object, yas *v1beta1.YurtAppSet) bool { + return isNodePoolRelated(nodePool, yas.Spec.Pools, yas.Spec.NodePoolSelector) +} + +func isNodePoolRelated(nodePool client.Object, pools []string, npSelector *metav1.LabelSelector) bool { + + if pools != nil && StringsContain(pools, nodePool.GetName()) { + return true + } + if npSelector != nil { + npSelector, _ := metav1.LabelSelectorAsSelector(npSelector) + if npSelector.Matches(labels.Set(nodePool.GetLabels())) { + return true + } + } + return false +} + +func CombineMaps(maps ...map[string]string) map[string]string { + result := map[string]string{} + for _, m := range maps { + for k, v := range m { + result[k] = v + } + } + return result +} + +func StringsContain(strs []string, str string) bool { + for _, s := range strs { + if s == str { + return true + } + } + return false +} + +func GetWorkloadRefNodePool(workload metav1.Object) string { + if nodePool, ok := workload.GetLabels()[apps.PoolNameLabelKey]; ok { + return nodePool + } + return "" +} + +func GetWorkloadHash(workload metav1.Object) string { + if hash, ok := workload.GetLabels()[apps.ControllerRevisionHashLabelKey]; ok { + return hash + } + return "" +} diff --git a/pkg/yurtmanager/controller/yurtappset/workloadmanager/util_test.go b/pkg/yurtmanager/controller/yurtappset/workloadmanager/util_test.go new file mode 100644 index 00000000000..6778495a301 --- /dev/null +++ b/pkg/yurtmanager/controller/yurtappset/workloadmanager/util_test.go @@ -0,0 +1,388 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workloadmanager + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/openyurtio/openyurt/pkg/apis/apps" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" + "github.com/openyurtio/openyurt/pkg/projectinfo" +) + +const ( + failed = "\u2717" + succeed = "\u2713" +) + +func TestGetWorkloadPrefix(t *testing.T) { + tests := []struct { + name string + controllerName string + nodepoolName string + expect string + }{ + { + "true", + "a", + "b", + "a-b-", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + get := getWorkloadPrefix(tt.controllerName, tt.nodepoolName) + t.Logf("expect: %v, get: %v", tt.expect, get) + if !reflect.DeepEqual(get, tt.expect) { + t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) + } + t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) + }) + } +} + +func TestGetLabelSelectorFromYurtAppSet(t *testing.T) { + tests := []struct { + name string + yas *v1beta1.YurtAppSet + expect *metav1.LabelSelector + wantErr bool + }{ + { + "normal", + &v1beta1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yas", + }, + }, + &metav1.LabelSelector{ + MatchLabels: map[string]string{ + apps.YurtAppSetOwnerLabelKey: "test-yas", + }, + }, + false, + }, + { + "yas is nil", + nil, + nil, + true, + }, + { + "yas'name is empty", + &v1beta1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "", + }, + }, + nil, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + selector, err := NewLabelSelectorForYurtAppSet(tt.yas) + if (err != nil) != tt.wantErr { + t.Errorf("GetLabelSelectorFromYurtAppSet() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(selector, tt.expect) { + t.Errorf("GetLabelSelectorFromYurtAppSet() got = %v, want %v", selector, tt.expect) + } + }) + } + +} + +func TestCreateNodeSelectorByNodepoolName(t *testing.T) { + tests := []struct { + name string + nodepool string + expect map[string]string + }{ + { + "normal", + "a", + map[string]string{ + projectinfo.GetNodePoolLabel(): "a", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + get := CreateNodeSelectorByNodepoolName(tt.nodepool) + t.Logf("expect: %v, get: %v", tt.expect, get) + if !reflect.DeepEqual(get, tt.expect) { + t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) + } + t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) + }) + } +} + +func TestGetNodePoolsFromYurtAppSet(t *testing.T) { + type args struct { + cli client.Client + yas *v1beta1.YurtAppSet + } + + scheme := runtime.NewScheme() + assert.Nil(t, v1beta1.AddToScheme(scheme)) + + tests := []struct { + name string + args args + wantNps []string + wantErr bool + }{ + { + name: "TestGetNodePoolsFromYurtAppSetNilPools", + args: args{ + cli: fake.NewClientBuilder().WithScheme(scheme).Build(), + yas: &v1beta1.YurtAppSet{ + Spec: v1beta1.YurtAppSetSpec{ + Pools: nil, + }, + }, + }, + wantNps: []string{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotNps, err := GetNodePoolsFromYurtAppSet(tt.args.cli, tt.args.yas) + if (err != nil) != tt.wantErr { + t.Errorf("GetNodePoolsFromYurtAppSet() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotNps.List(), tt.wantNps) { + t.Errorf("GetNodePoolsFromYurtAppSet() gotNps = %v, want %v", gotNps.List(), tt.wantNps) + } + }) + } +} + +// TestIsNodePoolRelated 测试isNodePoolRelated函数 +func TestIsNodePoolRelated(t *testing.T) { + // 测试用例1: pools为空，npSelector不为空，匹配成功 + nodePool := &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nodepool1", + Labels: map[string]string{"label": "value"}, + }, + } + pools := []string{} + npSelector := &metav1.LabelSelector{ + MatchLabels: map[string]string{"label": "value"}, + } + assert.True(t, isNodePoolRelated(nodePool, pools, npSelector)) + + // 测试用例2: pools不为空，包含nodePool的名字，匹配成功 + pools = []string{"nodepool1"} + npSelector = nil + assert.True(t, isNodePoolRelated(nodePool, pools, npSelector)) + + // 测试用例3: pools不为空，不包含nodePool的名字，匹配失败 + pools = []string{"nodepool2"} + npSelector = nil + assert.False(t, isNodePoolRelated(nodePool, pools, npSelector)) + + // 测试用例4: pools为空，npSelector为空，匹配失败 + pools = []string{} + npSelector = nil + assert.False(t, isNodePoolRelated(nodePool, pools, npSelector)) + + // 测试用例5: pools不为空，npSelector不为空，匹配失败 + pools = []string{"nodepool2"} + npSelector = &metav1.LabelSelector{ + MatchLabels: map[string]string{"label": "value2"}, + } + assert.False(t, isNodePoolRelated(nodePool, pools, npSelector)) +} + +// TestCombineLabels 测试CombineLabels函数，测试组合两个map的情况 +func TestCombineMaps(t *testing.T) { + // 测试case 1: label1为空，期望返回label2 + var label1 map[string]string + label2 := map[string]string{"key1": "value1", "key2": "value2"} + label1 = CombineMaps(label1, label2) + expectedLabel := map[string]string{ + "key1": "value1", + "key2": "value2", + } + if !reflect.DeepEqual(label1, expectedLabel) { + t.Errorf("CombineLabels() label1 = %v, want %v", label1, label2) + } + + // 测试case 2: label2为空，期望返回label1 + label1 = map[string]string{"key1": "value1", "key2": "value2"} + label2 = make(map[string]string) + expectedLabel = map[string]string{ + "key1": "value1", + "key2": "value2", + } + label1 = CombineMaps(label1, label2) + if !reflect.DeepEqual(label1, expectedLabel) { + t.Errorf("CombineLabels() label1 = %v, want %v", label1, label2) + } + + // 测试case 3: label1和label2都不为空，期望返回合并后的map + label1 = map[string]string{"key1": "value1", "key2": "value2"} + label2 = map[string]string{"key3": "value3", "key4": "value4"} + label1 = CombineMaps(label1, label2) + expectedLabel = map[string]string{ + "key1": "value1", + "key2": "value2", + "key3": "value3", + "key4": "value4", + } + if !reflect.DeepEqual(label1, expectedLabel) { + t.Errorf("CombineLabels() label1 = %v, want %v", label1, label2) + } + + // 测试case 4: label1和label2都不为空，label1和label2都有相同的key，期望返回合并后的map + label1 = map[string]string{"key1": "value1", "key2": "value2"} + label2 = map[string]string{"key1": "value3", "key4": "value4"} + label1 = CombineMaps(label1, label2) + expectedLabel = map[string]string{ + "key1": "value3", + "key2": "value2", + "key4": "value4", + } + if !reflect.DeepEqual(label1, expectedLabel) { + t.Errorf("CombineLabels() label1 = %v, want %v", label1, label2) + } + + // 测试case 5: label2为nil，期望返回label2 + label1 = nil + label2 = map[string]string{"key1": "value1", "key2": "value2"} + expectedLabel = map[string]string{ + "key1": "value1", + "key2": "value2", + } + label1 = CombineMaps(label1, label2) + if !reflect.DeepEqual(label1, expectedLabel) { + t.Errorf("CombineLabels() label1 = %v, want %v", label1, label2) + } +} + +// TestStringsContain 测试字符串切片是否包含特定字符串 +func TestStringsContain(t *testing.T) { + // T0: 切片为空，目标字符串不为空 + strs := []string{} + str := "test" + assert.False(t, StringsContain(strs, str), "T0") + + // T1: 切片为空，目标字符串为空 + str = "" + assert.False(t, StringsContain(strs, str), "T1") + + // T2: 切片不为空，目标字符串在切片内 + strs = []string{"test", "hello"} + str = "test" + assert.True(t, StringsContain(strs, str), "T2") + + // T3: 切片不为空，目标字符串不在切片内 + str = "world" + assert.False(t, StringsContain(strs, str), "T3") +} + +func TestGetWorkloadRefNodePool(t *testing.T) { + type args struct { + workload metav1.Object + } + tests := []struct { + name string + args args + want string + }{ + { + name: "TestGetWorkloadRefNodePool_WithAnnotation", + args: args{ + workload: &metav1.ObjectMeta{ + Labels: map[string]string{ + apps.PoolNameLabelKey: "test-nodepool", + }, + }, + }, + want: "test-nodepool", + }, + { + name: "TestGetWorkloadRefNodePool_WithoutAnnotation", + args: args{ + workload: &metav1.ObjectMeta{}, + }, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetWorkloadRefNodePool(tt.args.workload) + if got != tt.want { + t.Errorf("GetWorkloadRefNodePool() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetWorkloadHash(t *testing.T) { + type args struct { + workload metav1.Object + } + tests := []struct { + name string + args args + want string + }{ + { + name: "TestGetWorkloadHash_WithAnnotation", + args: args{ + workload: &metav1.ObjectMeta{ + Labels: map[string]string{ + apps.ControllerRevisionHashLabelKey: "test-hash", + }, + }, + }, + want: "test-hash", + }, + { + name: "TestGetWorkloadHash_WithoutAnnotation", + args: args{ + workload: &metav1.ObjectMeta{}, + }, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetWorkloadHash(tt.args.workload) + if got != tt.want { + t.Errorf("GetWorkloadHash() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go index 043978ab8ff..c17c4814d00 100644 --- a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go +++ b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go @@ -1,4 +1,5 @@ /* +Copyright 2024 The OpenYurt Authors. Copyright 2021 The OpenYurt Authors. Copyright 2019 The Kruise Authors. @@ -26,24 +27,33 @@ import ( "flag" "fmt" "reflect" + "time" + apps "k8s.io/api/apps/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" "github.com/openyurtio/openyurt/cmd/yurt-manager/names" - unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/adapter" + unitv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/workloadmanager" ) func init() { @@ -52,24 +62,20 @@ func init() { var ( concurrentReconciles = 3 - controllerResource = unitv1alpha1.SchemeGroupVersion.WithResource("yurtappsets") + controllerResource = unitv1beta1.SchemeGroupVersion.WithResource("yurtappsets") ) const ( - eventTypeRevisionProvision = "RevisionProvision" - eventTypeFindPools = "FindPools" - eventTypeDupPoolsDelete = "DeleteDuplicatedPools" - eventTypePoolsUpdate = "UpdatePool" - eventTypeTemplateController = "TemplateController" + eventTypeRevisionProvision = "RevisionProvision" + eventTypeFindPools = "FindPools" + + eventTypeWorkloadsCreated = "CreateWorkload" + eventTypeWorkloadsUpdated = "UpdateWorkload" + eventTypeWorkloadsDeleted = "DeleteWorkload" slowStartInitialBatchSize = 1 ) -func Format(format string, args ...interface{}) string { - s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s: %s", names.YurtAppSetController, s) -} - // Add creates a new YurtAppSet Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(ctx context.Context, c *config.CompletedConfig, mgr manager.Manager) error { @@ -89,11 +95,11 @@ func newReconciler(c *config.CompletedConfig, mgr manager.Manager) reconcile.Rec scheme: mgr.GetScheme(), recorder: mgr.GetEventRecorderFor(names.YurtAppSetController), - poolControls: map[unitv1alpha1.TemplateType]ControlInterface{ - unitv1alpha1.StatefulSetTemplateType: &PoolControl{Client: mgr.GetClient(), scheme: mgr.GetScheme(), - adapter: &adapter.StatefulSetAdapter{Client: mgr.GetClient(), Scheme: mgr.GetScheme()}}, - unitv1alpha1.DeploymentTemplateType: &PoolControl{Client: mgr.GetClient(), scheme: mgr.GetScheme(), - adapter: &adapter.DeploymentAdapter{Client: mgr.GetClient(), Scheme: mgr.GetScheme()}}, + workloadManagers: map[workloadmanager.TemplateType]workloadmanager.WorkloadManager{ + workloadmanager.DeploymentTemplateType: &workloadmanager.DeploymentManager{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }, }, } } @@ -106,23 +112,63 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { return err } - // Watch for changes to YurtAppSet - err = c.Watch(&source.Kind{Type: &unitv1alpha1.YurtAppSet{}}, &handler.EnqueueRequestForObject{}) + nodePoolPredicate := predicate.Funcs{ + CreateFunc: func(evt event.CreateEvent) bool { + return true + }, + DeleteFunc: func(evt event.DeleteEvent) bool { + return true + }, + UpdateFunc: func(evt event.UpdateEvent) bool { + oldNodePool, ok := evt.ObjectOld.(*unitv1beta1.NodePool) + if !ok { + return false + } + newNodePool, ok := evt.ObjectNew.(*unitv1beta1.NodePool) + if !ok { + return false + } + // only enqueue if nodepool labels changed + if !reflect.DeepEqual(oldNodePool.Labels, newNodePool.Labels) { + return true + } + return false + }, + GenericFunc: func(evt event.GenericEvent) bool { + return false + }, + } + + nodePoolToYurtAppSet := func(nodePool client.Object) (res []reconcile.Request) { + res = make([]reconcile.Request, 0) + yasList := &unitv1beta1.YurtAppSetList{} + if err := mgr.GetClient().List(context.TODO(), yasList); err != nil { + return + } + + for _, yas := range yasList.Items { + if workloadmanager.IsNodePoolRelatedToYurtAppSet(nodePool, &yas) { + res = append(res, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: yas.GetName(), Namespace: yas.GetNamespace()}, + }) + } + } + return + } + + err = c.Watch(&source.Kind{Type: &unitv1beta1.NodePool{}}, handler.EnqueueRequestsFromMapFunc(nodePoolToYurtAppSet), nodePoolPredicate) if err != nil { return err } - err = c.Watch(&source.Kind{Type: &appsv1.StatefulSet{}}, &handler.EnqueueRequestForOwner{ - IsController: true, - OwnerType: &unitv1alpha1.YurtAppSet{}, - }) + err = c.Watch(&source.Kind{Type: &unitv1beta1.YurtAppSet{}}, &handler.EnqueueRequestForObject{}) if err != nil { return err } err = c.Watch(&source.Kind{Type: &appsv1.Deployment{}}, &handler.EnqueueRequestForOwner{ IsController: true, - OwnerType: &unitv1alpha1.YurtAppSet{}, + OwnerType: &unitv1beta1.YurtAppSet{}, }) if err != nil { return err @@ -138,11 +184,12 @@ type ReconcileYurtAppSet struct { client.Client scheme *runtime.Scheme - recorder record.EventRecorder - poolControls map[unitv1alpha1.TemplateType]ControlInterface + recorder record.EventRecorder + workloadManagers map[workloadmanager.TemplateType]workloadmanager.WorkloadManager } // +kubebuilder:rbac:groups=apps.openyurt.io,resources=yurtappsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps.openyurt.io,resources=nodepools,verbs=get;list;watch; // +kubebuilder:rbac:groups=apps.openyurt.io,resources=yurtappsets/status,verbs=get;update;patch // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get;update;patch @@ -153,246 +200,330 @@ type ReconcileYurtAppSet struct { // Reconcile reads that state of the cluster for a YurtAppSet object and makes changes based on the state read // and what is in the YurtAppSet.Spec -func (r *ReconcileYurtAppSet) Reconcile(_ context.Context, request reconcile.Request) (reconcile.Result, error) { - klog.V(4).Infof("Reconcile YurtAppSet %s/%s", request.Namespace, request.Name) - // Fetch the YurtAppSet instance - instance := &unitv1alpha1.YurtAppSet{} - err := r.Get(context.TODO(), request.NamespacedName, instance) - if err != nil { - if errors.IsNotFound(err) { - return reconcile.Result{}, nil - } - return reconcile.Result{}, err - } - - if instance.DeletionTimestamp != nil { - return reconcile.Result{}, nil - } - oldStatus := instance.Status.DeepCopy() +func (r *ReconcileYurtAppSet) Reconcile(_ context.Context, request reconcile.Request) (res reconcile.Result, err error) { + klog.V(2).Infof("Reconcile YurtAppSet %s/%s Start.", request.Namespace, request.Name) + res = reconcile.Result{} - currentRevision, updatedRevision, collisionCount, err := r.constructYurtAppSetRevisions(instance) + // Get yas instance + yas := &unitv1beta1.YurtAppSet{} + err = r.Get(context.TODO(), client.ObjectKey{Namespace: request.Namespace, Name: request.Name}, yas) if err != nil { - klog.Errorf("could not construct controller revision of YurtAppSet %s/%s: %s", instance.Namespace, instance.Name, err) - r.recorder.Event(instance.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("Failed%s", eventTypeRevisionProvision), err.Error()) - return reconcile.Result{}, err + klog.Warningf("YurtAppSet %s/%s get fail, %v", request.Namespace, request.Name, err) + return reconcile.Result{}, client.IgnoreNotFound(err) } - control, poolType, err := r.getPoolControls(instance) - if err != nil { - r.recorder.Event(instance.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("Failed%s", eventTypeTemplateController), err.Error()) - return reconcile.Result{}, err + if yas.DeletionTimestamp != nil { + return } - klog.V(4).Infof("Get YurtAppSet %s/%s all pools", request.Namespace, request.Name) + // Get yas original status + yasStatus := yas.Status.DeepCopy() - nameToPool, err := r.getNameToPool(instance, control) + // Get yas histories, create a new revision based on current spec + allRevisions, expectedRevision, collisionCount, err := r.constructYurtAppSetRevisions(yas) + yasStatus.CollisionCount = &collisionCount if err != nil { - klog.Errorf("could not get Pools of YurtAppSet %s/%s: %s", instance.Namespace, instance.Name, err) - r.recorder.Event(instance.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("Failed %s", - eventTypeFindPools), err.Error()) - return reconcile.Result{}, nil + klog.Errorf("could not construct controller revision of YurtAppSet %s/%s: %s", yas.Namespace, yas.Name, err) + r.recorder.Event(yas.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("Failed%s", eventTypeRevisionProvision), err.Error()) + return + } else if isRevisionInvalid(expectedRevision) { + // if expectedRevision is invalid (current yas workload template is invalid), we should report error and not retry + klog.Warningf("YurtAppSet[%s/%s] expectedRevision is invalid", yas.Namespace, yas.Name) + return } - nextPatches := GetNextPatches(instance) - klog.V(4).Infof("Get YurtAppSet %s/%s next Patches %v", instance.Namespace, instance.Name, nextPatches) - - expectedRevision := currentRevision - if updatedRevision != nil { - expectedRevision = updatedRevision + // Conciliate workloads, udpate yas related workloads (deploy/sts) + // this may infect yas appdispatched/appupdated/appdeleted condition + expectedNps, curWorkloads, nErr := r.conciliateWorkloads(yas, expectedRevision, yasStatus) + if nErr != nil { + // if err, retry after 1s; err caused by invalid revision, no need to retry + if !isRevisionInvalid(expectedRevision) { + res.RequeueAfter = 1 * time.Second + } + klog.Warningf("YurtAppSet[%s/%s] conciliate workloads error: %v", yas.Namespace, yas.Name, nErr) } - newStatus, err := r.managePools(instance, nameToPool, nextPatches, expectedRevision, poolType) - if err != nil { - klog.Errorf("could not update YurtAppSet %s/%s: %s", instance.Namespace, instance.Name, err) - r.recorder.Event(instance.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("Failed%s", eventTypePoolsUpdate), err.Error()) + + // Concilaiate yas, update yas status and clean yas related revisions + if nErr := r.conciliateYurtAppSet(yas, curWorkloads, allRevisions, expectedRevision, expectedNps, yasStatus); nErr != nil { + // if err, retry after 1s to wait for latest updates synced + res.RequeueAfter = 1 * time.Second + klog.Warningf("YurtAppSet[%s/%s] conciliate yurtappset error: %v", yas.GetNamespace(), yas.GetName(), nErr) } - return r.updateStatus(instance, newStatus, oldStatus, nameToPool, expectedRevision, collisionCount, control) + return } -func (r *ReconcileYurtAppSet) getNameToPool(instance *unitv1alpha1.YurtAppSet, control ControlInterface) (map[string]*Pool, error) { - pools, err := control.GetAllPools(instance) +func (r *ReconcileYurtAppSet) getNodePoolsFromYurtAppSet(yas *unitv1beta1.YurtAppSet, newStatus *unitv1beta1.YurtAppSetStatus) (npNames sets.String, err error) { + expectedNps, err := workloadmanager.GetNodePoolsFromYurtAppSet(r.Client, yas) if err != nil { - r.recorder.Event(instance.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("Failed%s", eventTypeFindPools), err.Error()) - return nil, fmt.Errorf("could not get all Pools for YurtAppSet %s/%s: %s", instance.Namespace, instance.Name, err) + return nil, err } - - klog.V(4).Infof("Classify YurtAppSet %s/%s by pool name", instance.Namespace, instance.Name) - nameToPools := r.classifyPoolByPoolName(pools) - - nameToPool, err := r.deleteDupPool(nameToPools, control) - if err != nil { - r.recorder.Event(instance.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("Failed%s", eventTypeDupPoolsDelete), err.Error()) - return nil, fmt.Errorf("could not manage duplicate Pool of YurtAppSet %s/%s: %s", instance.Namespace, instance.Name, err) + if expectedNps.Len() == 0 { + klog.V(4).Infof("No NodePools found for YurtAppSet %s/%s", yas.Namespace, yas.Name) + r.recorder.Event(yas.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("No%s", eventTypeFindPools), fmt.Sprintf("There are no matched nodepools for YurtAppSet %s/%s", yas.Namespace, yas.Name)) + SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1beta1.AppSetPoolFound, corev1.ConditionFalse, fmt.Sprintf("No%s", eventTypeFindPools), "There are no matched nodepools for YurtAppSet")) + } else { + klog.V(4).Infof("NodePools matched for YurtAppSet %s/%s: %v", yas.Namespace, yas.Name, expectedNps.List()) + SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1beta1.AppSetPoolFound, corev1.ConditionTrue, eventTypeFindPools, fmt.Sprintf("There are %d matched nodepools: %v", expectedNps.Len(), expectedNps.List()))) } + return expectedNps, nil +} - return nameToPool, nil +func (r *ReconcileYurtAppSet) getWorkloadManagerFromYurtAppSet(yas *unitv1beta1.YurtAppSet) (workloadmanager.WorkloadManager, error) { + switch { + case yas.Spec.Workload.WorkloadTemplate.StatefulSetTemplate != nil: + return r.workloadManagers[workloadmanager.StatefulSetTemplateType], nil + case yas.Spec.Workload.WorkloadTemplate.DeploymentTemplate != nil: + return r.workloadManagers[workloadmanager.DeploymentTemplateType], nil + default: + klog.Errorf("Invalid WorkloadTemplate") + return nil, fmt.Errorf("The appropriate WorkloadTemplate was not found, Now Support(%s/%s)", + workloadmanager.StatefulSetTemplateType, workloadmanager.DeploymentTemplateType) + } } -func (r *ReconcileYurtAppSet) deleteDupPool(nameToPools map[string][]*Pool, control ControlInterface) (map[string]*Pool, error) { - nameToPool := map[string]*Pool{} - for name, pools := range nameToPools { - if len(pools) > 1 { - for _, pool := range pools[1:] { - klog.V(0).Infof("Delete duplicated Pool %s/%s for pool name %s", pool.Namespace, pool.Name, name) - if err := control.DeletePool(pool); err != nil { - if errors.IsNotFound(err) { - continue - } - return nameToPool, err +func classifyWorkloads(yas *unitv1beta1.YurtAppSet, currentWorkloads []metav1.Object, + expectedNodePools sets.String, expectedRevision string) (needDeleted, needUpdate []metav1.Object, needCreate []string) { + + // classify workloads by nodepool name + nodePoolsToWorkloads := make(map[string]metav1.Object) + for i, w := range currentWorkloads { + if nodePool := workloadmanager.GetWorkloadRefNodePool(w); nodePool != "" { + nodePoolsToWorkloads[nodePool] = currentWorkloads[i] + } else { + klog.Warningf("YurtAppSet [%s/%s] %d's workload[%s/%s] has no nodepool annotation", + yas.GetNamespace(), yas.GetName(), i, w.GetNamespace(), w.GetName()) + } + } + klog.V(4).Infof("YurtAppSet [%s/%s] get %d workloads", + yas.GetNamespace(), yas.GetName(), len(nodePoolsToWorkloads)) + + for npName, load := range nodePoolsToWorkloads { + if _, ok := expectedNodePools[npName]; ok { + // workload already exist in expectedNp, check its revision is latest + // if not, add workload to needUpdate list + if curRevision := workloadmanager.GetWorkloadHash(load); curRevision != "" { + if curRevision != expectedRevision { + klog.V(4).Infof("YurtAppSet[%s/%s] need update [%s/%s]", yas.GetNamespace(), + yas.GetName(), load.GetNamespace(), load.GetName()) + needUpdate = append(needUpdate, load) } + } else { + klog.Warningf("YurtAppSet[%s/%s] workload[%s/%s] has no revision", yas.GetNamespace(), + yas.GetName(), load.GetNamespace(), load.GetName()) + needUpdate = append(needUpdate, load) } + + } else { + // workload not exist in expectedNp, add workload to needDelete list + needDeleted = append(needDeleted, load) + klog.V(4).Infof("YurtAppSet[%s/%s] need delete [%s/%s]", yas.GetNamespace(), + yas.GetName(), load.GetNamespace(), load.GetName()) } + } - if len(pools) > 0 { - nameToPool[name] = pools[0] + for np := range expectedNodePools { + // expected np not exist in current np, add workload to needCreate list + if _, ok := nodePoolsToWorkloads[np]; !ok && np != "" { + needCreate = append(needCreate, np) + klog.V(4).Infof("YurtAppSet[%s/%s] need create new workload by nodepool %s", yas.GetNamespace(), + yas.GetName(), np) } } - return nameToPool, nil + return } -func (r *ReconcileYurtAppSet) getPoolControls(instance *unitv1alpha1.YurtAppSet) (ControlInterface, - unitv1alpha1.TemplateType, error) { - switch { +// Conciliate workloads as yas spec expect +func (r *ReconcileYurtAppSet) conciliateWorkloads(yas *unitv1beta1.YurtAppSet, expectedRevision *appsv1.ControllerRevision, newStatus *unitv1beta1.YurtAppSetStatus) (expectedNps sets.String, curWorkloads []metav1.Object, err error) { - case instance.Spec.WorkloadTemplate.StatefulSetTemplate != nil: - return r.poolControls[unitv1alpha1.StatefulSetTemplateType], unitv1alpha1.StatefulSetTemplateType, nil - case instance.Spec.WorkloadTemplate.DeploymentTemplate != nil: - return r.poolControls[unitv1alpha1.DeploymentTemplateType], unitv1alpha1.DeploymentTemplateType, nil - default: - klog.Errorf("The appropriate WorkloadTemplate was not found") - return nil, "", fmt.Errorf("The appropriate WorkloadTemplate was not found, Now Support(%s/%s)", - unitv1alpha1.StatefulSetTemplateType, unitv1alpha1.DeploymentTemplateType) + // Get yas selected NodePools + // this may infect yas poolfound condition + expectedNps, err = r.getNodePoolsFromYurtAppSet(yas, newStatus) + if err != nil { + klog.Errorf("could not get expected nodepools from YurtAppSet %s/%s: %s", yas.Namespace, yas.Name, err) + return } -} -func (r *ReconcileYurtAppSet) classifyPoolByPoolName(pools []*Pool) map[string][]*Pool { - mapping := map[string][]*Pool{} + // Get yas workloadManager + workloadManager, err := r.getWorkloadManagerFromYurtAppSet(yas) + if err != nil { + return + } - for _, ss := range pools { - poolName := ss.Name - mapping[poolName] = append(mapping[poolName], ss) + // Get yas managed workloads + curWorkloads, err = workloadManager.List(yas) + if err != nil { + klog.Errorf("could not get all workloads of YurtAppSet %s/%s: %s", yas.Namespace, yas.Name, err) + return } - return mapping -} -func (r *ReconcileYurtAppSet) updateStatus(instance *unitv1alpha1.YurtAppSet, newStatus, oldStatus *unitv1alpha1.YurtAppSetStatus, - nameToPool map[string]*Pool, currentRevision *appsv1.ControllerRevision, - collisionCount int32, control ControlInterface) (reconcile.Result, error) { + var errs []error + + templateType := workloadManager.GetTemplateType() + // Classify workloads into del/create/update 3 categories + needDelWorkloads, needUpdateWorkloads, needCreateNodePools := classifyWorkloads(yas, curWorkloads, expectedNps, expectedRevision.GetName()) + + // Manipulate resources + // 1. create workloads + if len(needCreateNodePools) > 0 { + createdNum, createdErr := util.SlowStartBatch(len(needCreateNodePools), slowStartInitialBatchSize, func(idx int) error { + nodepoolName := needCreateNodePools[idx] + err := workloadManager.Create(yas, nodepoolName, expectedRevision.GetName()) + if err != nil { + klog.Errorf("YurtAppSet[%s/%s] templatetype %s create workload by nodepool %s error: %s", + yas.GetNamespace(), yas.GetName(), templateType, nodepoolName, err.Error()) + if !errors.IsTimeout(err) { + if errors.IsInvalid(err) { + // notify users provided workload template+tweak is not valid + r.recorder.Event(yas.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("Failed %s: invalid workload template in YurtAppSet", eventTypeWorkloadsCreated), err.Error()) + setRevisionInvalid(r.Client, expectedRevision) + } + return fmt.Errorf("YurtAppSet[%s/%s] templatetype %s create workload by nodepool %s error: %s", + yas.GetNamespace(), yas.GetName(), templateType, nodepoolName, err.Error()) + } + } + klog.Infof("YurtAppSet[%s/%s] create workload %s[%s/%s] success", + yas.GetNamespace(), yas.GetName(), templateType, nodepoolName) + return nil + }) - newStatus = r.calculateStatus(instance, newStatus, nameToPool, currentRevision, collisionCount, control) - _, err := r.updateYurtAppSet(instance, oldStatus, newStatus) + if createdErr == nil { + r.recorder.Eventf(yas.DeepCopy(), corev1.EventTypeNormal, fmt.Sprintf("Successful %s", eventTypeWorkloadsCreated), "Create %d %s", createdNum, templateType) + SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1beta1.AppSetAppDispatchced, corev1.ConditionTrue, "", "All expected workloads are created successfully")) + } else { + errs = append(errs, createdErr) + SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1beta1.AppSetAppDispatchced, corev1.ConditionFalse, "CreateWorkloadError", createdErr.Error())) + } + } - return reconcile.Result{}, err -} + // 2. delete workloads + if len(needDelWorkloads) > 0 { + delNum, delErr := util.SlowStartBatch(len(needDelWorkloads), slowStartInitialBatchSize, func(idx int) error { + workloadTobeDeleted := needDelWorkloads[idx] + err := workloadManager.Delete(yas, workloadTobeDeleted) + if err != nil { + klog.Errorf("YurtAppSet[%s/%s] delete %s[%s/%s] error: %s", + yas.GetNamespace(), yas.GetName(), templateType, workloadTobeDeleted.GetNamespace(), workloadTobeDeleted.GetName(), err.Error()) + if !errors.IsTimeout(err) { + return fmt.Errorf("YurtAppSet[%s/%s] delete %s[%s/%s] error: %s", + yas.GetNamespace(), yas.GetName(), templateType, workloadTobeDeleted.GetNamespace(), workloadTobeDeleted.GetName(), err.Error()) + } + } + klog.Infof("YurtAppSet[%s/%s] templatetype delete %s[%s/%s] success", + yas.GetNamespace(), yas.GetName(), templateType, workloadTobeDeleted.GetNamespace(), workloadTobeDeleted.GetName()) + return nil + }) -func (r *ReconcileYurtAppSet) calculateStatus(instance *unitv1alpha1.YurtAppSet, newStatus *unitv1alpha1.YurtAppSetStatus, - nameToPool map[string]*Pool, currentRevision *appsv1.ControllerRevision, - collisionCount int32, control ControlInterface) *unitv1alpha1.YurtAppSetStatus { + if delErr == nil { + r.recorder.Eventf(yas.DeepCopy(), corev1.EventTypeNormal, fmt.Sprintf("Successful %s", eventTypeWorkloadsDeleted), "Delete %d %s", delNum, templateType) + SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1beta1.AppSetAppDeleted, corev1.ConditionTrue, "", "Unexpected workloads are deleted successfully")) + } else { + errs = append(errs, delErr) + SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1beta1.AppSetAppDeleted, corev1.ConditionFalse, "DeleteWorkloadError", delErr.Error())) + } + } - newStatus.CollisionCount = &collisionCount + // 3. update workloads + if len(needUpdateWorkloads) > 0 { + updatedNum, updateErr := util.SlowStartBatch(len(needUpdateWorkloads), slowStartInitialBatchSize, func(index int) error { + workloadTobeUpdated := needUpdateWorkloads[index] + err := workloadManager.Update(yas, workloadTobeUpdated, workloadmanager.GetWorkloadRefNodePool(workloadTobeUpdated), expectedRevision.GetName()) + if err != nil { + if errors.IsInvalid(err) { + // notify users provided workload template+tweak is not valid + r.recorder.Event(yas.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("Failed %s: invalid workload template in YurtAppSet", eventTypeWorkloadsUpdated), err.Error()) + setRevisionInvalid(r.Client, expectedRevision) + } + r.recorder.Event(yas.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("Failed %s", eventTypeWorkloadsUpdated), + fmt.Sprintf("Error updating %s %s when updating: %s", templateType, workloadTobeUpdated.GetName(), err)) + klog.Errorf("YurtAppSet[%s/%s] update workload[%s/%s/%s] error %v", yas.GetNamespace(), yas.GetName(), + templateType, workloadTobeUpdated.GetNamespace(), workloadTobeUpdated.GetName(), err) + } + klog.Infof("YurtAppSet[%s/%s] templatetype %s update workload by nodepool %s success", + yas.GetNamespace(), yas.GetName(), templateType, workloadmanager.GetWorkloadRefNodePool(workloadTobeUpdated)) + return err + }) - if newStatus.CurrentRevision == "" { - // init with current revision - newStatus.CurrentRevision = currentRevision.Name + if updateErr == nil { + r.recorder.Eventf(yas.DeepCopy(), corev1.EventTypeNormal, fmt.Sprintf("Successful %s", eventTypeWorkloadsUpdated), "Update %d %s", updatedNum, templateType) + SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1beta1.AppSetAppUpdated, corev1.ConditionTrue, "", "All expected workloads are updated successfully")) + } else { + errs = append(errs, updateErr) + SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1beta1.AppSetAppUpdated, corev1.ConditionFalse, "UpdateWorkloadError", updateErr.Error())) + } } - // update OverriderRef - var poolFailure *string - overriderList := unitv1alpha1.YurtAppOverriderList{} - if err := r.List(context.TODO(), &overriderList); err != nil { - message := fmt.Sprintf("could not list yurtappoverrider: %v", err) - poolFailure = &message - } - for _, overrider := range overriderList.Items { - if overrider.Subject.Kind == "YurtAppSet" && overrider.Subject.Name == instance.Name { - newStatus.OverriderRef = overrider.Name - break - } + err = utilerrors.NewAggregate(errs) + return +} + +func (r *ReconcileYurtAppSet) conciliateYurtAppSet(yas *unitv1beta1.YurtAppSet, curWorkloads []metav1.Object, allRevisions []*apps.ControllerRevision, expecteddRevison *appsv1.ControllerRevision, expectedNps sets.String, newStatus *unitv1beta1.YurtAppSetStatus) error { + if err := r.conciliateYurtAppSetStatus(yas, curWorkloads, expecteddRevison, expectedNps, newStatus); err != nil { + return err } - // sync from status - newStatus.WorkloadSummaries = make([]unitv1alpha1.WorkloadSummary, 0) - newStatus.PoolReplicas = make(map[string]int32) - newStatus.ReadyReplicas = 0 - newStatus.Replicas = 0 - for _, pool := range nameToPool { - newStatus.PoolReplicas[pool.Name] = pool.Status.Replicas - newStatus.WorkloadSummaries = append(newStatus.WorkloadSummaries, unitv1alpha1.WorkloadSummary{ - AvailableCondition: pool.Status.AvailableCondition, - Replicas: pool.Status.Replicas, - ReadyReplicas: pool.Status.ReadyReplicas, - WorkloadName: pool.Spec.PoolRef.GetName(), - }) - newStatus.Replicas += pool.Status.Replicas - newStatus.ReadyReplicas += pool.Status.ReadyReplicas + if isRevisionInvalid(expecteddRevison) { + // donot clean revisions, when it is invalid, because it will trigger a new reconcile which will lead to endless loop + return nil } + return cleanRevisions(r.Client, yas, allRevisions) +} - newStatus.TemplateType = getPoolTemplateType(instance) +// update yas status and clean unused revisions +func (r *ReconcileYurtAppSet) conciliateYurtAppSetStatus(yas *unitv1beta1.YurtAppSet, curWorkloads []metav1.Object, expecteddRevison *appsv1.ControllerRevision, expectedNps sets.String, newStatus *unitv1beta1.YurtAppSetStatus) error { - for _, pool := range nameToPool { - failureMessage := control.GetPoolFailure(pool) - if failureMessage != nil { - poolFailure = failureMessage - break + // calculate yas current status + readyWorkloads, updatedWorkloads := 0, 0 + for _, workload := range curWorkloads { + workloadObj := workload.(*appsv1.Deployment) + if workloadObj.Status.ReadyReplicas == workloadObj.Status.Replicas { + readyWorkloads++ + } + if workloadmanager.GetWorkloadHash(workloadObj) == expecteddRevison.GetName() && workloadObj.Status.UpdatedReplicas == workloadObj.Status.Replicas { + updatedWorkloads++ } } - if poolFailure == nil { - RemoveYurtAppSetCondition(newStatus, unitv1alpha1.PoolFailure) + newStatus.ReadyWorkloads = int32(readyWorkloads) + newStatus.TotalWorkloads = int32(len(curWorkloads)) + newStatus.UpdatedWorkloads = int32(updatedWorkloads) + + if isRevisionInvalid(expecteddRevison) { + klog.Infof("YurtAppSet[%s/%s] expected revision is invalid, no need to update revision", yas.GetNamespace(), yas.GetName()) } else { - SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1alpha1.PoolFailure, corev1.ConditionTrue, "Error", *poolFailure)) + newStatus.CurrentRevision = expecteddRevison.GetName() } - return newStatus -} - -func getPoolTemplateType(obj *unitv1alpha1.YurtAppSet) (templateType unitv1alpha1.TemplateType) { - template := obj.Spec.WorkloadTemplate - switch { - case template.StatefulSetTemplate != nil: - templateType = unitv1alpha1.StatefulSetTemplateType - case template.DeploymentTemplate != nil: - templateType = unitv1alpha1.DeploymentTemplateType - default: - klog.Warning("YurtAppSet.Spec.WorkloadTemplate exist wrong template") + if newStatus.TotalWorkloads == 0 { + SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1beta1.AppSetAppReady, corev1.ConditionFalse, "NoWorkloadFound", "")) + } else if newStatus.TotalWorkloads == newStatus.ReadyWorkloads { + SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1beta1.AppSetAppReady, corev1.ConditionTrue, "AllWorkloadsReady", "")) + } else { + SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1beta1.AppSetAppReady, corev1.ConditionFalse, "NotAllWorkloadsReady", "")) } - return -} -func (r *ReconcileYurtAppSet) updateYurtAppSet(yas *unitv1alpha1.YurtAppSet, oldStatus, newStatus *unitv1alpha1.YurtAppSetStatus) (*unitv1alpha1.YurtAppSet, error) { + // update yas status + oldStatus := yas.Status if oldStatus.CurrentRevision == newStatus.CurrentRevision && - oldStatus.CollisionCount == newStatus.CollisionCount && - oldStatus.Replicas == newStatus.Replicas && - oldStatus.ReadyReplicas == newStatus.ReadyReplicas && + oldStatus.CollisionCount != nil && newStatus.CollisionCount != nil && *oldStatus.CollisionCount == *newStatus.CollisionCount && + oldStatus.TotalWorkloads == newStatus.TotalWorkloads && + oldStatus.ReadyWorkloads == newStatus.ReadyWorkloads && + oldStatus.UpdatedWorkloads == newStatus.UpdatedWorkloads && yas.Generation == newStatus.ObservedGeneration && - reflect.DeepEqual(oldStatus.WorkloadSummaries, newStatus.WorkloadSummaries) && reflect.DeepEqual(oldStatus.Conditions, newStatus.Conditions) { - return yas, nil + klog.Infof("YurtAppSet[%s/%s] oldStatus==newStatus, no need to update status", yas.GetNamespace(), yas.GetName()) + return nil + } else { + klog.V(5).Infof("YurtAppSet[%s/%s] oldStatus=%+v, newStatus=%+v, need to update status", yas.GetNamespace(), yas.GetName(), oldStatus, newStatus) } newStatus.ObservedGeneration = yas.Generation - var getErr, updateErr error - for i, obj := 0, yas; ; i++ { - klog.V(4).Infof(fmt.Sprintf("The %d th time updating status for %v: %s/%s, ", i, obj.Kind, obj.Namespace, obj.Name) + - fmt.Sprintf("sequence No: %v->%v", obj.Status.ObservedGeneration, newStatus.ObservedGeneration)) - - obj.Status = *newStatus - - updateErr = r.Client.Status().Update(context.TODO(), obj) - if updateErr == nil { - return obj, nil - } - if i >= updateRetries { - break - } - tmpObj := &unitv1alpha1.YurtAppSet{} - if getErr = r.Client.Get(context.TODO(), client.ObjectKey{Namespace: obj.Namespace, Name: obj.Name}, tmpObj); getErr != nil { - return nil, getErr - } - obj = tmpObj + yas.Status = *newStatus + if err := r.Client.Status().Update(context.TODO(), yas); err != nil { + return err } + klog.Infof("YurtAppSet[%s/%s] update status success.", yas.Namespace, yas.Name) - klog.Errorf("could not update YurtAppSet %s/%s status: %s", yas.Namespace, yas.Name, updateErr) - return nil, updateErr + return nil } diff --git a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_test.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_test.go index 18a2a364a10..f34bead7a13 100644 --- a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_test.go +++ b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_test.go @@ -1,12 +1,11 @@ /* -Copyright 2020 The OpenYurt Authors. -Copyright 2019 The Kruise Authors. +Copyright 2024 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -19,280 +18,387 @@ package yurtappset import ( "context" - "strconv" "testing" + "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - fakeclint "sigs.k8s.io/controller-runtime/pkg/client/fake" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" - appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/adapter" + "github.com/openyurtio/openyurt/pkg/apis/apps" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/workloadmanager" ) -var ( - ten int32 = 10 - forteen int64 = 14 - fifteen int64 = 15 -) - -func TestReconcileYurtAppSet_Reconcile2(t *testing.T) { - -} - -func TestReconcileYurtAppSet_Reconcile(t *testing.T) { - instance := &appsv1alpha1.YurtAppSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "foo-ns", - }, - Spec: appsv1alpha1.YurtAppSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "foo", - }, - }, - WorkloadTemplate: appsv1alpha1.WorkloadTemplate{ - DeploymentTemplate: &appsv1alpha1.DeploymentTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &two, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "name": "foo", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "container", - Image: "nginx:1.0", - }, - }, +func TestGetWorkloadManagerFromYurtAppSet(t *testing.T) { + type args struct { + yas *v1beta1.YurtAppSet + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "StatefulSetTemplate is set in YurtAppSet's Spec.Workload.WorkloadTemplate.", + args: args{ + yas: &v1beta1.YurtAppSet{ + Spec: v1beta1.YurtAppSetSpec{ + Workload: v1beta1.Workload{ + WorkloadTemplate: v1beta1.WorkloadTemplate{ + StatefulSetTemplate: &v1beta1.StatefulSetTemplateSpec{}, }, }, }, }, }, - Topology: appsv1alpha1.Topology{ - Pools: []appsv1alpha1.Pool{ - { - Name: "foo-0", - Replicas: &one, - NodeSelectorTerm: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "app.openyurt.io/nodepool", - Operator: corev1.NodeSelectorOpIn, - Values: []string{ - "foo-0", - }, - }, - }, - }, - }, - { - Name: "foo-1", - Replicas: &two, - NodeSelectorTerm: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "app.openyurt.io/nodepool", - Operator: corev1.NodeSelectorOpIn, - Values: []string{ - "foo-1", - }, - }, + wantErr: false, + }, + { + name: "DeploymentTemplate is set in YurtAppSet's Spec.Workload.WorkloadTemplate.", + args: args{ + yas: &v1beta1.YurtAppSet{ + Spec: v1beta1.YurtAppSetSpec{ + Workload: v1beta1.Workload{ + WorkloadTemplate: v1beta1.WorkloadTemplate{ + DeploymentTemplate: &v1beta1.DeploymentTemplateSpec{}, }, }, }, }, }, - RevisionHistoryLimit: &two, + wantErr: false, }, - Status: appsv1alpha1.YurtAppSetStatus{ - ObservedGeneration: fifteen, - CollisionCount: &two, - CurrentRevision: "v0.1.0", - Replicas: 2, - ReadyReplicas: 2, - TemplateType: appsv1alpha1.DeploymentTemplateType, + { + name: "Neither StatefulSetTemplate nor DeploymentTemplate is set in YurtAppSet's Spec.Workload.WorkloadTemplate.", + args: args{ + yas: &v1beta1.YurtAppSet{ + Spec: v1beta1.YurtAppSetSpec{ + Workload: v1beta1.Workload{}, + }, + }, + }, + wantErr: true, }, } - scheme := runtime.NewScheme() - if err := appsv1alpha1.AddToScheme(scheme); err != nil { - t.Logf("failed to add yurt custom resource") - return + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := ReconcileYurtAppSet{} + _, err := r.getWorkloadManagerFromYurtAppSet(tt.args.yas) + if (err != nil) != tt.wantErr { + t.Errorf("getWorkloadManagerFromYurtAppSet() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) } - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Logf("failed to add kubernetes clint-go custom resource") - return - } - fc := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance).Build() +} - var req = reconcile.Request{NamespacedName: types.NamespacedName{Name: "foo", Namespace: "foo-ns"}} - ryas := ReconcileYurtAppSet{ - Client: fc, - scheme: scheme, - poolControls: map[appsv1alpha1.TemplateType]ControlInterface{ - appsv1alpha1.StatefulSetTemplateType: &PoolControl{ - Client: fc, - scheme: scheme, - adapter: &adapter.StatefulSetAdapter{Client: fc, Scheme: scheme}, +func TestClassifyWorkloads(t *testing.T) { + yas := &v1beta1.YurtAppSet{} + expectedNodePools := sets.NewString("test-np1", "test-np3") + expectedRevision := "test-revision-2" + workloadTobeDeleted := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment-1", + Labels: map[string]string{ + apps.PoolNameLabelKey: "test-np2", }, - appsv1alpha1.DeploymentTemplateType: &PoolControl{ - Client: fc, - scheme: scheme, - adapter: &adapter.DeploymentAdapter{Client: fc, Scheme: scheme}, + }, + } + workloadTobeUpdated := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment-2", + Labels: map[string]string{ + apps.PoolNameLabelKey: "test-np1", + apps.ControllerRevisionHashLabelKey: "test-revision-1", }, }, } + currentWorkloads := []metav1.Object{workloadTobeDeleted, workloadTobeUpdated} - for i := 0; i < 2; i++ { - tf := ryas.poolControls[appsv1alpha1.DeploymentTemplateType].CreatePool(instance, "foo-"+strconv.FormatInt(int64(i), 10), "v0.1.0", two) - if tf != nil { - t.Logf("failed create node pool resource") - } + needDeleted, needUpdate, needCreate := classifyWorkloads(yas, currentWorkloads, expectedNodePools, expectedRevision) + if len(needDeleted) != 1 || needDeleted[0].GetName() != workloadTobeDeleted.GetName() { + t.Errorf("classifyWorkloads() needDeleted = %v, want %v", needDeleted, workloadTobeDeleted) } - - _, err := ryas.Reconcile(context.TODO(), req) - if err != nil { - t.Logf("failed to control yurtappset controller") + if len(needUpdate) != 1 || needUpdate[0].GetName() != workloadTobeUpdated.GetName() { + t.Errorf("classifyWorkloads() needUpdate = %v, want %v", needUpdate, workloadTobeUpdated) + } + if len(needCreate) != 1 || needCreate[0] != "test-np3" { + t.Errorf("classifyWorkloads() needCreate = %v, want %v", needCreate, []string{"test-np3"}) } } -func TestGetPoolTemplateType(t *testing.T) { - instances := []struct { - yas *appsv1alpha1.YurtAppSet - want appsv1alpha1.TemplateType +type fakeEventRecorder struct { +} + +func (f *fakeEventRecorder) Event(object runtime.Object, eventtype, reason, message string) { +} + +func (f *fakeEventRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { +} + +func (f *fakeEventRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { +} + +func TestReconcile(t *testing.T) { + tests := []struct { + name string + request reconcile.Request + yas *v1beta1.YurtAppSet + npList []client.Object + deployList []client.Object + + expectedDeployNum int + expectedErr bool + isUpdated bool }{ { - yas: &appsv1alpha1.YurtAppSet{ + name: "yas create 2 deployment", + request: reconcile.Request{ + NamespacedName: client.ObjectKey{ + Name: "test-yurtappset", + Namespace: "default", + }, + }, + yas: &v1beta1.YurtAppSet{ ObjectMeta: metav1.ObjectMeta{ - Name: "fooYurtAppSet", - Namespace: "foo", + Name: "test-yurtappset", + Namespace: "default", }, - Spec: appsv1alpha1.YurtAppSetSpec{ - WorkloadTemplate: appsv1alpha1.WorkloadTemplate{ - DeploymentTemplate: &appsv1alpha1.DeploymentTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fooo", - Namespace: "foo-ns", + Spec: v1beta1.YurtAppSetSpec{ + Pools: []string{"test-np1", "test-np2"}, + Workload: v1beta1.Workload{ + WorkloadTemplate: v1beta1.WorkloadTemplate{ + DeploymentTemplate: &v1beta1.DeploymentTemplateSpec{ + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test-yurtappset", + }, + }, + }, }, }, }, }, }, - want: "Deployment", + npList: []client.Object{ + &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-np1", + }, + }, + &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-np2", + }, + }, + }, + expectedDeployNum: 2, + expectedErr: false, }, { - yas: &appsv1alpha1.YurtAppSet{ + name: "no np found, should delete 1 deployment", + request: reconcile.Request{ + NamespacedName: client.ObjectKey{ + Name: "test-yurtappset", + Namespace: "default", + }, + }, + yas: &v1beta1.YurtAppSet{ ObjectMeta: metav1.ObjectMeta{ - Name: "fooYurtAppSet", - Namespace: "foo", + Name: "test-yurtappset", + Namespace: "default", }, - Spec: appsv1alpha1.YurtAppSetSpec{ - WorkloadTemplate: appsv1alpha1.WorkloadTemplate{ - StatefulSetTemplate: &appsv1alpha1.StatefulSetTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "foo-ns", + Spec: v1beta1.YurtAppSetSpec{ + Pools: []string{"test-np1"}, + Workload: v1beta1.Workload{ + WorkloadTemplate: v1beta1.WorkloadTemplate{ + DeploymentTemplate: &v1beta1.DeploymentTemplateSpec{ + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test-yurtappset", + }, + }, + }, }, }, }, }, }, - want: "StatefulSet", + deployList: []client.Object{ + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment-1", + Namespace: "default", + Labels: map[string]string{ + apps.PoolNameLabelKey: "test-np2", + apps.YurtAppSetOwnerLabelKey: "test-yurtappset", + }, + }, + }, + }, + expectedDeployNum: 0, + expectedErr: false, }, { - yas: &appsv1alpha1.YurtAppSet{ + name: "1 np found and 1 np miss, should create 1 deployment and delete 1 deployment", + request: reconcile.Request{ + NamespacedName: client.ObjectKey{ + Name: "test-yurtappset", + Namespace: "default", + }, + }, + yas: &v1beta1.YurtAppSet{ ObjectMeta: metav1.ObjectMeta{ - Name: "fooYurtAppSet", - Namespace: "foo", + Name: "test-yurtappset", + Namespace: "default", + }, + Spec: v1beta1.YurtAppSetSpec{ + Pools: []string{"test-np1"}, + Workload: v1beta1.Workload{ + WorkloadTemplate: v1beta1.WorkloadTemplate{ + DeploymentTemplate: &v1beta1.DeploymentTemplateSpec{ + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test-yurtappset", + }, + }, + }, + }, + }, + }, }, }, - want: "", - }, - } - - for _, v := range instances { - tt := getPoolTemplateType(v.yas) - if tt != v.want { - t.Logf("failed to get pool template type") - } - } -} - -func TestReconcileYurtAppSet_UpdateYurtAppSet(t *testing.T) { - instance := struct { - yas *appsv1alpha1.YurtAppSet - oldStatus *appsv1alpha1.YurtAppSetStatus - newStatus *appsv1alpha1.YurtAppSetStatus - }{ - yas: &appsv1alpha1.YurtAppSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fooYurtAppSet", - Namespace: "foo", - Generation: forteen, + npList: []client.Object{ + &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-np1", + }, + }, }, - Spec: appsv1alpha1.YurtAppSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "name": "foo", + deployList: []client.Object{ + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment-1", + Namespace: "default", + Labels: map[string]string{ + apps.PoolNameLabelKey: "test-np2", + apps.YurtAppSetOwnerLabelKey: "test-yurtappset", + }, }, }, }, - Status: appsv1alpha1.YurtAppSetStatus{}, + expectedDeployNum: 1, + expectedErr: false, }, - oldStatus: &appsv1alpha1.YurtAppSetStatus{ - CurrentRevision: "v0.1.0", - CollisionCount: &ten, - Replicas: two, - ReadyReplicas: one, - ObservedGeneration: fifteen, - PoolReplicas: map[string]int32{ - "foo-pool": ten, + { + name: "update 1 deployment", + request: reconcile.Request{ + NamespacedName: client.ObjectKey{ + Name: "test-yurtappset", + Namespace: "default", + }, }, - Conditions: []appsv1alpha1.YurtAppSetCondition{}, - }, - newStatus: &appsv1alpha1.YurtAppSetStatus{ - CurrentRevision: "v0.2.0", - CollisionCount: &ten, - Replicas: two, - ReadyReplicas: one, - ObservedGeneration: fifteen, - PoolReplicas: map[string]int32{ - "foo-pool": ten, + yas: &v1beta1.YurtAppSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-yurtappset", + Namespace: "default", + }, + Spec: v1beta1.YurtAppSetSpec{ + Pools: []string{"test-np1"}, + Workload: v1beta1.Workload{ + WorkloadTemplate: v1beta1.WorkloadTemplate{ + DeploymentTemplate: &v1beta1.DeploymentTemplateSpec{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test-container", + Image: "test-image", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + npList: []client.Object{ + &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-np1", + }, + }, + }, + deployList: []client.Object{ + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment-1", + Namespace: "default", + Labels: map[string]string{ + apps.PoolNameLabelKey: "test-np1", + apps.YurtAppSetOwnerLabelKey: "test-yurtappset", + apps.ControllerRevisionHashLabelKey: "test-revision-1", + }, + }, + }, }, - Conditions: []appsv1alpha1.YurtAppSetCondition{}, + expectedDeployNum: 1, + expectedErr: false, + isUpdated: true, }, } - scheme := runtime.NewScheme() - if err := appsv1alpha1.AddToScheme(scheme); err != nil { - t.Logf("failed to add yurt custom resource") - return - } - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Logf("failed to add kubernetes clint-go custom resource") - return - } - fc := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance.yas).Build() - r := ReconcileYurtAppSet{Client: fc, scheme: scheme} - o, err := r.updateYurtAppSet(instance.yas, instance.oldStatus, instance.newStatus) - if err != nil || o.Status.CurrentRevision != instance.newStatus.CurrentRevision { - t.Logf("failed to update yurtappset") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + objList := []client.Object{tt.yas} + if len(tt.npList) > 0 { + objList = append(objList, tt.npList...) + } + if len(tt.deployList) > 0 { + objList = append(objList, tt.deployList...) + } + + fakeClient := fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(objList...).Build() + r := &ReconcileYurtAppSet{ + scheme: fakeScheme, + Client: fakeClient, + recorder: &fakeEventRecorder{}, + workloadManagers: map[workloadmanager.TemplateType]workloadmanager.WorkloadManager{ + workloadmanager.DeploymentTemplateType: &workloadmanager.DeploymentManager{ + Client: fakeClient, + Scheme: fakeScheme, + }, + }, + } + + _, err := r.Reconcile(context.TODO(), tt.request) + if tt.expectedErr { + assert.NotNil(t, err) + } + + deployList := &appsv1.DeploymentList{} + if err := fakeClient.List(context.TODO(), deployList); err == nil { + assert.Len(t, deployList.Items, tt.expectedDeployNum) + } + + if tt.isUpdated { + for _, deploy := range deployList.Items { + assert.NotEqual(t, deploy.Labels[apps.ControllerRevisionHashLabelKey], tt.yas.Status.CurrentRevision) + } + } + }) } } diff --git a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_utils.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_utils.go deleted file mode 100644 index af3c4e90f40..00000000000 --- a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_utils.go +++ /dev/null @@ -1,121 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. -Copyright 2019 The Kruise Authors. -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -@CHANGELOG -OpenYurt Authors: -change some functions -*/ - -package yurtappset - -import ( - "fmt" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/openyurtio/openyurt/pkg/apis/apps" - unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" -) - -const updateRetries = 5 - -type YurtAppSetPatches struct { - Replicas int32 - Patch string -} - -func getPoolNameFrom(metaObj metav1.Object) (string, error) { - name, exist := metaObj.GetLabels()[apps.PoolNameLabelKey] - if !exist { - return "", fmt.Errorf("could not get pool name from label of pool %s/%s: no label %s found", metaObj.GetNamespace(), metaObj.GetName(), apps.PoolNameLabelKey) - } - - if len(name) == 0 { - return "", fmt.Errorf("could not get pool name from label of pool %s/%s: label %s has an empty value", metaObj.GetNamespace(), metaObj.GetName(), apps.PoolNameLabelKey) - } - - return name, nil -} - -// NewYurtAppSetCondition creates a new YurtAppSet condition. -func NewYurtAppSetCondition(condType unitv1alpha1.YurtAppSetConditionType, status corev1.ConditionStatus, reason, message string) *unitv1alpha1.YurtAppSetCondition { - return &unitv1alpha1.YurtAppSetCondition{ - Type: condType, - Status: status, - LastTransitionTime: metav1.Now(), - Reason: reason, - Message: message, - } -} - -// GetYurtAppSetCondition returns the condition with the provided type. -func GetYurtAppSetCondition(status unitv1alpha1.YurtAppSetStatus, condType unitv1alpha1.YurtAppSetConditionType) *unitv1alpha1.YurtAppSetCondition { - for i := range status.Conditions { - c := status.Conditions[i] - if c.Type == condType { - return &c - } - } - return nil -} - -// SetYurtAppSetCondition updates the YurtAppSet to include the provided condition. If the condition that -// we are about to add already exists and has the same status, reason and message then we are not going to update. -func SetYurtAppSetCondition(status *unitv1alpha1.YurtAppSetStatus, condition *unitv1alpha1.YurtAppSetCondition) { - currentCond := GetYurtAppSetCondition(*status, condition.Type) - if currentCond != nil && currentCond.Status == condition.Status && currentCond.Reason == condition.Reason { - return - } - - if currentCond != nil && currentCond.Status == condition.Status { - condition.LastTransitionTime = currentCond.LastTransitionTime - } - newConditions := filterOutCondition(status.Conditions, condition.Type) - status.Conditions = append(newConditions, *condition) -} - -// RemoveYurtAppSetCondition removes the YurtAppSet condition with the provided type. -func RemoveYurtAppSetCondition(status *unitv1alpha1.YurtAppSetStatus, condType unitv1alpha1.YurtAppSetConditionType) { - status.Conditions = filterOutCondition(status.Conditions, condType) -} - -func filterOutCondition(conditions []unitv1alpha1.YurtAppSetCondition, condType unitv1alpha1.YurtAppSetConditionType) []unitv1alpha1.YurtAppSetCondition { - var newConditions []unitv1alpha1.YurtAppSetCondition - for _, c := range conditions { - if c.Type == condType { - continue - } - newConditions = append(newConditions, c) - } - return newConditions -} - -func GetNextPatches(yas *unitv1alpha1.YurtAppSet) map[string]YurtAppSetPatches { - next := make(map[string]YurtAppSetPatches) - for _, pool := range yas.Spec.Topology.Pools { - t := YurtAppSetPatches{} - if pool.Replicas != nil { - t.Replicas = *pool.Replicas - } - if pool.Patch != nil { - t.Patch = string(pool.Patch.Raw) - } - next[pool.Name] = t - } - return next -} diff --git a/pkg/yurtmanager/controller/yurtappset/yurtappset_update.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_update.go deleted file mode 100644 index 509c8bd654c..00000000000 --- a/pkg/yurtmanager/controller/yurtappset/yurtappset_update.go +++ /dev/null @@ -1,198 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. -Copyright 2019 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -@CHANGELOG -OpenYurt Authors: -Subset to pool -*/ - -package yurtappset - -import ( - "fmt" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/klog/v2" - - unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util" -) - -func (r *ReconcileYurtAppSet) managePools(yas *unitv1alpha1.YurtAppSet, - nameToPool map[string]*Pool, nextPatches map[string]YurtAppSetPatches, - expectedRevision *appsv1.ControllerRevision, - poolType unitv1alpha1.TemplateType) (newStatus *unitv1alpha1.YurtAppSetStatus, updateErr error) { - - newStatus = yas.Status.DeepCopy() - exists, provisioned, err := r.managePoolProvision(yas, nameToPool, nextPatches, expectedRevision, poolType) - if err != nil { - SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1alpha1.PoolProvisioned, corev1.ConditionFalse, "Error", err.Error())) - return newStatus, fmt.Errorf("could not manage Pool provision: %s", err) - } - - if provisioned { - SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1alpha1.PoolProvisioned, corev1.ConditionTrue, "", "")) - } - - var needUpdate []string - for _, name := range exists.List() { - pool := nameToPool[name] - if r.poolControls[poolType].IsExpected(pool, expectedRevision.Name) || - pool.Status.ReplicasInfo.Replicas != nextPatches[name].Replicas || - pool.Status.PatchInfo != nextPatches[name].Patch { - needUpdate = append(needUpdate, name) - } - } - - if len(needUpdate) > 0 { - _, updateErr = util.SlowStartBatch(len(needUpdate), slowStartInitialBatchSize, func(index int) error { - cell := needUpdate[index] - pool := nameToPool[cell] - replicas := nextPatches[cell].Replicas - - klog.Infof("YurtAppSet %s/%s needs to update Pool (%s) %s/%s with revision %s, replicas %d ", - yas.Namespace, yas.Name, poolType, pool.Namespace, pool.Name, expectedRevision.Name, replicas) - - updatePoolErr := r.poolControls[poolType].UpdatePool(pool, yas, expectedRevision.Name, replicas) - if updatePoolErr != nil { - r.recorder.Event(yas.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("Failed%s", eventTypePoolsUpdate), fmt.Sprintf("Error updating PodSet (%s) %s when updating: %s", poolType, pool.Name, updatePoolErr)) - } - return updatePoolErr - }) - } - - if updateErr == nil { - SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1alpha1.PoolUpdated, corev1.ConditionTrue, "", "")) - } else { - SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1alpha1.PoolUpdated, corev1.ConditionFalse, "Error", updateErr.Error())) - } - return -} - -func (r *ReconcileYurtAppSet) managePoolProvision(yas *unitv1alpha1.YurtAppSet, - nameToPool map[string]*Pool, nextPatches map[string]YurtAppSetPatches, - expectedRevision *appsv1.ControllerRevision, workloadType unitv1alpha1.TemplateType) (sets.String, bool, error) { - expectedPools := sets.String{} - gotPools := sets.String{} - - for _, pool := range yas.Spec.Topology.Pools { - expectedPools.Insert(pool.Name) - } - - for poolName := range nameToPool { - gotPools.Insert(poolName) - } - klog.V(4).Infof("YurtAppSet %s/%s has pools %v, expects pools %v", yas.Namespace, yas.Name, gotPools.List(), expectedPools.List()) - - var creates []string - for _, expectPool := range expectedPools.List() { - if gotPools.Has(expectPool) { - continue - } - - creates = append(creates, expectPool) - } - - var deletes []string - for _, gotPool := range gotPools.List() { - if expectedPools.Has(gotPool) { - continue - } - - deletes = append(deletes, gotPool) - } - - revision := expectedRevision.Name - - var errs []error - // manage creating - if len(creates) > 0 { - // do not consider deletion - klog.Infof("YurtAppSet %s/%s needs creating pool (%s) with name: %v", yas.Namespace, yas.Name, workloadType, creates) - createdPools := make([]string, len(creates)) - for i, pool := range creates { - createdPools[i] = pool - } - - var createdNum int - var createdErr error - createdNum, createdErr = util.SlowStartBatch(len(creates), slowStartInitialBatchSize, func(idx int) error { - poolName := createdPools[idx] - - replicas := nextPatches[poolName].Replicas - err := r.poolControls[workloadType].CreatePool(yas, poolName, revision, replicas) - if err != nil { - if !errors.IsTimeout(err) { - return fmt.Errorf("could not create Pool (%s) %s: %s", workloadType, poolName, err.Error()) - } - } - - return nil - }) - if createdErr == nil { - r.recorder.Eventf(yas.DeepCopy(), corev1.EventTypeNormal, fmt.Sprintf("Successful%s", eventTypePoolsUpdate), "Create %d Pool (%s)", createdNum, workloadType) - } else { - errs = append(errs, createdErr) - } - } - - // manage deleting - if len(deletes) > 0 { - klog.Infof("YurtAppSet %s/%s needs deleting pool (%s) with name: [%v]", yas.Namespace, yas.Name, workloadType, deletes) - var deleteErrs []error - for _, poolName := range deletes { - pool := nameToPool[poolName] - if err := r.poolControls[workloadType].DeletePool(pool); err != nil { - deleteErrs = append(deleteErrs, fmt.Errorf("could not delete Pool (%s) %s/%s for %s: %s", workloadType, pool.Namespace, pool.Name, poolName, err)) - } - } - - if len(deleteErrs) > 0 { - errs = append(errs, deleteErrs...) - } else { - r.recorder.Eventf(yas.DeepCopy(), corev1.EventTypeNormal, fmt.Sprintf("Successful%s", eventTypePoolsUpdate), "Delete %d Pool (%s)", len(deletes), workloadType) - } - } - - // clean the other kind of pools - // maybe user can chagne yas.Spec.WorkloadTemplate - cleaned := false - for t, control := range r.poolControls { - if t == workloadType { - continue - } - - pools, err := control.GetAllPools(yas) - if err != nil { - errs = append(errs, fmt.Errorf("could not list Pool of other type %s for YurtAppSet %s/%s: %s", t, yas.Namespace, yas.Name, err)) - continue - } - - for _, pool := range pools { - cleaned = true - if err := control.DeletePool(pool); err != nil { - errs = append(errs, fmt.Errorf("could not delete Pool %s of other type %s for YurtAppSet %s/%s: %s", pool.Name, t, yas.Namespace, yas.Name, err)) - continue - } - } - } - - return expectedPools.Intersection(gotPools), len(creates) > 0 || len(deletes) > 0 || cleaned, utilerrors.NewAggregate(errs) -} diff --git a/pkg/yurtmanager/webhook/server.go b/pkg/yurtmanager/webhook/server.go index f2777b0f671..9989a153760 100644 --- a/pkg/yurtmanager/webhook/server.go +++ b/pkg/yurtmanager/webhook/server.go @@ -40,7 +40,7 @@ import ( webhookcontroller "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util/controller" v1alpha1yurtappdaemon "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1" v1alpha1yurtappoverrider "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1" - v1alpha1yurtappset "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtappset/v1alpha1" + v1beta1yurtappset "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtappset/v1beta1" v1alpha1yurtstaticset "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtstaticset/v1alpha1" ) @@ -73,7 +73,7 @@ func init() { addControllerWebhook(names.GatewayPickupController, &v1beta1gateway.GatewayHandler{}) addControllerWebhook(names.NodePoolController, &v1beta1nodepool.NodePoolHandler{}) addControllerWebhook(names.YurtStaticSetController, &v1alpha1yurtstaticset.YurtStaticSetHandler{}) - addControllerWebhook(names.YurtAppSetController, &v1alpha1yurtappset.YurtAppSetHandler{}) + addControllerWebhook(names.YurtAppSetController, &v1beta1yurtappset.YurtAppSetHandler{}) addControllerWebhook(names.YurtAppDaemonController, &v1alpha1yurtappdaemon.YurtAppDaemonHandler{}) addControllerWebhook(names.PlatformAdminController, &v1alpha1platformadmin.PlatformAdminHandler{}) addControllerWebhook(names.PlatformAdminController, &v1alpha2platformadmin.PlatformAdminHandler{}) diff --git a/pkg/yurtmanager/webhook/yurtappset/v1alpha1/validate.go b/pkg/yurtmanager/webhook/yurtappset/v1alpha1/validate.go deleted file mode 100644 index 00e336eebe2..00000000000 --- a/pkg/yurtmanager/webhook/yurtappset/v1alpha1/validate.go +++ /dev/null @@ -1,322 +0,0 @@ -/* -Copyright 2020 The OpenYurt Authors. -Copyright 2019 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "fmt" - "strings" - - appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" - apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - unversionedvalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/klog/v2" - appsvalidation "k8s.io/kubernetes/pkg/apis/apps/validation" - "k8s.io/kubernetes/pkg/apis/core" - corev1 "k8s.io/kubernetes/pkg/apis/core/v1" - apivalidation "k8s.io/kubernetes/pkg/apis/core/validation" - "sigs.k8s.io/controller-runtime/pkg/client" - - unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" -) - -// ValidateYurtAppSetSpec tests if required fields in the YurtAppSet spec are set. -func validateYurtAppSetSpec(c client.Client, spec *unitv1alpha1.YurtAppSetSpec, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - if spec.Selector == nil { - allErrs = append(allErrs, field.Required(fldPath.Child("selector"), "")) - } else { - allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(spec.Selector, fldPath.Child("selector"))...) - if len(spec.Selector.MatchLabels)+len(spec.Selector.MatchExpressions) == 0 { - allErrs = append(allErrs, field.Invalid(fldPath.Child("selector"), spec.Selector, "empty selector is invalid for statefulset")) - } - } - - klog.Infof("templatePath:%s", fldPath.Child("workloadTemplate").String()) - - selector, err := metav1.LabelSelectorAsSelector(spec.Selector) - if err != nil { - allErrs = append(allErrs, field.Invalid(fldPath.Child("selector"), spec.Selector, "")) - } else { - allErrs = append(allErrs, validatePoolTemplate(&(spec.WorkloadTemplate), spec, selector, fldPath.Child("workloadTemplate"))...) - } - - poolNames := sets.String{} - for i, pool := range spec.Topology.Pools { - if len(pool.Name) == 0 { - allErrs = append(allErrs, field.Required(fldPath.Child("topology", "pools").Index(i).Child("name"), "")) - } - - if poolNames.Has(pool.Name) { - allErrs = append(allErrs, field.Invalid(fldPath.Child("topology", "pools").Index(i).Child("name"), pool.Name, - fmt.Sprintf("duplicated pool name %s", pool.Name))) - } - - poolNames.Insert(pool.Name) - if errs := apimachineryvalidation.NameIsDNSLabel(pool.Name, false); len(errs) > 0 { - allErrs = append(allErrs, field.Invalid(fldPath.Child("topology", "pools").Index(i).Child("name"), pool.Name, - fmt.Sprintf("invalid pool name %s", strings.Join(errs, ", ")))) - } - - coreNodeSelectorTerm := &core.NodeSelectorTerm{} - if err := corev1.Convert_v1_NodeSelectorTerm_To_core_NodeSelectorTerm(pool.NodeSelectorTerm.DeepCopy(), coreNodeSelectorTerm, nil); err != nil { - allErrs = append(allErrs, field.Invalid(fldPath.Child("topology", "pools").Index(i).Child("nodeSelectorTerm"), - pool.NodeSelectorTerm, fmt.Sprintf("Convert_v1_NodeSelectorTerm_To_core_NodeSelectorTerm failed: %v", err))) - } else { - allErrs = append(allErrs, apivalidation.ValidateNodeSelectorTerm(*coreNodeSelectorTerm, fldPath.Child("topology", "pools").Index(i).Child("nodeSelectorTerm"))...) - } - - if pool.Tolerations != nil { - var coreTolerations []core.Toleration - for i, toleration := range pool.Tolerations { - coreToleration := &core.Toleration{} - if err := corev1.Convert_v1_Toleration_To_core_Toleration(&toleration, coreToleration, nil); err != nil { - allErrs = append(allErrs, field.Invalid(fldPath.Child("topology", "pools").Index(i).Child("tolerations"), pool.Tolerations, - fmt.Sprintf("Convert_v1_Toleration_To_core_Toleration failed: %v", err))) - } else { - coreTolerations = append(coreTolerations, *coreToleration) - } - } - allErrs = append(allErrs, apivalidation.ValidateTolerations(coreTolerations, fldPath.Child("topology", "pools").Index(i).Child("tolerations"))...) - } - - } - - return allErrs -} - -// validateYurtAppSet validates a YurtAppSet. -func validateYurtAppSet(c client.Client, yurtAppSet *unitv1alpha1.YurtAppSet) field.ErrorList { - allErrs := apivalidation.ValidateObjectMeta(&yurtAppSet.ObjectMeta, true, apimachineryvalidation.NameIsDNSSubdomain, field.NewPath("metadata")) - allErrs = append(allErrs, validateYurtAppSetSpec(c, &yurtAppSet.Spec, field.NewPath("spec"))...) - return allErrs -} - -// ValidateYurtAppSetUpdate tests if required fields in the YurtAppSet are set. -func ValidateYurtAppSetUpdate(yurtAppSet, oldYurtAppSet *unitv1alpha1.YurtAppSet) field.ErrorList { - allErrs := apivalidation.ValidateObjectMetaUpdate(&yurtAppSet.ObjectMeta, &oldYurtAppSet.ObjectMeta, field.NewPath("metadata")) - allErrs = append(allErrs, validateYurtAppSetSpecUpdate(&yurtAppSet.Spec, &oldYurtAppSet.Spec, field.NewPath("spec"))...) - return allErrs -} - -func convertPodTemplateSpec(template *v1.PodTemplateSpec) (*core.PodTemplateSpec, error) { - coreTemplate := &core.PodTemplateSpec{} - if err := corev1.Convert_v1_PodTemplateSpec_To_core_PodTemplateSpec(template.DeepCopy(), coreTemplate, nil); err != nil { - return nil, err - } - return coreTemplate, nil -} - -func validateYurtAppSetSpecUpdate(spec, oldSpec *unitv1alpha1.YurtAppSetSpec, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - allErrs = append(allErrs, validateWorkloadTemplateUpdate(&spec.WorkloadTemplate, &oldSpec.WorkloadTemplate, fldPath.Child("workloadTemplate"))...) - allErrs = append(allErrs, validateYurtAppSetTopology(&spec.Topology, &oldSpec.Topology, fldPath.Child("topology"))...) - return allErrs -} - -func validateYurtAppSetTopology(topology, oldTopology *unitv1alpha1.Topology, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - if topology == nil || oldTopology == nil { - return allErrs - } - - oldPools := map[string]*unitv1alpha1.Pool{} - for i, pool := range oldTopology.Pools { - oldPools[pool.Name] = &oldTopology.Pools[i] - } - - for i, pool := range topology.Pools { - if oldPool, exist := oldPools[pool.Name]; exist { - if !apiequality.Semantic.DeepEqual(oldPool.NodeSelectorTerm, pool.NodeSelectorTerm) { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("pools").Index(i).Child("nodeSelectorTerm"), "may not be changed in an update")) - } - if !apiequality.Semantic.DeepEqual(oldPool.Tolerations, pool.Tolerations) { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("pools").Index(i).Child("tolerations"), "may not be changed in an update")) - } - } - } - return allErrs -} - -func validateWorkloadTemplateUpdate(template, oldTemplate *unitv1alpha1.WorkloadTemplate, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - if template.StatefulSetTemplate != nil && oldTemplate.StatefulSetTemplate != nil { - allErrs = append(allErrs, validateStatefulSetUpdate(template.StatefulSetTemplate, oldTemplate.StatefulSetTemplate, - fldPath.Child("statefulSetTemplate"))...) - } - if template.DeploymentTemplate != nil && oldTemplate.DeploymentTemplate != nil { - allErrs = append(allErrs, validateDeploymentUpdate(template.DeploymentTemplate, oldTemplate.DeploymentTemplate, - fldPath.Child("deploymentTemplate"))...) - } - return allErrs -} - -func validatePoolTemplate(template *unitv1alpha1.WorkloadTemplate, spec *unitv1alpha1.YurtAppSetSpec, - selector labels.Selector, fldPath *field.Path) field.ErrorList { - - allErrs := field.ErrorList{} - - var templateCount int - if template.StatefulSetTemplate != nil { - templateCount++ - } - if template.DeploymentTemplate != nil { - templateCount++ - } - - if templateCount < 1 { - allErrs = append(allErrs, field.Required(fldPath, "should provide one of (statefulSetTemplate/deploymentTemplate/daemonSetTemplate)")) - } else if templateCount > 1 { - allErrs = append(allErrs, field.Invalid(fldPath, template, "should provide only one of (statefulSetTemplate/deploymentTemplate/daemonSetTemplate)")) - } - - if template.StatefulSetTemplate != nil { - labels := labels.Set(template.StatefulSetTemplate.Labels) - if !selector.Matches(labels) { - allErrs = append(allErrs, field.Invalid(fldPath.Child("statefulSetTemplate", "metadata", "labels"), template.StatefulSetTemplate.Labels, "`selector` does not match template `labels`")) - } - allErrs = append(allErrs, validateStatefulSet(template.StatefulSetTemplate, fldPath.Child("statefulSetTemplate"))...) - sstemplate := template.StatefulSetTemplate.Spec.Template - coreTemplate, err := convertPodTemplateSpec(&sstemplate) - if err != nil { - allErrs = append(allErrs, field.Invalid(fldPath.Root(), sstemplate, fmt.Sprintf("Convert_v1_PodTemplateSpec_To_core_PodTemplateSpec failed: %v", err))) - return allErrs - } - allErrs = append(allErrs, appsvalidation.ValidatePodTemplateSpecForStatefulSet(coreTemplate, selector, fldPath.Child("statefulSetTemplate", "spec", "template"), apivalidation.PodValidationOptions{})...) - } - - klog.Infoln("call webhook validatePoolTemplate") - if template.DeploymentTemplate != nil { - klog.Infof("sel:%v, label: %v\n", spec.Selector, spec.WorkloadTemplate.DeploymentTemplate.Labels) - - labels := labels.Set(template.DeploymentTemplate.Labels) - if !selector.Matches(labels) { - klog.Errorf("labels: %v, selector: %v", labels, selector) - allErrs = append(allErrs, field.Invalid(fldPath.Child("deploymentTemplate", "metadata", "labels"), - template.DeploymentTemplate.Labels, "`selector` does not match template `labels`")) - } - allErrs = append(allErrs, validateDeployment(template.DeploymentTemplate, fldPath.Child("deploymentTemplate"))...) - template := template.DeploymentTemplate.Spec.Template - coreTemplate, err := convertPodTemplateSpec(&template) - if err != nil { - allErrs = append(allErrs, field.Invalid(fldPath.Root(), template, fmt.Sprintf("Convert_v1_PodTemplateSpec_To_core_PodTemplateSpec failed: %v", err))) - return allErrs - } - allErrs = append(allErrs, validatePodTemplateSpec(coreTemplate, selector, fldPath.Child("deploymentTemplate", "spec", "template"))...) - allErrs = append(allErrs, apivalidation.ValidatePodTemplateSpec(coreTemplate, - fldPath.Child("deploymentTemplate", "spec", "template"), apivalidation.PodValidationOptions{})...) - } - - return allErrs -} - -func validatePodTemplateSpec(template *core.PodTemplateSpec, selector labels.Selector, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - if template == nil { - allErrs = append(allErrs, field.Required(fldPath, "")) - } else { - if !selector.Empty() { - // Verify that the Deployment selector matches the labels in template. - labels := labels.Set(template.Labels) - if !selector.Matches(labels) { - allErrs = append(allErrs, field.Invalid(fldPath.Child("metadata", "labels"), template.Labels, "`selector` does not match template `labels`")) - } - } - } - return allErrs -} - -func validateStatefulSet(statefulSet *unitv1alpha1.StatefulSetTemplateSpec, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - if statefulSet.Spec.Replicas != nil { - allErrs = append(allErrs, field.Invalid(fldPath.Child("spec", "replicas"), *statefulSet.Spec.Replicas, "replicas in statefulSetTemplate will not be used")) - } - if statefulSet.Spec.UpdateStrategy.Type == appsv1.RollingUpdateStatefulSetStrategyType && - statefulSet.Spec.UpdateStrategy.RollingUpdate != nil && - statefulSet.Spec.UpdateStrategy.RollingUpdate.Partition != nil { - allErrs = append(allErrs, field.Invalid(fldPath.Child("spec", "updateStrategy", "rollingUpdate", "partition"), *statefulSet.Spec.UpdateStrategy.RollingUpdate.Partition, "partition in statefulSetTemplate will not be used")) - } - - return allErrs -} - -func validateDeployment(deployment *unitv1alpha1.DeploymentTemplateSpec, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - if deployment.Spec.Replicas != nil { - allErrs = append(allErrs, field.Invalid(fldPath.Child("spec", "replicas"), *deployment.Spec.Replicas, "replicas in deploymentTemplate will not be used")) - } - return allErrs -} - -func validateDeploymentUpdate(deployment, oldDeployment *unitv1alpha1.DeploymentTemplateSpec, - fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - restoreReplicas := deployment.Spec.Replicas - deployment.Spec.Replicas = oldDeployment.Spec.Replicas - - restoreTemplate := deployment.Spec.Template - deployment.Spec.Template = oldDeployment.Spec.Template - - restoreStrategy := deployment.Spec.Strategy - deployment.Spec.Strategy = oldDeployment.Spec.Strategy - - if !apiequality.Semantic.DeepEqual(deployment.Spec, oldDeployment.Spec) { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("spec"), - "updates to deployTemplate spec for fields other than 'template', 'strategy' and 'replicas' are forbidden")) - } - deployment.Spec.Replicas = restoreReplicas - deployment.Spec.Template = restoreTemplate - deployment.Spec.Strategy = restoreStrategy - - if deployment.Spec.Replicas != nil { - allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(*deployment.Spec.Replicas), - fldPath.Child("spec", "replicas"))...) - } - return allErrs - -} - -func validateStatefulSetUpdate(statefulSet, oldStatefulSet *unitv1alpha1.StatefulSetTemplateSpec, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - restoreReplicas := statefulSet.Spec.Replicas - statefulSet.Spec.Replicas = oldStatefulSet.Spec.Replicas - - restoreTemplate := statefulSet.Spec.Template - statefulSet.Spec.Template = oldStatefulSet.Spec.Template - - restoreStrategy := statefulSet.Spec.UpdateStrategy - statefulSet.Spec.UpdateStrategy = oldStatefulSet.Spec.UpdateStrategy - - if !apiequality.Semantic.DeepEqual(statefulSet.Spec, oldStatefulSet.Spec) { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("spec"), "updates to statefulsetTemplate spec for fields other than 'template', and 'updateStrategy' are forbidden")) - } - statefulSet.Spec.Replicas = restoreReplicas - statefulSet.Spec.Template = restoreTemplate - statefulSet.Spec.UpdateStrategy = restoreStrategy - - if statefulSet.Spec.Replicas != nil { - allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(*statefulSet.Spec.Replicas), fldPath.Child("spec", "replicas"))...) - } - return allErrs -} diff --git a/pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_validation.go b/pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_validation.go deleted file mode 100644 index 5179f244c94..00000000000 --- a/pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_validation.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the License); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an AS IS BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "context" - "fmt" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - - "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" -) - -const YurtAppSetKind = "YurtAppSet" - -// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtAppSetHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { - appset, ok := obj.(*v1alpha1.YurtAppSet) - if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppSet but got a %T", obj)) - } - - if allErrs := validateYurtAppSet(webhook.Client, appset); len(allErrs) > 0 { - return apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), appset.Name, allErrs) - } - - return nil -} - -// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtAppSetHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { - newAppSet, ok := newObj.(*v1alpha1.YurtAppSet) - if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppSet but got a %T", newObj)) - } - oldAppSet, ok := oldObj.(*v1alpha1.YurtAppSet) - if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppSet but got a %T", oldObj)) - } - - validationErrorList := validateYurtAppSet(webhook.Client, newAppSet) - updateErrorList := ValidateYurtAppSetUpdate(newAppSet, oldAppSet) - - if allErrs := append(validationErrorList, updateErrorList...); len(allErrs) > 0 { - return apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), newAppSet.Name, allErrs) - } - - return nil -} - -// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtAppSetHandler) ValidateDelete(_ context.Context, obj runtime.Object) error { - return nil -} diff --git a/pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_default.go b/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_default.go similarity index 65% rename from pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_default.go rename to pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_default.go index a928a0cbea1..2bc67c8c4af 100644 --- a/pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_default.go +++ b/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_default.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1alpha1 +package v1beta1 import ( "context" @@ -22,29 +22,25 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" + utilpointer "k8s.io/utils/pointer" - "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" ) // Default satisfies the defaulting webhook interface. func (webhook *YurtAppSetHandler) Default(ctx context.Context, obj runtime.Object) error { - appset, ok := obj.(*v1alpha1.YurtAppSet) + set, ok := obj.(*v1beta1.YurtAppSet) if !ok { return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppSet but got a %T", obj)) } - v1alpha1.SetDefaultsYurtAppSet(appset) - appset.Status = v1alpha1.YurtAppSetStatus{} - - statefulSetTemp := appset.Spec.WorkloadTemplate.StatefulSetTemplate - deployTem := appset.Spec.WorkloadTemplate.DeploymentTemplate - - if statefulSetTemp != nil { - statefulSetTemp.Spec.Selector = appset.Spec.Selector - } - if deployTem != nil { - deployTem.Spec.Selector = appset.Spec.Selector + if set.Spec.RevisionHistoryLimit == nil { + set.Spec.RevisionHistoryLimit = utilpointer.Int32Ptr(10) + klog.V(4).Info("defaulting YurtAppSet.Spec.RevisionHistoryLimit to 10") } + klog.V(5).Info("received a YurtAppSet: %v", obj) + return nil } diff --git a/pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_handler.go b/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_handler.go similarity index 64% rename from pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_handler.go rename to pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_handler.go index 9b0efc7085b..aa6f8949d41 100644 --- a/pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_handler.go +++ b/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_handler.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1alpha1 +package v1beta1 import ( ctrl "sigs.k8s.io/controller-runtime" @@ -22,7 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/webhook" - "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) @@ -31,23 +31,23 @@ func (webhook *YurtAppSetHandler) SetupWebhookWithManager(mgr ctrl.Manager) (str // init webhook.Client = mgr.GetClient() - gvk, err := apiutil.GVKForObject(&v1alpha1.YurtAppSet{}, mgr.GetScheme()) + gvk, err := apiutil.GVKForObject(&v1beta1.YurtAppSet{}, mgr.GetScheme()) if err != nil { return "", "", err } return util.GenerateMutatePath(gvk), util.GenerateValidatePath(gvk), ctrl.NewWebhookManagedBy(mgr). - For(&v1alpha1.YurtAppSet{}). + For(&v1beta1.YurtAppSet{}). WithDefaulter(webhook). WithValidator(webhook). Complete() } -// +kubebuilder:webhook:path=/validate-apps-openyurt-io-v1alpha1-yurtappset,mutating=false,failurePolicy=fail,groups=apps.openyurt.io,resources=yurtappsets,verbs=create;update,versions=v1alpha1,name=vyurtappset.kb.io,sideEffects=None,admissionReviewVersions=v1 -// +kubebuilder:webhook:path=/mutate-apps-openyurt-io-v1alpha1-yurtappset,mutating=true,failurePolicy=fail,groups=apps.openyurt.io,resources=yurtappsets,verbs=create;update,versions=v1alpha1,name=myurtappset.kb.io,sideEffects=None,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/validate-apps-openyurt-io-v1beta1-yurtappset,mutating=false,failurePolicy=fail,groups=apps.openyurt.io,resources=yurtappsets,verbs=create;update,versions=v1beta1,name=vyurtappset.kb.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// +kubebuilder:webhook:path=/mutate-apps-openyurt-io-v1beta1-yurtappset,mutating=true,failurePolicy=fail,groups=apps.openyurt.io,resources=yurtappsets,verbs=create;update,versions=v1beta1,name=myurtappset.kb.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 -// Cluster implements a validating and defaulting webhook for Cluster. +// YurtAppSetHandler implements a validating and defaulting webhook for Cluster. type YurtAppSetHandler struct { Client client.Client } diff --git a/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_validation.go b/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_validation.go new file mode 100644 index 00000000000..04881c83576 --- /dev/null +++ b/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_validation.go @@ -0,0 +1,84 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" +) + +const YurtAppSetKind = "YurtAppSet" + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *YurtAppSetHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { + set, ok := obj.(*v1beta1.YurtAppSet) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppSet but got a %T", obj)) + } + + template := set.Spec.Workload.WorkloadTemplate + if template.DeploymentTemplate == nil && template.StatefulSetTemplate == nil { + return apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), set.Name, + field.ErrorList{field.Invalid(field.NewPath("spec").Child("workload").Child("WorkloadTemplate"), template, "no workload template is configured")}) + } else if template.DeploymentTemplate != nil && template.StatefulSetTemplate != nil { + return apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), set.Name, + field.ErrorList{field.Invalid(field.NewPath("spec").Child("workload").Child("WorkloadTemplate"), template, "only one workload template should be configured")}) + } + + return nil +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *YurtAppSetHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { + newSet, ok := newObj.(*v1beta1.YurtAppSet) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppSet but got a %T", newObj)) + } + oldSet, ok := oldObj.(*v1beta1.YurtAppSet) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppSet but got a %T", oldObj)) + } + + newTemplate := newSet.Spec.Workload.WorkloadTemplate + if newTemplate.DeploymentTemplate == nil && newTemplate.StatefulSetTemplate == nil { + return apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), newSet.Name, + field.ErrorList{field.Invalid(field.NewPath("spec").Child("workload").Child("WorkloadTemplate"), newTemplate, "no workload template is configured")}) + } else if newTemplate.DeploymentTemplate != nil && newTemplate.StatefulSetTemplate != nil { + return apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), newSet.Name, + field.ErrorList{field.Invalid(field.NewPath("spec").Child("workload").Child("WorkloadTemplate"), newTemplate, "only one workload template should be configured")}) + } + + oldTemplate := oldSet.Spec.Workload.WorkloadTemplate + if (oldTemplate.DeploymentTemplate == nil && newTemplate.DeploymentTemplate != nil) || + (oldTemplate.StatefulSetTemplate == nil && newTemplate.StatefulSetTemplate != nil) { + return apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), newSet.Name, + field.ErrorList{field.Invalid(field.NewPath("spec").Child("workload").Child("WorkloadTemplate"), newTemplate, "the kind of workload template should not be changed")}) + } + + return nil +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *YurtAppSetHandler) ValidateDelete(_ context.Context, obj runtime.Object) error { + return nil +} diff --git a/pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_webhook_test.go b/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_webhook_test.go similarity index 62% rename from pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_webhook_test.go rename to pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_webhook_test.go index 8249ec3aff3..3ae74f01c12 100644 --- a/pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_webhook_test.go +++ b/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_webhook_test.go @@ -12,7 +12,7 @@ limitations under the License. */ // +kubebuilder:docs-gen:collapse=Apache License -package v1alpha1 +package v1beta1 import ( "context" @@ -22,31 +22,31 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" ) -var defaultAppSet = &v1alpha1.YurtAppSet{ +var defaultAppSet = &v1beta1.YurtAppSet{ ObjectMeta: metav1.ObjectMeta{ Name: "fooboo", Namespace: "default", }, - Spec: v1alpha1.YurtAppSetSpec{ - Topology: v1alpha1.Topology{Pools: []v1alpha1.Pool{{Name: "beijing"}}}, - Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "demo"}}, - WorkloadTemplate: v1alpha1.WorkloadTemplate{ - DeploymentTemplate: &v1alpha1.DeploymentTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "demo"}, - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "demo"}}, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "demo"}, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - {Name: "demo", Image: "nginx"}, + Spec: v1beta1.YurtAppSetSpec{ + Workload: v1beta1.Workload{ + WorkloadTemplate: v1beta1.WorkloadTemplate{ + DeploymentTemplate: &v1beta1.DeploymentTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "demo"}, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "demo"}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "demo"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "demo", Image: "nginx"}, + }, }, }, }, @@ -80,14 +80,13 @@ func TestYurtAppSetValidator(t *testing.T) { } dupTopology := defaultAppSet.DeepCopy() - dupTopology.Spec.Topology = v1alpha1.Topology{Pools: []v1alpha1.Pool{{Name: "beijing"}, {Name: "beijing"}}} - if err := webhook.ValidateCreate(context.TODO(), dupTopology); err == nil { + if err := webhook.ValidateCreate(context.TODO(), dupTopology); err != nil { t.Fatal("topology dup should not fail") } updateAppSet := defaultAppSet.DeepCopy() updateAppSet.Spec.WorkloadTemplate.DeploymentTemplate.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"app": "demo2"}} - if err := webhook.ValidateUpdate(context.TODO(), defaultAppSet, updateAppSet); err == nil { - t.Fatal("workload selector change should fail") + if err := webhook.ValidateUpdate(context.TODO(), defaultAppSet, updateAppSet); err != nil { + t.Fatal("workload selector change should not fail") } }