diff --git a/charts/openyurt/templates/yurt-manager-auto-generated.yaml b/charts/openyurt/templates/yurt-manager-auto-generated.yaml index 2a6a1c3ae91..8419a3ba5e4 100644 --- a/charts/openyurt/templates/yurt-manager-auto-generated.yaml +++ b/charts/openyurt/templates/yurt-manager-auto-generated.yaml @@ -5,6 +5,160 @@ # # --------------------------------------------------- +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + name: staticpods.apps.openyurt.io +spec: + group: apps.openyurt.io + names: + categories: + - all + kind: StaticPod + listKind: StaticPodList + plural: staticpods + shortNames: + - sp + singular: staticpod + scope: Cluster + versions: + - additionalPrinterColumns: + - description: CreationTimestamp is a timestamp representing the server time when + this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. + jsonPath: .metadata.creationTimestamp + name: AGE + type: date + - description: The total number of static pods + jsonPath: .status.totalNumber + name: TotalNumber + type: integer + - description: The number of static pods that desired to be upgraded + jsonPath: .status.desiredNumber + name: DesiredNumber + type: integer + - description: The number of static pods that have been upgraded + jsonPath: .status.upgradedNumber + name: UpgradedNumber + type: integer + - jsonPath: .status.conditions[0].type + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: StaticPod is the Schema for the staticpods API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: StaticPodSpec defines the desired state of StaticPod + properties: + staticPodManifest: + description: StaticPodManifest indicates the Static Pod desired to + be upgraded. The corresponding manifest file name is `StaticPodManifest.yaml`. + type: string + staticPodName: + description: StaticPodName indicates the static pod desired to be + upgraded. + type: string + staticPodNamespace: + description: Namespace indicates the namespace of target static pod + type: string + template: + description: An object that describes the desired upgrade static pod. + x-kubernetes-preserve-unknown-fields: true + upgradeStrategy: + description: An upgrade strategy to replace existing static pods with + new ones. + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: Auto upgrade config params. Present only if type + = "auto". + x-kubernetes-int-or-string: true + type: + description: Type of Static Pod upgrade. Can be "auto" or "ota". + type: string + type: object + required: + - staticPodName + type: object + status: + description: StaticPodStatus defines the observed state of StaticPod + properties: + conditions: + description: Represents the latest available observations of StaticPod's + current state. + items: + description: StaticPodCondition describes the state of a StaticPodCondition + at a certain point. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + message: + description: A human-readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of StaticPod condition. + type: string + type: object + type: array + desiredNumber: + description: The number of static pods that should be upgraded. + format: int32 + type: integer + totalNumber: + description: The total number of static pods + format: int32 + type: integer + upgradedNumber: + description: The number of static pods that have been upgraded. + format: int32 + type: integer + required: + - desiredNumber + - totalNumber + - upgradedNumber + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -42,6 +196,32 @@ rules: - patch - update - watch +- apiGroups: + - apps.openyurt.io + resources: + - staticpods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.openyurt.io + resources: + - staticpods/finalizers + verbs: + - update +- apiGroups: + - apps.openyurt.io + resources: + - staticpods/status + verbs: + - get + - patch + - update - apiGroups: - coordination.k8s.io resources: @@ -54,6 +234,18 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -66,6 +258,28 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -78,3 +292,59 @@ rules: - patch - update - watch +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + creationTimestamp: null + name: yurt-manager-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: kube-system + path: /mutate-apps-openyurt-io-v1alpha1-staticpod + failurePolicy: Fail + name: mutate.apps.v1alpha1.staticpod.openyurt.io + rules: + - apiGroups: + - apps.openyurt.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - staticpods + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + creationTimestamp: null + name: yurt-manager-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: kube-system + path: /validate-apps-openyurt-io-v1alpha1-staticpod + failurePolicy: Fail + name: validate.apps.v1alpha1.staticpod.openyurt.io + rules: + - apiGroups: + - apps.openyurt.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - staticpods + sideEffects: None diff --git a/go.mod b/go.mod index fc5bb96aa9a..00876617647 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( k8s.io/api v0.22.3 k8s.io/apimachinery v0.22.3 k8s.io/apiserver v0.22.3 + k8s.io/cli-runtime v0.22.3 k8s.io/client-go v0.22.3 k8s.io/cluster-bootstrap v0.22.3 k8s.io/component-base v0.22.3 @@ -75,6 +76,7 @@ require ( github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.2 // indirect diff --git a/go.sum b/go.sum index 7deb0dfbbc8..3452d858d38 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,7 @@ github.com/aliyun/alibaba-cloud-sdk-go v1.62.156 h1:K4N91T1+RlSlx+t2dujeDviy4ehS github.com/aliyun/alibaba-cloud-sdk-go v1.62.156/go.mod h1:Api2AkmMgGaSUAhmk76oaFObkoeCPc/bKAqcyplPODs= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -117,6 +118,7 @@ github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -178,6 +180,7 @@ github.com/fsnotify/fsnotify v1.4.10-0.20200417215612-7f4cf4dd2b52/go.mod h1:znq github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -294,6 +297,7 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -312,6 +316,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -393,6 +398,9 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -432,6 +440,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -524,6 +533,7 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -542,6 +552,7 @@ github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= @@ -552,6 +563,7 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -579,6 +591,7 @@ github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaO github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vishvananda/netlink v1.1.1-0.20200603190939-5a869a71f0cb h1:MY3XXjEi7+I9L6iwK4x0KWNL9OaWMQ5CntP06o+8zZc= github.com/vishvananda/netlink v1.1.1-0.20200603190939-5a869a71f0cb/go.mod h1:FSQhuTO7eHT34mPzX+B04SUAjiqLxtXs1et0S6l9k4k= @@ -586,6 +599,8 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7Zo github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -643,6 +658,7 @@ go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52l go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -722,6 +738,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -810,6 +827,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1027,6 +1045,7 @@ google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMo google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -1128,6 +1147,8 @@ k8s.io/apiserver v0.21.1/go.mod h1:nLLYZvMWn35glJ4/FZRhzLG/3MPxAaZTgV4FJZdr+tY= k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= k8s.io/apiserver v0.22.3 h1:x21xyLQ2qvPr5vjOTVOBaSJu8svnU2wfLOfSjNJEOdw= k8s.io/apiserver v0.22.3/go.mod h1:oam7lH/F1Kto/WTamyQYrD68fS0mGUBORAFf6x/9Mxs= +k8s.io/cli-runtime v0.22.3 h1:AeOgaDpb/k36amWsjyyIU+FLpLzzdmoLD5gn38c5fio= +k8s.io/cli-runtime v0.22.3/go.mod h1:um6JvCxV9Hrhq0zCUxcqYoY7/wF64g6IYgOViI8sg6Q= k8s.io/client-go v0.18.8/go.mod h1:HqFqMllQ5NnQJNwjro9k5zMyfhZlOwpuTLVrxjkYSxU= k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs= k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= @@ -1174,6 +1195,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyz sigs.k8s.io/controller-runtime v0.9.0/go.mod h1:TgkfvrhhEw3PlI0BRL/5xM+89y3/yc0ZDfdbTl84si8= sigs.k8s.io/controller-runtime v0.10.3 h1:s5Ttmw/B4AuIbwrXD3sfBkXwnPMMWrqpVj4WRt1dano= sigs.k8s.io/controller-runtime v0.10.3/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= +sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g= +sigs.k8s.io/kustomize/kyaml v0.11.0/go.mod h1:GNMwjim4Ypgp/MueD3zXHLRJEjz7RvtPae0AwlvEMFM= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/pkg/apis/addtoscheme_apps_v1alpha1.go b/pkg/apis/addtoscheme_apps_v1alpha1.go new file mode 100644 index 00000000000..d1dc94bc526 --- /dev/null +++ b/pkg/apis/addtoscheme_apps_v1alpha1.go @@ -0,0 +1,26 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apis + +import ( + version "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +func init() { + // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back + AddToSchemes = append(AddToSchemes, version.SchemeBuilder.AddToScheme) +} diff --git a/pkg/apis/apps/v1alpha1/default.go b/pkg/apis/apps/v1alpha1/default.go new file mode 100644 index 00000000000..3a8ef9a5d34 --- /dev/null +++ b/pkg/apis/apps/v1alpha1/default.go @@ -0,0 +1,42 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// SetDefaultsStaticPod set default values for StaticPod. +func SetDefaultsStaticPod(obj *StaticPod) { + // 1. Set default max-unavailable to "10%" in auto mode if it's not set + strategy := obj.Spec.UpgradeStrategy.DeepCopy() + if strategy != nil && strategy.Type == AutoStaticPodUpgradeStrategyType && strategy.MaxUnavailable == nil { + v := intstr.FromString("10%") + obj.Spec.UpgradeStrategy.MaxUnavailable = &v + } + + // 2. Set StaticPodNamespace to `default` + if obj.Spec.StaticPodNamespace == "" { + obj.Spec.StaticPodNamespace = metav1.NamespaceDefault + } + + // 3. Set StaticPodManifest to the same as StaticPodName if it's not set + if obj.Spec.StaticPodManifest == "" { + obj.Spec.StaticPodManifest = obj.Spec.StaticPodName + } +} diff --git a/pkg/apis/apps/v1alpha1/doc.go b/pkg/apis/apps/v1alpha1/doc.go new file mode 100644 index 00000000000..d14b5ee8b4a --- /dev/null +++ b/pkg/apis/apps/v1alpha1/doc.go @@ -0,0 +1,17 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 diff --git a/pkg/apis/apps/v1alpha1/groupversion_info.go b/pkg/apis/apps/v1alpha1/groupversion_info.go new file mode 100644 index 00000000000..7b37ca91b88 --- /dev/null +++ b/pkg/apis/apps/v1alpha1/groupversion_info.go @@ -0,0 +1,44 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// Package v1alpha1 contains API Schema definitions for the apps v1alpha1API group +// +kubebuilder:object:generate=true +// +groupName=apps.openyurt.io + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "apps.openyurt.io", Version: "v1alpha1"} + + SchemeGroupVersion = GroupVersion + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +// Resource is required by pkg/client/listers/... +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} diff --git a/pkg/apis/apps/v1alpha1/staticpod_types.go b/pkg/apis/apps/v1alpha1/staticpod_types.go new file mode 100644 index 00000000000..080b27b5c3b --- /dev/null +++ b/pkg/apis/apps/v1alpha1/staticpod_types.go @@ -0,0 +1,145 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// StaticPodUpgradeStrategy defines a strategy to upgrade a static pod. +type StaticPodUpgradeStrategy struct { + // Type of Static Pod upgrade. Can be "auto" or "ota". + Type StaticPodUpgradeStrategyType `json:"type,omitempty"` + + // Auto upgrade config params. Present only if type = "auto". + //+optional + MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` +} + +// StaticPodUpgradeStrategyType is a strategy according to which a static pod gets upgraded. +type StaticPodUpgradeStrategyType string + +const ( + AutoStaticPodUpgradeStrategyType StaticPodUpgradeStrategyType = "auto" + OTAStaticPodUpgradeStrategyType StaticPodUpgradeStrategyType = "ota" +) + +// StaticPodSpec defines the desired state of StaticPod +type StaticPodSpec struct { + // StaticPodName indicates the static pod desired to be upgraded. + StaticPodName string `json:"staticPodName"` + + // StaticPodManifest indicates the Static Pod desired to be upgraded. The corresponding + // manifest file name is `StaticPodManifest.yaml`. + // +optional + StaticPodManifest string `json:"staticPodManifest,omitempty"` + + // Namespace indicates the namespace of target static pod + // +optional + StaticPodNamespace string `json:"staticPodNamespace,omitempty"` + + // An upgrade strategy to replace existing static pods with new ones. + UpgradeStrategy StaticPodUpgradeStrategy `json:"upgradeStrategy,omitempty"` + + // An object that describes the desired upgrade static pod. + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + Template corev1.PodTemplateSpec `json:"template,omitempty"` +} + +type StaticPodConditionType string + +const ( + // StaticPodUpgradeSuccess means static pods on all nodes have been upgraded to the latest version + StaticPodUpgradeSuccess StaticPodConditionType = "UpgradeSuccess" + + // StaticPodUpgradeExecuting means static pods upgrade task is in progress + StaticPodUpgradeExecuting StaticPodConditionType = "Upgrading" + + // StaticPodUpgradeFailed means that exist pods failed to upgrade during the upgrade process + StaticPodUpgradeFailed StaticPodConditionType = "UpgradeFailed" +) + +// StaticPodCondition describes the state of a StaticPodCondition at a certain point. +type StaticPodCondition struct { + // Type of StaticPod condition. + Type StaticPodConditionType `json:"type,omitempty"` + + // Status of the condition, one of True, False, Unknown. + Status corev1.ConditionStatus `json:"status,omitempty"` + + // Last time the condition transitioned from one status to another. + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + + // The reason for the condition's last transition. + Reason string `json:"reason,omitempty"` + + // A human-readable message indicating details about the transition. + Message string `json:"message,omitempty"` +} + +// StaticPodStatus defines the observed state of StaticPod +type StaticPodStatus struct { + // The total number of static pods + TotalNumber int32 `json:"totalNumber"` + + // The number of static pods that should be upgraded. + DesiredNumber int32 `json:"desiredNumber"` + + // The number of static pods that have been upgraded. + UpgradedNumber int32 `json:"upgradedNumber"` + + // Represents the latest available observations of StaticPod's current state. + // +optional + Conditions []StaticPodCondition `json:"conditions"` +} + +// +genclient +// +k8s:openapi-gen=true +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster,path=staticpods,shortName=sp,categories=all +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC." +//+kubebuilder:printcolumn:name="TotalNumber",type="integer",JSONPath=".status.totalNumber",description="The total number of static pods" +//+kubebuilder:printcolumn:name="DesiredNumber",type="integer",JSONPath=".status.desiredNumber",description="The number of static pods that desired to be upgraded" +//+kubebuilder:printcolumn:name="UpgradedNumber",type="integer",JSONPath=".status.upgradedNumber",description="The number of static pods that have been upgraded" +//+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].type" + +// StaticPod is the Schema for the staticpods API +type StaticPod struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec StaticPodSpec `json:"spec,omitempty"` + Status StaticPodStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// StaticPodList contains a list of StaticPod +type StaticPodList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []StaticPod `json:"items"` +} + +func init() { + SchemeBuilder.Register(&StaticPod{}, &StaticPodList{}) +} diff --git a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000000..7d1d391fb55 --- /dev/null +++ b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,161 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticPod) DeepCopyInto(out *StaticPod) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticPod. +func (in *StaticPod) DeepCopy() *StaticPod { + if in == nil { + return nil + } + out := new(StaticPod) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StaticPod) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticPodCondition) DeepCopyInto(out *StaticPodCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticPodCondition. +func (in *StaticPodCondition) DeepCopy() *StaticPodCondition { + if in == nil { + return nil + } + out := new(StaticPodCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticPodList) DeepCopyInto(out *StaticPodList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]StaticPod, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticPodList. +func (in *StaticPodList) DeepCopy() *StaticPodList { + if in == nil { + return nil + } + out := new(StaticPodList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StaticPodList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticPodSpec) DeepCopyInto(out *StaticPodSpec) { + *out = *in + in.UpgradeStrategy.DeepCopyInto(&out.UpgradeStrategy) + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticPodSpec. +func (in *StaticPodSpec) DeepCopy() *StaticPodSpec { + if in == nil { + return nil + } + out := new(StaticPodSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticPodStatus) DeepCopyInto(out *StaticPodStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]StaticPodCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticPodStatus. +func (in *StaticPodStatus) DeepCopy() *StaticPodStatus { + if in == nil { + return nil + } + out := new(StaticPodStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticPodUpgradeStrategy) DeepCopyInto(out *StaticPodUpgradeStrategy) { + *out = *in + if in.MaxUnavailable != nil { + in, out := &in.MaxUnavailable, &out.MaxUnavailable + *out = new(intstr.IntOrString) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticPodUpgradeStrategy. +func (in *StaticPodUpgradeStrategy) DeepCopy() *StaticPodUpgradeStrategy { + if in == nil { + return nil + } + out := new(StaticPodUpgradeStrategy) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/controller/add_staticpod.go b/pkg/controller/add_staticpod.go new file mode 100644 index 00000000000..eae1f74183e --- /dev/null +++ b/pkg/controller/add_staticpod.go @@ -0,0 +1,30 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "github.com/openyurtio/openyurt/pkg/controller/staticpod" +) + +// Note !!! @kadisi +// Do not change the name of the file @kadisi +// Auto generate by make addcontroller command !!! +// Note !!! + +func init() { + controllerAddFuncs = append(controllerAddFuncs, staticpod.Add) +} diff --git a/pkg/controller/staticpod/staticpod_controller.go b/pkg/controller/staticpod/staticpod_controller.go new file mode 100644 index 00000000000..e0bd900fe3f --- /dev/null +++ b/pkg/controller/staticpod/staticpod_controller.go @@ -0,0 +1,585 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package staticpod + +import ( + "context" + "flag" + "fmt" + + corev1 "k8s.io/api/core/v1" + kerr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/controller/staticpod/upgradeinfo" + "github.com/openyurtio/openyurt/pkg/controller/staticpod/util" + utilclient "github.com/openyurtio/openyurt/pkg/util/client" + utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery" +) + +func init() { + flag.IntVar(&concurrentReconciles, "staticpod-workers", concurrentReconciles, "Max concurrent workers for StaticPod controller.") +} + +var ( + concurrentReconciles = 3 + controllerKind = appsv1alpha1.SchemeGroupVersion.WithKind("StaticPod") + True = true + upgradeSuccessCondition = util.NewStaticPodCondition(appsv1alpha1.StaticPodUpgradeSuccess, corev1.ConditionTrue, "", "") + upgradeExecutingCondition = util.NewStaticPodCondition(appsv1alpha1.StaticPodUpgradeExecuting, corev1.ConditionTrue, "", "") +) + +const ( + controllerName = "StaticPod-controller" + + StaticPodHashAnnotation = "openyurt.io/static-pod-hash" + + hostPathVolumeName = "hostpath" + hostPathVolumeMountPath = "/etc/kubernetes/manifests/" + configMapVolumeName = "configmap" + configMapVolumeMountPath = "/data" + hostPathVolumeSourcePath = hostPathVolumeMountPath + + // UpgradeWorkerPodPrefix is the name prefix of worker pod which used for static pod upgrade + UpgradeWorkerPodPrefix = "yurt-static-pod-upgrade-worker-" + UpgradeWorkerContainerName = "upgrade-worker" + UpgradeWorkerImage = "openyurt/yurt-static-pod-upgrade:latest" + UpgradeServiceAccount = "yurt-manager" + + ArgTmpl = "/usr/local/bin/yurt-static-pod-upgrade --name=%s --manifest=%s --hash=%s --namespace=%s --mode=%s" +) + +// upgradeWorker is the pod template used for static pod upgrade +// Fields need be set +// 1. name of worker pod: `yurt-static-pod-upgrade-worker-node-hash` +// 2. node of worker pod +// 3. annotation `openyurt.io/static-pod-hash` +// 4. the corresponding configmap +var ( + upgradeWorker = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem}, + Spec: corev1.PodSpec{ + HostPID: true, + HostNetwork: true, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: UpgradeServiceAccount, + Containers: []corev1.Container{{ + Name: UpgradeWorkerContainerName, + Command: []string{"/bin/sh", "-c"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: hostPathVolumeName, + MountPath: hostPathVolumeMountPath, + }, + { + Name: configMapVolumeName, + MountPath: configMapVolumeMountPath, + }, + }, + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: &corev1.SecurityContext{ + Privileged: &True, + }, + Image: UpgradeWorkerImage, + }}, + Volumes: []corev1.Volume{{ + Name: hostPathVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: hostPathVolumeSourcePath, + }, + }}, + }, + }, + } +) + +func Format(format string, args ...interface{}) string { + s := fmt.Sprintf(format, args...) + return fmt.Sprintf("%s: %s", controllerName, s) +} + +// Add creates a new StaticPod Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager) error { + if !utildiscovery.DiscoverGVK(controllerKind) { + return nil + } + return add(mgr, newReconciler(mgr)) +} + +var _ reconcile.Reconciler = &ReconcileStaticPod{} + +// ReconcileStaticPod reconciles a StaticPod object +type ReconcileStaticPod struct { + client.Client + scheme *runtime.Scheme + recorder record.EventRecorder +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + return &ReconcileStaticPod{ + Client: utilclient.NewClientFromManager(mgr, controllerName), + scheme: mgr.GetScheme(), + recorder: mgr.GetEventRecorderFor(controllerName), + } +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + if err != nil { + return err + } + + // 1. Watch for changes to StaticPod + if err := c.Watch(&source.Kind{Type: &appsv1alpha1.StaticPod{}}, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + + // 2. Watch for changes to node + // When node turn ready, reconcile all StaticPod instances + // nodeReadyPredicate filter events which are node turn ready + nodeReadyPredicate := predicate.Funcs{ + UpdateFunc: func(evt event.UpdateEvent) bool { + return nodeTurnReady(evt) + }, + } + + reconcileAllStaticPods := func(c client.Client) []reconcile.Request { + staticPodList := &appsv1alpha1.StaticPodList{} + if err := c.List(context.TODO(), staticPodList); err != nil { + return nil + } + var requests []reconcile.Request + for _, staticPod := range staticPodList.Items { + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Name: staticPod.Name, + }}) + } + return requests + } + + if err := c.Watch(&source.Kind{Type: &corev1.Node{}}, + handler.EnqueueRequestsFromMapFunc( + func(client.Object) []reconcile.Request { + return reconcileAllStaticPods(mgr.GetClient()) + }), nodeReadyPredicate); err != nil { + return err + } + + // 3. Watch for changes to upgrade worker pods which are created by static-pod-controller + if err := c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{IsController: true, OwnerType: &appsv1alpha1.StaticPod{}}); err != nil { + return err + } + + return nil +} + +// nodeTurnReady filter events: old node is not-ready or unknown, new node is ready +func nodeTurnReady(evt event.UpdateEvent) bool { + if _, ok := evt.ObjectOld.(*corev1.Node); !ok { + return false + } + + oldNode := evt.ObjectOld.(*corev1.Node) + newNode := evt.ObjectNew.(*corev1.Node) + + _, onc := util.GetNodeCondition(&oldNode.Status, corev1.NodeReady) + _, nnc := util.GetNodeCondition(&newNode.Status, corev1.NodeReady) + + oldReady := (onc != nil) && ((onc.Status == corev1.ConditionFalse) || (onc.Status == corev1.ConditionUnknown)) + newReady := (nnc != nil) && (nnc.Status == corev1.ConditionTrue) + + return oldReady && newReady +} + +//+kubebuilder:rbac:groups=apps.openyurt.io,resources=staticpods,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=apps.openyurt.io,resources=staticpods/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=apps.openyurt.io,resources=staticpods/finalizers,verbs=update +//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch;update;patch +//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete + +// Reconcile reads that state of the cluster for a StaticPod object and makes changes based on the state read +// and what is in the StaticPod.Spec +func (r *ReconcileStaticPod) Reconcile(_ context.Context, request reconcile.Request) (reconcile.Result, error) { + + // Note !!!!!!!!!! + // We strongly recommend use Format() to encapsulation because Format() can print logs by module + // @kadisi + klog.V(4).Infof(Format("Reconcile StaticPod %s", request.Name)) + + // Fetch the StaticPod instance + instance := &appsv1alpha1.StaticPod{} + if err := r.Get(context.TODO(), request.NamespacedName, instance); err != nil { + klog.Errorf("Fail to get StaticPod %v, %v", request.NamespacedName.Name, err) + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if instance.DeletionTimestamp != nil { + return reconcile.Result{}, nil + } + + var ( + // totalNumber represents the total number of nodes running the target static pod + totalNumber int32 + + // desiredNumber represents the desired upgraded number of nodes running the target static pod + // In auto mode: it's the number of ready nodes running the target static pod + // In ota mode: it's equal to totalNumber + desiredNumber int32 + + // upgradedNumber represents the number of nodes that have been upgraded + upgradedNumber int32 + ) + + // The later upgrade operation is conducted based on upgradeInfos + upgradeInfos, err := upgradeinfo.New(r.Client, instance, UpgradeWorkerPodPrefix) + if err != nil { + klog.Errorf(Format("Fail to get static pod and worker pod upgrade info for nodes of StaticPod %v, %v", + request.NamespacedName.Name, err)) + return ctrl.Result{}, err + } + totalNumber = int32(len(upgradeInfos)) + // There are no nodes running target static pods in the cluster + if totalNumber == 0 { + klog.Infof(Format("No static pods need to be upgraded of StaticPod %v", request.NamespacedName.Name)) + return r.updateStaticPodStatus(instance, totalNumber, totalNumber, totalNumber, upgradeSuccessCondition) + } + + // The latest hash value for static pod spec + // This hash value is used in three places + // 1. Automatically added to the annotation of static pods to facilitate checking if the running static pods are up-to-date + // 2. Automatically added to the annotation of worker pods to facilitate checking if the worker pods are up-to-date + // 3. Added to static pods' corresponding configmap to facilitate checking if the configmap is up-to-date + latestHash := util.ComputeHash(&instance.Spec.Template) + + // The latest static pod manifest generated from user-specified template + // The above hash value will be added to the annotation + latestManifest, err := util.GenStaticPodManifest(&instance.Spec.Template, latestHash) + if err != nil { + klog.Errorf(Format("Fail to generate static pod manifest of StaticPod %v, %v", request.NamespacedName.Name, err)) + return ctrl.Result{}, err + } + + // Sync the corresponding configmap to the latest state + if err := r.syncConfigMap(instance, latestHash, latestManifest); err != nil { + klog.Errorf(Format("Fail to sync the corresponding configmap of StaticPod %v, %v", request.NamespacedName.Name, err)) + return ctrl.Result{}, err + } + + // Count the number of upgraded nodes + upgradedNumber = setUpgradeNeededInfos(upgradeInfos, latestHash) + + // Check node ready info + if err := checkReadyNodes(r.Client, upgradeInfos); err != nil { + klog.Errorf(Format("Fail to check node ready status of StaticPod %v,%v", request.NamespacedName.Name, err)) + return ctrl.Result{}, err + } + + // allSucceeded flag is used to indicate whether the worker pods in last round have been all succeeded. + // In auto upgrade mode, if the value is false, it will wait util all the worker pods succeed. + failedNode, allSucceeded, deletePods := checkWorkerPodInfos(upgradeInfos, latestHash) + // If node is not empty, it means the worker pod failed in this node. + if failedNode != "" { + klog.Errorf(Format("Fail to continue upgrade, cause worker pod of StaticPod %v in node %s failed", request.NamespacedName.Name, failedNode)) + return r.updateStaticPodStatus(instance, totalNumber, instance.Status.DesiredNumber, upgradedNumber, util.UpgradeFailedConditionWithNode(failedNode)) + } + + // Clean up the unused worker pods + if err := r.removeUnusedPods(deletePods); err != nil { + klog.Errorf("Fail to delete out-of-date worker pods of StaticPod %v, %v", request.NamespacedName.Name, err) + return ctrl.Result{}, nil + } + + // If all nodes have been upgraded, just return + if totalNumber == upgradedNumber { + klog.Infof(Format("All static pods have been upgraded of StaticPod %v", request.NamespacedName.Name)) + return r.updateStaticPodStatus(instance, totalNumber, instance.Status.DesiredNumber, upgradedNumber, upgradeSuccessCondition) + } + + switch instance.Spec.UpgradeStrategy.Type { + // Auto Upgrade is to automate the upgrade process for the target static pods on ready nodes + // It supports rolling update and the max-unavailable number can be specified by users + case appsv1alpha1.AutoStaticPodUpgradeStrategyType: + // In auto upgrade mode, desiredNumber is the number of ready nodes + desiredNumber = int32(len(upgradeinfo.ReadyNodes(upgradeInfos))) + // This means that all the desired nodes are upgraded. It's considered successful. + if desiredNumber == upgradedNumber { + return r.updateStaticPodStatus(instance, totalNumber, desiredNumber, upgradedNumber, upgradeSuccessCondition) + } + + if !allSucceeded { + klog.V(5).Infof(Format("Wait last round auto upgrade to finish of StaticPod %v", request.NamespacedName.Name)) + return r.updateStaticPodStatus(instance, totalNumber, desiredNumber, upgradedNumber, upgradeExecutingCondition) + } + + if err := r.autoUpgrade(instance, upgradeInfos, latestHash); err != nil { + klog.Errorf(Format("Fail to auto upgrade of StaticPod %v, %v", request.NamespacedName.Name, err)) + return ctrl.Result{}, err + } + return r.updateStaticPodStatus(instance, totalNumber, desiredNumber, upgradedNumber, upgradeExecutingCondition) + + // OTA Upgrade can help users control the timing of static pods upgrade + // It will set PodNeedUpgrade condition and work with YurtHub component + case appsv1alpha1.OTAStaticPodUpgradeStrategyType: + if err := r.otaUpgrade(instance, upgradeInfos, latestHash); err != nil { + klog.Errorf(Format("Fail to ota upgrade of StaticPod %v, %v", request.NamespacedName.Name, err)) + return ctrl.Result{}, err + } + return r.updateStaticPodStatus(instance, totalNumber, totalNumber, upgradedNumber, upgradeExecutingCondition) + } + + return ctrl.Result{}, nil +} + +// syncConfigMap moves the target static pod's corresponding configmap to the latest state +func (r *ReconcileStaticPod) syncConfigMap(instance *appsv1alpha1.StaticPod, hash, data string) error { + cmName := util.WithConfigMapPrefix(util.Hyphen(instance.Spec.StaticPodNamespace, instance.Spec.StaticPodName)) + cm := &corev1.ConfigMap{} + if err := r.Get(context.TODO(), types.NamespacedName{Name: cmName, Namespace: metav1.NamespaceSystem}, cm); err != nil { + // if the configmap does not exist, then create a new one + if kerr.IsNotFound(err) { + cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: metav1.NamespaceSystem, + Annotations: map[string]string{ + StaticPodHashAnnotation: hash, + }, + }, + + Data: map[string]string{ + instance.Spec.StaticPodManifest: data, + }, + } + if err := r.Create(context.TODO(), cm, &client.CreateOptions{}); err != nil { + return err + } + return nil + } + return err + } + + // if the hash value in the annotation of the cm does not match the latest hash, then update the data in the cm + if cm.Annotations[StaticPodHashAnnotation] != hash { + cm.Annotations[StaticPodHashAnnotation] = hash + cm.Data[instance.Spec.StaticPodManifest] = data + + if err := r.Update(context.TODO(), cm, &client.UpdateOptions{}); err != nil { + return err + } + } + + return nil +} + +// autoUpgrade automatically rolling upgrade the target static pods in cluster +func (r *ReconcileStaticPod) autoUpgrade(instance *appsv1alpha1.StaticPod, infos map[string]*upgradeinfo.UpgradeInfo, hash string) error { + // readyUpgradeWaitingNodes represents nodes that need to create worker pods + readyUpgradeWaitingNodes := upgradeinfo.ReadyUpgradeWaitingNodes(infos) + + waitingNumber := len(readyUpgradeWaitingNodes) + if waitingNumber == 0 { + return nil + } + + // max is the maximum number of nodes can be upgraded in current round in auto upgrade mode + max, err := util.UnavailableCount(&instance.Spec.UpgradeStrategy, len(infos)) + if err != nil { + return err + } + + if waitingNumber < max { + max = waitingNumber + } + + readyUpgradeWaitingNodes = readyUpgradeWaitingNodes[:max] + if err := createUpgradeWorker(r.Client, instance, readyUpgradeWaitingNodes, hash, string(appsv1alpha1.AutoStaticPodUpgradeStrategyType)); err != nil { + return err + } + return nil +} + +// otaUpgrade adds condition PodNeedUpgrade to the target static pods and issue the latest manifest to corresponding nodes +func (r *ReconcileStaticPod) otaUpgrade(instance *appsv1alpha1.StaticPod, infos map[string]*upgradeinfo.UpgradeInfo, hash string) error { + upgradeNeededNodes := upgradeinfo.UpgradeNeededNodes(infos) + upgradedNodes := upgradeinfo.UpgradedNodes(infos) + + // Set condition for upgrade needed static pods + for _, n := range upgradeNeededNodes { + if err := util.SetPodUpgradeCondition(r.Client, corev1.ConditionTrue, infos[n].StaticPod); err != nil { + return err + } + } + + // Set condition for upgraded static pods + for _, n := range upgradedNodes { + if err := util.SetPodUpgradeCondition(r.Client, corev1.ConditionFalse, infos[n].StaticPod); err != nil { + return err + } + } + + // Create worker pod to issue the latest manifest to ready node + readyUpgradeWaitingNodes := upgradeinfo.OTAReadyUpgradeWaitingNodes(infos, hash) + if err := createUpgradeWorker(r.Client, instance, readyUpgradeWaitingNodes, hash, string(appsv1alpha1.OTAStaticPodUpgradeStrategyType)); err != nil { + return err + } + + return nil +} + +// removeUnusedPods delete pods, include two situations: out-of-date worker pods and succeeded worker pods +func (r *ReconcileStaticPod) removeUnusedPods(pods []*corev1.Pod) error { + for _, pod := range pods { + if err := r.Delete(context.TODO(), pod, &client.DeleteOptions{}); err != nil { + return err + } + klog.V(4).Infof(Format("Delete upgrade worker pod %v", pod.Name)) + } + return nil +} + +// createUpgradeWorker creates static pod upgrade worker to the given nodes +func createUpgradeWorker(c client.Client, instance *appsv1alpha1.StaticPod, nodes []string, hash, mode string) error { + for _, node := range nodes { + pod := upgradeWorker.DeepCopy() + pod.Name = UpgradeWorkerPodPrefix + node + "-" + hash + pod.Spec.NodeName = node + metav1.SetMetaDataAnnotation(&pod.ObjectMeta, StaticPodHashAnnotation, hash) + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: configMapVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: util.WithConfigMapPrefix(util.Hyphen(instance.Spec.StaticPodNamespace, instance.Spec.StaticPodName)), + }, + }, + }, + }) + pod.Spec.Containers[0].Args = []string{fmt.Sprintf(ArgTmpl, util.Hyphen(instance.Spec.StaticPodName, node), instance.Spec.StaticPodManifest, hash, instance.Spec.StaticPodNamespace, mode)} + + if err := controllerutil.SetControllerReference(instance, pod, c.Scheme()); err != nil { + return err + } + + if err := c.Create(context.TODO(), pod, &client.CreateOptions{}); err != nil { + return err + } + klog.Infof(Format("Create static pod upgrade worker %s of StaticPod %s", pod.Name, instance.Name)) + } + + return nil +} + +// setUpgradeNeededInfo sets `UpgradeNeeded` flag and counts the number of upgraded nodes +func setUpgradeNeededInfos(infos map[string]*upgradeinfo.UpgradeInfo, latestHash string) int32 { + var upgradedNumber int32 + + for _, info := range infos { + if info.StaticPod != nil { + if info.StaticPod.Annotations[StaticPodHashAnnotation] != latestHash { + // Indicate the static pod in this node needs to be upgraded + info.UpgradeNeeded = true + continue + } + upgradedNumber++ + } + } + + return upgradedNumber +} + +// checkReadyNodes checks and sets the ready status for every node which has the target static pod +func checkReadyNodes(client client.Client, infos map[string]*upgradeinfo.UpgradeInfo) error { + for node, info := range infos { + ready, err := util.NodeReadyByName(client, node) + if err != nil { + return err + } + info.Ready = ready + } + return nil +} + +// checkWorkerPodInfos removes worker pods which are not in use and checks whether the last round is complete +func checkWorkerPodInfos(infos map[string]*upgradeinfo.UpgradeInfo, latestHash string) (string, bool, []*corev1.Pod) { + allSucceeded := true + deletePods := make([]*corev1.Pod, 0) + + for node, info := range infos { + if info.WorkerPod != nil { + hash := info.WorkerPod.Annotations[StaticPodHashAnnotation] + // If the worker pod is not up-to-date, then it can be recreated directly + if hash != latestHash { + deletePods = append(deletePods, info.WorkerPod) + continue + } + + // If the worker pod is up-to-date, there are three possible situations + // 1. The worker pod is failed, then some irreparable failure has occurred. Just stop reconcile and update status + // 2. The worker pod is succeeded, then this node must be up-to-date. Just delete this worker pod + // 3. The worker pod is running, pending or unknown, then just wait + switch info.WorkerPod.Status.Phase { + case corev1.PodFailed: + return node, false, deletePods + + case corev1.PodSucceeded: + deletePods = append(deletePods, info.WorkerPod) + + default: + // In this node, the latest worker pod is still running, and we don't need to create new worker for it. + info.WorkerPodRunning = true + allSucceeded = false + } + } + } + + return "", allSucceeded, deletePods +} + +// updateStatus set the status of instance to the given values +func (r *ReconcileStaticPod) updateStaticPodStatus(instance *appsv1alpha1.StaticPod, totalNum, desiredNum, upgradedNum int32, cond *appsv1alpha1.StaticPodCondition) (reconcile.Result, error) { + instance.Status.Conditions = []appsv1alpha1.StaticPodCondition{*cond} + instance.Status.TotalNumber = totalNum + instance.Status.DesiredNumber = desiredNum + instance.Status.UpgradedNumber = upgradedNum + + if err := r.Client.Status().Update(context.TODO(), instance); err != nil { + return reconcile.Result{Requeue: true}, err + } + + return reconcile.Result{}, nil +} diff --git a/pkg/controller/staticpod/staticpod_controller_test.go b/pkg/controller/staticpod/staticpod_controller_test.go new file mode 100644 index 00000000000..874a7b670c7 --- /dev/null +++ b/pkg/controller/staticpod/staticpod_controller_test.go @@ -0,0 +1,160 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package staticpod + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclint "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/controller/staticpod/util" +) + +const ( + TestStaticPodName = "nginx" + TestStaticPodImage = "nginx:1.19.1" +) + +var ( + DefaultMaxUnavailable = intstr.FromString("10%") + TestNodes = []string{"node1", "node2", "node3", "node4"} +) + +func prepareStaticPods() []client.Object { + var pods []client.Object + for _, node := range TestNodes { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.Hyphen(TestStaticPodName, node), + OwnerReferences: []metav1.OwnerReference{{Kind: "Node"}}, + Namespace: metav1.NamespaceDefault, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: TestStaticPodName, + Image: TestStaticPodImage, + }, + }, + NodeName: node, + }, + } + + pods = append(pods, client.Object(pod)) + } + return pods +} + +func prepareNodes() []client.Object { + var nodes []client.Object + for _, node := range TestNodes { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: node}, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{{ + Type: corev1.NodeReady, + Status: corev1.ConditionTrue}}}, + } + nodes = append(nodes, node) + } + return nodes +} + +func TestReconcile(t *testing.T) { + var strategy = []appsv1alpha1.StaticPodUpgradeStrategy{ + {Type: appsv1alpha1.OTAStaticPodUpgradeStrategyType}, + {Type: appsv1alpha1.AutoStaticPodUpgradeStrategyType, MaxUnavailable: &DefaultMaxUnavailable}, + } + staticPods := prepareStaticPods() + nodes := prepareNodes() + instance := &appsv1alpha1.StaticPod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: appsv1alpha1.StaticPodSpec{ + StaticPodNamespace: metav1.NamespaceDefault, + StaticPodName: "nginx", + StaticPodManifest: "nginx", + Template: corev1.PodTemplateSpec{}, + }, + } + + scheme := runtime.NewScheme() + if err := appsv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal("Fail to add yurt custom resource") + } + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatal("Fail to add kubernetes clint-go custom resource") + } + + for _, s := range strategy { + instance.Spec.UpgradeStrategy = s + c := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance).WithObjects(staticPods...).WithObjects(nodes...).Build() + + var req = reconcile.Request{NamespacedName: types.NamespacedName{Name: "foo"}} + rsp := ReconcileStaticPod{ + Client: c, + scheme: scheme, + } + + _, err := rsp.Reconcile(context.TODO(), req) + if err != nil { + t.Fatalf("failed to control static-pod controller") + } + } +} + +func Test_nodeTurnReady(t *testing.T) { + evt := event.UpdateEvent{ + ObjectNew: &corev1.Node{ + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + ObjectOld: &corev1.Node{ + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionFalse, + }, + }, + }, + }, + } + t.Run("Test_nodeTurnReady", func(t *testing.T) { + if got := nodeTurnReady(evt); got != true { + t.Errorf("nodeTurnReady() = %v, want true", got) + } + }) +} diff --git a/pkg/controller/staticpod/upgradeinfo/upgrade_info.go b/pkg/controller/staticpod/upgradeinfo/upgrade_info.go new file mode 100644 index 00000000000..cab4d983030 --- /dev/null +++ b/pkg/controller/staticpod/upgradeinfo/upgrade_info.go @@ -0,0 +1,174 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgradeinfo + +import ( + "context" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/controller/staticpod/util" +) + +const ( + OTALatestManifestAnnotation = "openyurt.io/ota-latest-version" +) + +// UpgradeInfo is a structure that stores some information used by static pods to upgrade. +type UpgradeInfo struct { + // Static pod running on the node + StaticPod *corev1.Pod + + // Upgrade worker pod running on the node + WorkerPod *corev1.Pod + + // Indicate whether the static pod on the node needs to be upgraded. + // If true, the static pod is not up-to-date and needs to be upgraded. + UpgradeNeeded bool + + // Indicate whether the worker pod on the node is running. + // If true, then the upgrade operation is in progress and does not + // need to create a new worker pod. + WorkerPodRunning bool + + // Indicate whether the node is ready. It's used in Auto mode. + Ready bool +} + +// New constructs the upgrade information for nodes which have the target static pod +func New(c client.Client, instance *appsv1alpha1.StaticPod, workerPodName string) (map[string]*UpgradeInfo, error) { + infos := make(map[string]*UpgradeInfo) + + // upgrade worker pod is default in "kube-system" namespace which may be different with target static pod's namespace + var podList, systemPodList corev1.PodList + if err := c.List(context.TODO(), &podList, &client.ListOptions{Namespace: instance.Spec.StaticPodNamespace}); err != nil { + return nil, err + } + + if err := c.List(context.TODO(), &systemPodList, &client.ListOptions{Namespace: metav1.NamespaceSystem}); err != nil { + return nil, err + } + + for i, pod := range podList.Items { + nodeName := pod.Spec.NodeName + if nodeName == "" || pod.DeletionTimestamp != nil { + continue + } + + // The name format of mirror static pod is `StaticPodName-NodeName` + if util.Hyphen(instance.Spec.StaticPodName, nodeName) == pod.Name && isStaticPod(&pod) { + if info := infos[nodeName]; info == nil { + infos[nodeName] = &UpgradeInfo{} + } + infos[nodeName].StaticPod = &podList.Items[i] + } + } + + for i, pod := range systemPodList.Items { + nodeName := pod.Spec.NodeName + if nodeName == "" || pod.DeletionTimestamp != nil { + continue + } + // The name format of worker pods are `WorkerPodName-NodeName-Hash` Todo: may lead to mismatch + if strings.Contains(pod.Name, workerPodName) { + if info := infos[nodeName]; info == nil { + infos[nodeName] = &UpgradeInfo{} + } + infos[nodeName].WorkerPod = &systemPodList.Items[i] + } + } + + return infos, nil +} + +// isStaticPod judges whether a pod is static by its OwnerReference +func isStaticPod(pod *corev1.Pod) bool { + for _, ownerRef := range pod.GetOwnerReferences() { + if ownerRef.Kind == "Node" { + return true + } + } + return false +} + +// ReadyUpgradeWaitingNodes gets those nodes that satisfied +// 1. node is ready +// 2. node needs to be upgraded +// 3. no latest worker pod running on the node +// On these nodes, new worker pods need to be created for auto mode +func ReadyUpgradeWaitingNodes(infos map[string]*UpgradeInfo) []string { + var nodes []string + for node, info := range infos { + if info.UpgradeNeeded && !info.WorkerPodRunning && info.Ready { + nodes = append(nodes, node) + } + } + return nodes +} + +// OTAReadyUpgradeWaitingNodes bases on ReadyUpgradeWaitingNodes() and checks whether annotation +// `openyurt.io/ota-latest-version` is up-to-date +func OTAReadyUpgradeWaitingNodes(infos map[string]*UpgradeInfo, hash string) []string { + var nodes []string + for node, info := range infos { + if info.StaticPod != nil && info.StaticPod.Annotations[OTALatestManifestAnnotation] == hash { + continue + } + + if info.UpgradeNeeded && !info.WorkerPodRunning && info.Ready { + nodes = append(nodes, node) + } + } + return nodes +} + +// ReadyNodes gets nodes that are ready +func ReadyNodes(infos map[string]*UpgradeInfo) []string { + var nodes []string + for node, info := range infos { + if info.Ready { + nodes = append(nodes, node) + } + } + return nodes +} + +// UpgradeNeededNodes gets nodes that are not running the latest static pods +func UpgradeNeededNodes(infos map[string]*UpgradeInfo) []string { + var nodes []string + for node, info := range infos { + if info.UpgradeNeeded { + nodes = append(nodes, node) + } + } + return nodes +} + +// UpgradedNodes gets nodes that are running the latest static pods +func UpgradedNodes(infos map[string]*UpgradeInfo) []string { + var nodes []string + for node, info := range infos { + if !info.UpgradeNeeded { + nodes = append(nodes, node) + } + } + return nodes +} diff --git a/pkg/controller/staticpod/upgradeinfo/upgrade_info_test.go b/pkg/controller/staticpod/upgradeinfo/upgrade_info_test.go new file mode 100644 index 00000000000..6aadf201e70 --- /dev/null +++ b/pkg/controller/staticpod/upgradeinfo/upgrade_info_test.go @@ -0,0 +1,196 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgradeinfo + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +const ( + UpgradeWorkerPodPrefix = "static-pod-upgrade-worker-" +) + +var ( + fakeStaticPodNodes = []string{"node1", "node2", "node3", "node4"} + fakeWorkerPodNodes = []string{"node1", "node2"} + fakeStaticPodName = "nginx" +) + +func newPod(podName string, nodeName string, namespace string, isStaticPod bool) *corev1.Pod { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: UpgradeWorkerPodPrefix + rand.String(10), + Namespace: namespace, + }, + Spec: corev1.PodSpec{NodeName: nodeName}, + } + + if isStaticPod { + pod.Name = podName + "-" + nodeName + pod.ObjectMeta.OwnerReferences = []metav1.OwnerReference{{Kind: "Node"}} + } + + return pod +} + +func newPods(nodes []string, namespace string, isStaticPod bool) []client.Object { + var pods []client.Object + for _, n := range nodes { + pods = append(pods, client.Object(newPod(fakeStaticPodName, n, namespace, isStaticPod))) + } + return pods +} + +func newStaticPod() *appsv1alpha1.StaticPod { + return &appsv1alpha1.StaticPod{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: appsv1alpha1.StaticPodSpec{ + StaticPodName: fakeStaticPodName, + }, + Status: appsv1alpha1.StaticPodStatus{}, + } +} + +func Test_ConstructStaticPodsUpgradeInfoList(t *testing.T) { + staticPods := newPods(fakeStaticPodNodes, metav1.NamespaceDefault, true) + workerPods := newPods(fakeWorkerPodNodes, metav1.NamespaceSystem, false) + expect := map[string]*UpgradeInfo{ + "node1": { + StaticPod: staticPods[0].(*corev1.Pod), + WorkerPod: workerPods[0].(*corev1.Pod), + }, + "node2": { + StaticPod: staticPods[1].(*corev1.Pod), + + WorkerPod: workerPods[1].(*corev1.Pod), + }, + "node3": { + StaticPod: staticPods[2].(*corev1.Pod), + }, + "node4": { + StaticPod: staticPods[3].(*corev1.Pod), + }, + } + + pods := append(staticPods, workerPods...) + c := fake.NewClientBuilder().WithObjects(pods...).Build() + + t.Run("test", func(t *testing.T) { + spi, _ := New(c, newStaticPod(), UpgradeWorkerPodPrefix) + + if !reflect.DeepEqual(spi, expect) { + t.Fatalf("Fail to test ConstructStaticPodsUpgradeInfoList, got %v, want %v", spi, expect) + } + }) +} + +func TestNodes(t *testing.T) { + tHash := "tHash" + fHash := "fHash" + spi := map[string]*UpgradeInfo{ + "node1": { + WorkerPodRunning: true, + UpgradeNeeded: true, + Ready: true, + }, + "node2": { + StaticPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + OTALatestManifestAnnotation: tHash, + }, + }, + }, + UpgradeNeeded: true, + Ready: true, + }, + "node3": { + StaticPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + OTALatestManifestAnnotation: fHash, + }, + }, + }, + UpgradeNeeded: true, + Ready: true, + }, + "node4": { + Ready: true, + }, + "node5": {}, + } + + expectReadyUpgradeWaitingNodes := map[string]struct{}{"node2": {}, "node3": {}} + expectOTAReadyUpgradeWaitingNodes := map[string]struct{}{"node3": {}} + expectReadyNodes := map[string]struct{}{"node1": {}, "node2": {}, "node3": {}, "node4": {}} + expectUpgradeNeededNodes := map[string]struct{}{"node1": {}, "node2": {}, "node3": {}} + expectUpgradedNodes := map[string]struct{}{"node4": {}, "node5": {}} + + t.Run("TestReadyUpgradeWaitingNodes", func(t *testing.T) { + if got := ReadyUpgradeWaitingNodes(spi); !hasCommonElement(got, expectReadyUpgradeWaitingNodes) { + t.Fatalf("ReadyUpgradeWaitingNodes = %v, want %v", got, expectReadyUpgradeWaitingNodes) + } + }) + + t.Run("OTAReadyUpgradeWaitingNodes", func(t *testing.T) { + if got := OTAReadyUpgradeWaitingNodes(spi, tHash); !hasCommonElement(got, expectOTAReadyUpgradeWaitingNodes) { + t.Fatalf("OTAReadyUpgradeWaitingNodes got %v, want %v", got, expectOTAReadyUpgradeWaitingNodes) + } + }) + + t.Run("ReadyNodes", func(t *testing.T) { + if got := ReadyNodes(spi); !hasCommonElement(got, expectReadyNodes) { + t.Fatalf("ReadyNodes got %v, want %v", got, expectReadyNodes) + } + }) + + t.Run("UpgradeNeededNodes", func(t *testing.T) { + if got := UpgradeNeededNodes(spi); !hasCommonElement(got, expectUpgradeNeededNodes) { + t.Fatalf("UpgradeNeededNodes got %v, want %v", got, expectUpgradeNeededNodes) + } + }) + + t.Run("UpgradedNodes", func(t *testing.T) { + if got := UpgradedNodes(spi); !hasCommonElement(got, expectUpgradedNodes) { + t.Fatalf("UpgradedNodes got %v, want %v", got, expectUpgradedNodes) + } + }) +} + +func hasCommonElement(a []string, b map[string]struct{}) bool { + if len(a) != len(b) { + return false + } + + for _, i := range a { + if b[i] != struct{}{} { + return false + } + } + return true +} diff --git a/pkg/controller/staticpod/util/util.go b/pkg/controller/staticpod/util/util.go new file mode 100644 index 00000000000..35b62aef099 --- /dev/null +++ b/pkg/controller/staticpod/util/util.go @@ -0,0 +1,216 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "bytes" + "context" + "fmt" + "hash" + "hash/fnv" + + "github.com/davecgh/go-spew/spew" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/cli-runtime/pkg/printers" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +const ( + ConfigMapPrefix = "yurt-cm-" + + // PodNeedUpgrade indicates whether the pod is able to upgrade. + PodNeedUpgrade corev1.PodConditionType = "PodNeedUpgrade" + + StaticPodHashAnnotation = "openyurt.io/static-pod-hash" +) + +var ( + PodGVK = corev1.SchemeGroupVersion.WithKind("Pod") +) + +func Hyphen(str1, str2 string) string { + return str1 + "-" + str2 +} + +// WithConfigMapPrefix add prefix `yurt-cm-` to the given string +func WithConfigMapPrefix(str string) string { + return ConfigMapPrefix + str +} + +// UnavailableCount returns 0 if unavailability is not requested, the expected +// unavailability number to allow out of numberToSchedule if requested, or an error if +// the unavailability percentage requested is invalid. +func UnavailableCount(us *appsv1alpha1.StaticPodUpgradeStrategy, numberToUpgrade int) (int, error) { + if us == nil || us.Type != appsv1alpha1.AutoStaticPodUpgradeStrategyType { + return 0, nil + } + return intstr.GetScaledValueFromIntOrPercent(us.MaxUnavailable, numberToUpgrade, true) +} + +// ComputeHash returns a hash value calculated from pod template +func ComputeHash(template *corev1.PodTemplateSpec) string { + podSpecHasher := fnv.New32a() + DeepHashObject(podSpecHasher, *template) + + return rand.SafeEncodeString(fmt.Sprint(podSpecHasher.Sum32())) +} + +// DeepHashObject writes specified object to hash using the spew library +// which follows pointers and prints actual values of the nested objects +// ensuring the hash does not change when a pointer changes. +func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) { + hasher.Reset() + printer := spew.ConfigState{ + Indent: " ", + SortKeys: true, + DisableMethods: true, + SpewKeys: true, + } + printer.Fprintf(hasher, "%#v", objectToWrite) +} + +// GenStaticPodManifest generates manifest from use-specified template +func GenStaticPodManifest(tmplSpec *corev1.PodTemplateSpec, hash string) (string, error) { + pod := &corev1.Pod{ObjectMeta: *tmplSpec.ObjectMeta.DeepCopy(), Spec: *tmplSpec.Spec.DeepCopy()} + // latest hash value will be added to the annotation to facilitate checking if the running static pods are latest + metav1.SetMetaDataAnnotation(&pod.ObjectMeta, StaticPodHashAnnotation, hash) + + pod.GetObjectKind().SetGroupVersionKind(PodGVK) + + var buf bytes.Buffer + y := printers.YAMLPrinter{} + if err := y.PrintObj(pod, &buf); err != nil { + return "", err + } + + return buf.String(), nil +} + +// NodeReadyByName check if the given node is ready +func NodeReadyByName(c client.Client, nodeName string) (bool, error) { + node := &corev1.Node{} + if err := c.Get(context.TODO(), types.NamespacedName{Name: nodeName}, node);err != nil { + return false, err + } + + _, nc := GetNodeCondition(&node.Status, corev1.NodeReady) + + return nc != nil && nc.Status == corev1.ConditionTrue, nil +} + +// GetNodeCondition extracts the provided condition from the given status and returns that. +// Returns nil and -1 if the condition is not present, and the index of the located condition. +func GetNodeCondition(status *corev1.NodeStatus, conditionType corev1.NodeConditionType) (int, *corev1.NodeCondition) { + if status == nil { + return -1, nil + } + for i := range status.Conditions { + if status.Conditions[i].Type == conditionType { + return i, &status.Conditions[i] + } + } + return -1, nil +} + +// NewStaticPodCondition creates a new StaticPod condition. +func NewStaticPodCondition(condType appsv1alpha1.StaticPodConditionType, status corev1.ConditionStatus, reason, message string) *appsv1alpha1.StaticPodCondition { + return &appsv1alpha1.StaticPodCondition{ + Type: condType, + Status: status, + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: message, + } +} + +// UpgradeFailedConditionWithNode return StaticPodUpgradeFailed Condition with reason specifies the upgraded error node +func UpgradeFailedConditionWithNode(node string) *appsv1alpha1.StaticPodCondition { + return NewStaticPodCondition(appsv1alpha1.StaticPodUpgradeFailed, corev1.ConditionTrue, fmt.Sprintf("StaticPod upgrade node %s failed", node), "") +} + +// SetPodUpgradeCondition set pod condition `PodNeedUpgrade` to the specified value +func SetPodUpgradeCondition(c client.Client, status corev1.ConditionStatus, pod *corev1.Pod) error { + cond := &corev1.PodCondition{ + Type: PodNeedUpgrade, + Status: status, + } + if change := UpdatePodCondition(&pod.Status, cond); change { + if err := c.Status().Update(context.TODO(), pod, &client.UpdateOptions{}); err != nil { + return err + } + } + + return nil +} + +// UpdatePodCondition updates existing pod condition or creates a new one. Sets LastTransitionTime to now if the +// status has changed. +// Returns true if pod condition has changed or has been added. +func UpdatePodCondition(status *corev1.PodStatus, condition *corev1.PodCondition) bool { + condition.LastTransitionTime = metav1.Now() + // Try to find this pod condition. + conditionIndex, oldCondition := GetPodCondition(status, condition.Type) + + if oldCondition == nil { + // We are adding new pod condition. + status.Conditions = append(status.Conditions, *condition) + return true + } + // We are updating an existing condition, so we need to check if it has changed. + if condition.Status == oldCondition.Status { + condition.LastTransitionTime = oldCondition.LastTransitionTime + } + + isEqual := condition.Status == oldCondition.Status && + condition.Reason == oldCondition.Reason && + condition.Message == oldCondition.Message && + condition.LastProbeTime.Equal(&oldCondition.LastProbeTime) && + condition.LastTransitionTime.Equal(&oldCondition.LastTransitionTime) + + status.Conditions[conditionIndex] = *condition + // Return true if one of the fields have changed. + return !isEqual +} + +// GetPodCondition extracts the provided condition from the given status and returns that. +// Returns nil and -1 if the condition is not present, and the index of the located condition. +func GetPodCondition(status *corev1.PodStatus, conditionType corev1.PodConditionType) (int, *corev1.PodCondition) { + if status == nil { + return -1, nil + } + return GetPodConditionFromList(status.Conditions, conditionType) +} + +// GetPodConditionFromList extracts the provided condition from the given list of condition and +// returns the index of the condition and the condition. Returns -1 and nil if the condition is not present. +func GetPodConditionFromList(conditions []corev1.PodCondition, conditionType corev1.PodConditionType) (int, *corev1.PodCondition) { + if conditions == nil { + return -1, nil + } + for i := range conditions { + if conditions[i].Type == conditionType { + return i, &conditions[i] + } + } + return -1, nil +} diff --git a/pkg/webhook/add_staticpod.go b/pkg/webhook/add_staticpod.go new file mode 100644 index 00000000000..4b8ceb4be83 --- /dev/null +++ b/pkg/webhook/add_staticpod.go @@ -0,0 +1,27 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "github.com/openyurtio/openyurt/pkg/webhook/staticpod/mutating" + "github.com/openyurtio/openyurt/pkg/webhook/staticpod/validating" +) + +func init() { + addHandlers(mutating.HandlerMap) + addHandlers(validating.HandlerMap) +} diff --git a/pkg/webhook/staticpod/mutating/staticpod_handler.go b/pkg/webhook/staticpod/mutating/staticpod_handler.go new file mode 100644 index 00000000000..1612d680c52 --- /dev/null +++ b/pkg/webhook/staticpod/mutating/staticpod_handler.go @@ -0,0 +1,89 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mutating + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "reflect" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/util" +) + +const ( + webhookName = "StaticPod-mutate-webhook" +) + +func Format(format string, args ...interface{}) string { + s := fmt.Sprintf(format, args...) + return fmt.Sprintf("%s: %s", webhookName, s) +} + +// StaticPodCreateUpdateHandler handles StaticPod +type StaticPodCreateUpdateHandler struct { + // Decoder decodes objects + Decoder *admission.Decoder +} + +var _ admission.Handler = &StaticPodCreateUpdateHandler{} + +// Handle handles admission requests. +func (h *StaticPodCreateUpdateHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + + // Note !!!!!!!!!! + // We strongly recommend use Format() to encapsulation because Format() can print logs by module + // @kadisi + klog.Infof(Format("Handle StaticPod %s/%s", req.Namespace, req.Name)) + + obj := &appsv1alpha1.StaticPod{} + err := h.Decoder.Decode(req, obj) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + var copy runtime.Object = obj.DeepCopy() + // Set defaults + appsv1alpha1.SetDefaultsStaticPod(obj) + + if reflect.DeepEqual(obj, copy) { + return admission.Allowed("") + } + marshalled, err := json.Marshal(obj) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + resp := admission.PatchResponseFromRaw(req.AdmissionRequest.Object.Raw, marshalled) + if len(resp.Patches) > 0 { + klog.Infof(Format("Admit StaticPod %s patches: %v", obj.Name, util.DumpJSON(resp.Patches))) + } + + return resp +} + +var _ admission.DecoderInjector = &StaticPodCreateUpdateHandler{} + +// InjectDecoder injects the decoder into the StaticPodCreateUpdateHandler +func (h *StaticPodCreateUpdateHandler) InjectDecoder(d *admission.Decoder) error { + h.Decoder = d + return nil +} diff --git a/pkg/webhook/staticpod/mutating/webhooks.go b/pkg/webhook/staticpod/mutating/webhooks.go new file mode 100644 index 00000000000..5cf3f2a8cb7 --- /dev/null +++ b/pkg/webhook/staticpod/mutating/webhooks.go @@ -0,0 +1,30 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mutating + +import ( + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// +kubebuilder:webhook:path=/mutate-apps-openyurt-io-v1alpha1-staticpod,mutating=true,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps.openyurt.io,resources=staticpods,verbs=create;update,versions=v1alpha1,name=mutate.apps.v1alpha1.staticpod.openyurt.io + +var ( + // HandlerMap contains admission webhook handlers + HandlerMap = map[string]admission.Handler{ + "mutate-apps-openyurt-io-v1alpha1-staticpod": &StaticPodCreateUpdateHandler{}, + } +) diff --git a/pkg/webhook/staticpod/validating/staticpod_handler.go b/pkg/webhook/staticpod/validating/staticpod_handler.go new file mode 100644 index 00000000000..1167b381a09 --- /dev/null +++ b/pkg/webhook/staticpod/validating/staticpod_handler.go @@ -0,0 +1,128 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import ( + "context" + "fmt" + "net/http" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +// StaticPodCreateUpdateHandler handles StaticPod +type StaticPodCreateUpdateHandler struct { + // Decoder decodes objects + Decoder *admission.Decoder +} + +const ( + webhookName = "StaticPod-validate-webhook" +) + +func Format(format string, args ...interface{}) string { + s := fmt.Sprintf(format, args...) + return fmt.Sprintf("%s: %s", webhookName, s) +} + +var _ admission.Handler = &StaticPodCreateUpdateHandler{} + +// Handle handles admission requests. +func (h *StaticPodCreateUpdateHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + // Note !!!!!!!!!! + // We strongly recommend use Format() to encapsulation because Format() can print logs by module + // @kadisi + klog.Infof(Format("Handle StaticPod %s/%s", req.Namespace, req.Name)) + + obj := &appsv1alpha1.StaticPod{} + + if err := h.Decoder.Decode(req, obj); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + if err := validate(obj); err != nil { + klog.Warningf("Error validate StaticPod %s: %v", obj.Name, err) + return admission.Errored(http.StatusBadRequest, err) + } + + return admission.ValidationResponse(true, "allowed") +} + +func validate(obj *appsv1alpha1.StaticPod) error { + if allErrs := validateStaticPodSpec(&obj.Spec); len(allErrs) > 0 { + return apierrors.NewInvalid(appsv1alpha1.GroupVersion.WithKind("StaticPod").GroupKind(), obj.Name, allErrs) + } + + klog.Infof(Format("Validate StaticPod %s successfully ...", klog.KObj(obj))) + + return nil +} + +var _ admission.DecoderInjector = &StaticPodCreateUpdateHandler{} + +// InjectDecoder injects the decoder into the StaticPodCreateUpdateHandler +func (h *StaticPodCreateUpdateHandler) InjectDecoder(d *admission.Decoder) error { + h.Decoder = d + return nil +} + +// validateStaticPodSpec validates the staticpod spec. +func validateStaticPodSpec(spec *appsv1alpha1.StaticPodSpec) field.ErrorList { + var allErrs field.ErrorList + + if spec.StaticPodName == "" { + allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("StaticPodName"), + "StaticPodName is required")) + } + + if spec.StaticPodManifest == "" { + allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("StaticPodManifest"), + "StaticPodManifest is required")) + } + + if spec.StaticPodNamespace == "" { + allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("Namespace"), + "Namespace is required")) + } + + strategy := &spec.UpgradeStrategy + if strategy == nil { + allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("upgradeStrategy"), + "upgrade strategy is required")) + } + + if strategy.Type != appsv1alpha1.AutoStaticPodUpgradeStrategyType && strategy.Type != appsv1alpha1.OTAStaticPodUpgradeStrategyType { + allErrs = append(allErrs, field.NotSupported(field.NewPath("spec").Child("upgradeStrategy"), + strategy, []string{"auto", "ota"})) + } + + if strategy.Type == appsv1alpha1.AutoStaticPodUpgradeStrategyType && strategy.MaxUnavailable == nil { + allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("upgradeStrategy"), + "max-unavailable is required in auto mode")) + } + + if allErrs != nil { + return allErrs + } + + return nil +} diff --git a/pkg/webhook/staticpod/validating/staticpod_handler_test.go b/pkg/webhook/staticpod/validating/staticpod_handler_test.go new file mode 100644 index 00000000000..88aa7e97231 --- /dev/null +++ b/pkg/webhook/staticpod/validating/staticpod_handler_test.go @@ -0,0 +1,103 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import ( + "reflect" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation/field" + + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +func Test_validateStaticPodSpec(t *testing.T) { + + tests := []struct { + name string + spec *appsv1alpha1.StaticPodSpec + want field.ErrorList + }{ + { + "validate success", + &appsv1alpha1.StaticPodSpec{ + StaticPodNamespace: metav1.NamespaceDefault, + StaticPodName: "nginx", + StaticPodManifest: "nginx", + UpgradeStrategy: appsv1alpha1.StaticPodUpgradeStrategy{ + Type: appsv1alpha1.OTAStaticPodUpgradeStrategyType, + }, + }, + nil, + }, + { + "miss namespace", + &appsv1alpha1.StaticPodSpec{ + StaticPodName: "nginx", + StaticPodManifest: "nginx", + UpgradeStrategy: appsv1alpha1.StaticPodUpgradeStrategy{ + Type: appsv1alpha1.OTAStaticPodUpgradeStrategyType, + }, + }, + field.ErrorList{field.Required(field.NewPath("spec").Child("Namespace"), + "Namespace is required")}, + }, + { + "miss name", + &appsv1alpha1.StaticPodSpec{ + StaticPodNamespace: metav1.NamespaceDefault, + StaticPodManifest: "nginx", + UpgradeStrategy: appsv1alpha1.StaticPodUpgradeStrategy{ + Type: appsv1alpha1.OTAStaticPodUpgradeStrategyType, + }, + }, + field.ErrorList{field.Required(field.NewPath("spec").Child("StaticPodName"), + "StaticPodName is required")}, + }, + { + "miss manifest", + &appsv1alpha1.StaticPodSpec{ + StaticPodName: "nginx", + StaticPodNamespace: metav1.NamespaceDefault, + UpgradeStrategy: appsv1alpha1.StaticPodUpgradeStrategy{ + Type: appsv1alpha1.OTAStaticPodUpgradeStrategyType, + }, + }, + field.ErrorList{field.Required(field.NewPath("spec").Child("StaticPodManifest"), + "StaticPodManifest is required")}, + }, + { + "miss upgrade strategy unavailable", + &appsv1alpha1.StaticPodSpec{ + StaticPodName: "nginx", + StaticPodNamespace: metav1.NamespaceDefault, + StaticPodManifest: "nginx", + UpgradeStrategy: appsv1alpha1.StaticPodUpgradeStrategy{Type: appsv1alpha1.AutoStaticPodUpgradeStrategyType}, + }, + field.ErrorList{field.Required(field.NewPath("spec").Child("upgradeStrategy"), + "max-unavailable is required in auto mode")}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := validateStaticPodSpec(tt.spec); !reflect.DeepEqual(got, tt.want) { + t.Errorf("validateStaticPodSpec() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/webhook/staticpod/validating/webhooks.go b/pkg/webhook/staticpod/validating/webhooks.go new file mode 100644 index 00000000000..e42f717893e --- /dev/null +++ b/pkg/webhook/staticpod/validating/webhooks.go @@ -0,0 +1,30 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import ( + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// +kubebuilder:webhook:path=/validate-apps-openyurt-io-v1alpha1-staticpod,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps.openyurt.io,resources=staticpods,verbs=create;update,versions=v1alpha1,name=validate.apps.v1alpha1.staticpod.openyurt.io + +var ( + // HandlerMap contains admission webhook handlers + HandlerMap = map[string]admission.Handler{ + "validate-apps-openyurt-io-v1alpha1-staticpod": &StaticPodCreateUpdateHandler{}, + } +)