feat: add PersistentPodState custom workload support

Signed-off-by: peng.xin <[email protected]>
openkruise · Nov 21, 2022 · 0f47468 · 0f47468
1 parent bdd3efb
commit 0f47468
Show file tree

Hide file tree

Showing 162 changed files with 11,823 additions and 710 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -48,6 +48,7 @@ jobs:
           version: ${{ env.GOLANGCI_VERSION }}
           args: --verbose
           skip-pkg-cache: true
+          mod: readonly
 
   markdownlint-misspell-shellcheck:
     runs-on: ubuntu-18.04

diff --git a/.github/workflows/e2e-1.16.yaml b/.github/workflows/e2e-1.16.yaml
@@ -10,7 +10,7 @@ on:
 
 env:
   # Common versions
-  GO_VERSION: '1.17'
+  GO_VERSION: '1.18'
   KIND_VERSION: 'v0.14.0'
   KIND_IMAGE: 'kindest/node:v1.16.15'
   KIND_CLUSTER_NAME: 'ci-testing'
@@ -85,6 +85,18 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
 
   pullimages-containerrecreate:
@@ -154,6 +166,18 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
 
   advanced-daemonset:
@@ -223,6 +247,18 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
 
   sidecarset:
@@ -292,6 +328,18 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
 
   podUnavailableBudget:
@@ -419,4 +467,16 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
diff --git a/.github/workflows/e2e-1.24.yaml b/.github/workflows/e2e-1.24.yaml
@@ -154,6 +154,18 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
 
   advanced-daemonset:
@@ -223,6 +235,18 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
 
   sidecarset:
@@ -292,6 +316,18 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
 
   ephemeraljob:
@@ -477,4 +513,16 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,61 @@
 # Change Log
 
+## v1.3.0
+
+> Change log since v1.2.0
+
+### New CRD and Controller: PodProbeMarker
+
+Kubernetes provides three Pod lifecycle management:
+- **Readiness Probe** Used to determine whether the business container is ready to respond to user requests. If the probe fails, the Pod will be removed from Service Endpoints.
+- **Liveness Probe** Used to determine the health status of the container. If the probe fails, the kubelet will restart the container.
+- **Startup Probe** Used to know when a container application has started. If such a probe is configured, it disables liveness and readiness checks until it succeeds.
+
+So the Probe capabilities provided in Kubernetes have defined specific semantics and related behaviors.
+**In addition, there is actually a need to customize Probe semantics and related behaviors**, such as:
+- **GameServer defines Idle Probe to determine whether the Pod currently has a game match**, if not, from the perspective of cost optimization, the Pod can be scaled down.
+- **K8S Operator defines the main-secondary probe to determine the role of the current Pod (main or secondary)**. When upgrading, the secondary can be upgraded first,
+so as to achieve the behavior of selecting the main only once during the upgrade process, reducing the service interruption time during the upgrade process.
+
+So we provides the ability to customize the Probe and return the result to the Pod yaml.
+
+For more detail, please refer to its [documentation](https://openkruise.io/docs/user-manuals/podprobemarker) and [proposal](https://github.com/openkruise/kruise/blob/master/docs/proposals/20220728-pod-probe-marker.md).
+
+### SidecarSet
+- SidecarSet support to inject pods under kube-system,kube-public namespace. ([#1084](https://github.com/openkruise/kruise/pull/1084), [@zmberg](https://github.com/zmberg))
+- SidecarSet support to inject specific history sidecar container to Pods. ([#1021](https://github.com/openkruise/kruise/pull/1021), [@veophi](https://github.com/veophi))
+- SidecarSet support to inject pod annotations.([#992](https://github.com/openkruise/kruise/pull/992), [@zmberg](https://github.com/zmberg))
+
+### AdvancedCronJob
+- TimeZone support for AdvancedCronJob from [upstream](https://github.com/kubernetes/kubernetes/pull/108032). ([#1070](https://github.com/openkruise/kruise/pull/1070), [@FillZpp](https://github.com/FillZpp))
+
+### WorkloadSpread
+- WorkloadSpread support Native StatefulSet and Kruise Advanced StatefulSet. ([#1056](https://github.com/openkruise/kruise/pull/1056), [@veophi](https://github.com/veophi))
+
+### CloneSet
+- CloneSet supports to calculate scale number excluding Pods in PreparingDelete. ([#1024](https://github.com/openkruise/kruise/pull/1024), [@FillZpp](https://github.com/FillZpp))
+- Optimize CloneSet queuing when cache has just synced. ([#1026](https://github.com/openkruise/kruise/pull/1026), [@FillZpp](https://github.com/FillZpp))
+
+### PodUnavailableBudget
+- Optimize event handler performance for PodUnavailableBudget. ([#1027](https://github.com/openkruise/kruise/pull/1027), [@FillZpp](https://github.com/FillZpp))
+
+### Advanced DaemonSet
+- Allow optional filed max unavilable in ads, and set default value 1. ([#1007](https://github.com/openkruise/kruise/pull/1007), [@ABNER-1](https://github.com/ABNER-1))
+- Fix DaemonSet surging with minReadySeconds. ([#1014](https://github.com/openkruise/kruise/pull/1014), [@FillZpp](https://github.com/FillZpp))
+- Optimize Advanced DaemonSet internal new pod for imitating scheduling. ([#1011](https://github.com/openkruise/kruise/pull/1011), [@FillZpp](https://github.com/FillZpp))
+- Advanced DaemonSet support pre-download image. ([#1057](https://github.com/openkruise/kruise/pull/1057), [@ABNER-1](https://github.com/ABNER-1))
+
+### Advanced StatefulSet
+- Fix panic cased by statefulset pvc auto deletion. ([#999](https://github.com/openkruise/kruise/pull/999), [@veophi](https://github.com/veophi))
+
+### Others
+- Optimize performance of LabelSelector conversion. ([#1068](https://github.com/openkruise/kruise/pull/1068), [@FillZpp](https://github.com/FillZpp))
+- Reduce kruise-manager memory allocation. ([#1015](https://github.com/openkruise/kruise/pull/1015), [@FillZpp](https://github.com/FillZpp))
+- Pod state from updating to Normal should all hooked. ([#1022](https://github.com/openkruise/kruise/pull/1022), [@shiyan2016](https://github.com/shiyan2016))
+- Fix go get in Makefile with go 1.18. ([#1036](https://github.com/openkruise/kruise/pull/1036), [@astraw99](https://github.com/astraw99))
+- Fix EphemeralJob spec.replicas nil panic bug. ([#1016](https://github.com/openkruise/kruise/pull/1016), [@hellolijj](https://github.com/openkruise/kruise/pull/1016))
+- Fix UnitedDeployment reconcile don't return err bug. ([#991](https://github.com/openkruise/kruise/pull/991), [@huiwq1990](https://github.com/huiwq1990))
+
 ## v1.2.0
 
 > Change log since v1.1.0

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -88,7 +88,7 @@ We encourage contributors to follow the [PR template](./.github/PULL_REQUEST_TEM
 As a contributor, if you want to make any contribution to Kruise project, we should reach an agreement on the version of tools used in the development environment.
 Here are some dependents with specific version:
 
-- Golang : v1.17+
+- Golang : v1.18+
 - Kubernetes: v1.16+
 
 ### Developing guide

diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # Build the manager and daemon binaries
-FROM golang:1.17 as builder
+FROM golang:1.18 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests

diff --git a/Dockerfile_multiarch b/Dockerfile_multiarch
@@ -1,5 +1,5 @@
 # Build the manager and daemon binaries
-FROM --platform=$BUILDPLATFORM golang:1.17 as builder
+FROM --platform=$BUILDPLATFORM golang:1.18 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests

diff --git a/Makefile b/Makefile
@@ -22,7 +22,7 @@ all: build
 ##@ Development
 
 go_check:
-	@scripts/check_go_version "1.17.0"
+	@scripts/check_go_version "1.18.0"
 
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	@scripts/generate_client.sh

diff --git a/apis/apps/pub/lifecycle.go b/apis/apps/pub/lifecycle.go
@@ -20,10 +20,22 @@ const (
 	LifecycleStateKey     = "lifecycle.apps.kruise.io/state"
 	LifecycleTimestampKey = "lifecycle.apps.kruise.io/timestamp"
 
-	LifecycleStateNormal          LifecycleStateType = "Normal"
+	// LifecycleStatePreparingNormal means the Pod is created but unavailable.
+	// It will translate to Normal state if Lifecycle.PreNormal is hooked.
+	LifecycleStatePreparingNormal LifecycleStateType = "PreparingNormal"
+	// LifecycleStateNormal is a necessary condition for Pod to be available.
+	LifecycleStateNormal LifecycleStateType = "Normal"
+	// LifecycleStatePreparingUpdate means pod is being prepared to update.
+	// It will translate to Updating state if Lifecycle.InPlaceUpdate is Not hooked.
 	LifecycleStatePreparingUpdate LifecycleStateType = "PreparingUpdate"
-	LifecycleStateUpdating        LifecycleStateType = "Updating"
-	LifecycleStateUpdated         LifecycleStateType = "Updated"
+	// LifecycleStateUpdating means the Pod is being updated.
+	// It will translate to Updated state if the in-place update of the Pod is done.
+	LifecycleStateUpdating LifecycleStateType = "Updating"
+	// LifecycleStateUpdated means the Pod is updated, but unavailable.
+	// It will translate to Normal state if Lifecycle.InPlaceUpdate is hooked.
+	LifecycleStateUpdated LifecycleStateType = "Updated"
+	// LifecycleStatePreparingDelete means the Pod is prepared to delete.
+	// The Pod will be deleted by workload if Lifecycle.PreDelete is Not hooked.
 	LifecycleStatePreparingDelete LifecycleStateType = "PreparingDelete"
 )
 
@@ -35,6 +47,8 @@ type Lifecycle struct {
 	PreDelete *LifecycleHook `json:"preDelete,omitempty"`
 	// InPlaceUpdate is the hook before Pod to update and after Pod has been updated.
 	InPlaceUpdate *LifecycleHook `json:"inPlaceUpdate,omitempty"`
+	// PreNormal is the hook after Pod to be created and ready to be Normal.
+	PreNormal *LifecycleHook `json:"preNormal,omitempty"`
 }
 
 type LifecycleHook struct {
@@ -43,6 +57,7 @@ type LifecycleHook struct {
 	// MarkPodNotReady = true means:
 	// - Pod will be set to 'NotReady' at preparingDelete/preparingUpdate state.
 	// - Pod will be restored to 'Ready' at Updated state if it was set to 'NotReady' at preparingUpdate state.
+	// Currently, MarkPodNotReady only takes effect on InPlaceUpdate & PreDelete hook.
 	// Default to false.
 	MarkPodNotReady bool `json:"markPodNotReady,omitempty"`
 }
diff --git a/apis/apps/pub/zz_generated.deepcopy.go b/apis/apps/pub/zz_generated.deepcopy.go
diff --git a/apis/apps/v1alpha1/advancedcronjob_types.go b/apis/apps/v1alpha1/advancedcronjob_types.go
@@ -17,7 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
-	batchv1beta1 "k8s.io/api/batch/v1beta1"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -31,6 +31,11 @@ type AdvancedCronJobSpec struct {
 	// The schedule in Cron format, see https://en.wikipedia.org/wiki/Cron.
 	Schedule string `json:"schedule" protobuf:"bytes,1,opt,name=schedule"`
 
+	// The time zone name for the given schedule, see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones.
+	// If not specified, this will default to the time zone of the kruise-controller-manager process.
+	// +optional
+	TimeZone *string `json:"timeZone,omitempty" protobuf:"bytes,8,opt,name=timeZone"`
+
 	// +kubebuilder:validation:Minimum=0
 
 	// Optional deadline in seconds for starting the job if it misses scheduled
@@ -73,7 +78,7 @@ type CronJobTemplate struct {
 	// +optional
 	// +kubebuilder:pruning:PreserveUnknownFields
 	// +kubebuilder:validation:Schemaless
-	JobTemplate *batchv1beta1.JobTemplateSpec `json:"jobTemplate,omitempty" protobuf:"bytes,1,opt,name=jobTemplate"`
+	JobTemplate *batchv1.JobTemplateSpec `json:"jobTemplate,omitempty" protobuf:"bytes,1,opt,name=jobTemplate"`
 
 	// Specifies the broadcastjob that will be created when executing a BroadcastCronJob.
 	// +optional

diff --git a/apis/apps/v1alpha1/cloneset_types.go b/apis/apps/v1alpha1/cloneset_types.go
@@ -79,7 +79,7 @@ type CloneSetSpec struct {
 	// Defaults to 0 (pod will be considered available as soon as it is ready)
 	MinReadySeconds int32 `json:"minReadySeconds,omitempty"`
 
-	// Lifecycle defines the lifecycle hooks for Pods pre-delete, in-place update.
+	// Lifecycle defines the lifecycle hooks for Pods pre-available(pre-normal), pre-delete, in-place update.
 	Lifecycle *appspub.Lifecycle `json:"lifecycle,omitempty"`
 }