sidecarset inject history revision (openkruise#1021)

Signed-off-by: mingzhou.swx <[email protected]>

Co-authored-by: mingzhou.swx <[email protected]>
Signed-off-by: Liu Zhenwei <[email protected]>

apis/apps/defaults/v1alpha1.go

@@ -43,6 +43,15 @@ func SetDefaultsSidecarSet(obj *v1alpha1.SidecarSet) {
 
 	//default setting history revision limitation
 	SetDefaultRevisionHistoryLimit(&obj.Spec.RevisionHistoryLimit)
+
+	//default setting injectRevisionStrategy
+	SetDefaultInjectRevision(&obj.Spec.InjectionStrategy)
+}
+
+func SetDefaultInjectRevision(strategy *v1alpha1.SidecarSetInjectionStrategy) {
+	if strategy.Revision != nil && strategy.Revision.Policy == "" {
+		strategy.Revision.Policy = v1alpha1.AlwaysSidecarSetInjectRevisionPolicy
+	}
 }
 
 func SetDefaultRevisionHistoryLimit(revisionHistoryLimit **int32) {

apis/apps/v1alpha1/sidecarset_types.go

@@ -22,6 +22,16 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
+const (
+	// SidecarSetCustomVersionLabel is designed to record and label the controllerRevision of sidecarSet.
+	// This label will be passed from SidecarSet to its corresponding ControllerRevision, users can use
+	// this label to selector the ControllerRevision they want.
+	// For example, users can update the label from "version-1" to "version-2" when they upgrade the
+	// sidecarSet to "version-2", and they write the "version-2" to InjectionStrategy.Revision.CustomVersion
+	// when they decided to promote the "version-2", to avoid some risks about gray deployment of SidecarSet.
+	SidecarSetCustomVersionLabel = "apps.kruise.io/sidecarset-custom-version"
+)
+
 // SidecarSetSpec defines the desired state of SidecarSet
 type SidecarSetSpec struct {
 	// selector is a label query over pods that should be injected
@@ -142,8 +152,38 @@ type SidecarSetInjectionStrategy struct {
 	// but the injected sidecar container remains updating and running.
 	// default is false
 	Paused bool `json:"paused,omitempty"`
+
+	// Revision can help users rolling update SidecarSet safely. If users set
+	// this filed, SidecarSet will try to inject specific revision according to
+	// different policies.
+	Revision *SidecarSetInjectRevision `json:"revision,omitempty"`
+}
+
+type SidecarSetInjectRevision struct {
+	// CustomVersion corresponds to label 'apps.kruise.io/sidecarset-custom-version' of (History) SidecarSet.
+	// SidecarSet will select the specific ControllerRevision via this CustomVersion, and then restore the
+	// history SidecarSet to inject specific version of the sidecar to pods.
+	// + optional
+	CustomVersion *string `json:"customVersion,omitempty"`
+	// RevisionName corresponds to a specific ControllerRevision name of SidecarSet that you want to inject to Pods.
+	// + optional
+	RevisionName *string `json:"revisionName,omitempty"`
+	// Policy describes the behavior of revision injection.
+	// Defaults to Always.
+	Policy SidecarSetInjectRevisionPolicy `json:"policy,omitempty"`
 }
 
+type SidecarSetInjectRevisionPolicy string
+
+const (
+	// AlwaysSidecarSetInjectRevisionPolicy means the SidecarSet will always inject
+	// the specific revision to Pods when pod creating, except matching UpdateStrategy.Selector.
+	AlwaysSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "Always"
+	// PartitionBasedSidecarSetInjectRevisionPolicy means the SidecarSet will inject the
+	// specific or the latest revision according to Partition.
+	//PartitionBasedSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "PartitionBased"
+)
+
 // SidecarSetUpdateStrategy indicates the strategy that the SidecarSet
 // controller will use to perform updates. It includes any additional parameters
 // necessary to perform the update for the indicated strategy.

config/crd/bases/apps.kruise.io_sidecarsets.yaml

@@ -229,6 +229,27 @@ spec:
                       to newly created Pods, but the injected sidecar container remains
                       updating and running. default is false
                     type: boolean
+                  revision:
+                    description: Revision can help users rolling update SidecarSet
+                      safely. If users set this filed, SidecarSet will try to inject
+                      specific revision according to different policies.
+                    properties:
+                      customVersion:
+                        description: CustomVersion corresponds to label 'apps.kruise.io/sidecarset-custom-version'
+                          of (History) SidecarSet. SidecarSet will select the specific
+                          ControllerRevision via this CustomVersion, and then restore
+                          the history SidecarSet to inject specific version of the
+                          sidecar to pods.
+                        type: string
+                      policy:
+                        description: Policy describes the behavior of revision injection.
+                          Defaults to Always.
+                        type: string
+                      revisionName:
+                        description: RevisionName corresponds to a specific ControllerRevision
+                          name of SidecarSet that you want to inject to Pods.
+                        type: string
+                    type: object
                 type: object
               namespace:
                 description: Namespace sidecarSet will only match the pods in the

pkg/webhook/sidecarset/mutating/hash.go → pkg/control/sidecarcontrol/hash.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package mutating
+package sidecarcontrol
 
 import (
 	"crypto/sha256"

pkg/control/sidecarcontrol/history_control.go

@@ -23,13 +23,18 @@ import (
 	"fmt"
 
 	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
-
+	"github.com/openkruise/kruise/pkg/util"
+	webhookutil "github.com/openkruise/kruise/pkg/webhook/util"
 	apps "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/pkg/controller/history"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -42,7 +47,8 @@ type HistoryControl interface {
 	CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error)
 	NewRevision(s *appsv1alpha1.SidecarSet, namespace string, revision int64, collisionCount *int32) (*apps.ControllerRevision, error)
 	NextRevision(revisions []*apps.ControllerRevision) int64
-	GetRevisionLabelSelector(s *appsv1alpha1.SidecarSet) *metav1.LabelSelector
+	GetRevisionSelector(s *appsv1alpha1.SidecarSet) labels.Selector
+	GetHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*appsv1alpha1.SidecarSet, error)
 }
 
 type realControl struct {
@@ -80,6 +86,15 @@ func (r *realControl) NewRevision(s *appsv1alpha1.SidecarSet, namespace string,
 	if cr.ObjectMeta.Annotations == nil {
 		cr.ObjectMeta.Annotations = make(map[string]string)
 	}
+	if s.Annotations[SidecarSetHashAnnotation] != "" {
+		cr.Annotations[SidecarSetHashAnnotation] = s.Annotations[SidecarSetHashAnnotation]
+	}
+	if s.Annotations[SidecarSetHashWithoutImageAnnotation] != "" {
+		cr.Annotations[SidecarSetHashWithoutImageAnnotation] = s.Annotations[SidecarSetHashWithoutImageAnnotation]
+	}
+	if s.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] != "" {
+		cr.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] = s.Labels[appsv1alpha1.SidecarSetCustomVersionLabel]
+	}
 	cr.Labels[SidecarSetKindName] = s.Name
 	for key, value := range s.Annotations {
 		cr.ObjectMeta.Annotations[key] = value
@@ -120,12 +135,18 @@ func (r *realControl) NextRevision(revisions []*apps.ControllerRevision) int64 {
 	return revisions[count-1].Revision + 1
 }
 
-func (r *realControl) GetRevisionLabelSelector(s *appsv1alpha1.SidecarSet) *metav1.LabelSelector {
-	return &metav1.LabelSelector{
+func (r *realControl) GetRevisionSelector(s *appsv1alpha1.SidecarSet) labels.Selector {
+	labelSelector := &metav1.LabelSelector{
 		MatchLabels: map[string]string{
 			SidecarSetKindName: s.GetName(),
 		},
 	}
+	selector, err := util.GetFastLabelSelector(labelSelector)
+	if err != nil {
+		// static error, just panic
+		panic("Incorrect label selector for ControllerRevision of SidecarSet.")
+	}
+	return selector
 }
 
 func (r *realControl) CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) {
@@ -162,6 +183,79 @@ func (r *realControl) CreateControllerRevision(parent metav1.Object, revision *a
 	}
 }
 
+func (r *realControl) GetHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*appsv1alpha1.SidecarSet, error) {
+	revision, err := r.getControllerRevision(sidecarSet, revisionInfo)
+	if err != nil || revision == nil {
+		return nil, err
+	}
+	clone := sidecarSet.DeepCopy()
+	cloneBytes, err := runtime.Encode(patchCodec, clone)
+	if err != nil {
+		klog.Errorf("Failed to encode sidecarSet(%v), error: %v", sidecarSet.Name, err)
+		return nil, err
+	}
+	patched, err := strategicpatch.StrategicMergePatch(cloneBytes, revision.Data.Raw, clone)
+	if err != nil {
+		klog.Errorf("Failed to merge sidecarSet(%v) and controllerRevision(%v): %v, error: %v", sidecarSet.Name, revision.Name, err)
+		return nil, err
+	}
+	// restore history from patch
+	restoredSidecarSet := &appsv1alpha1.SidecarSet{}
+	if err := json.Unmarshal(patched, restoredSidecarSet); err != nil {
+		return nil, err
+	}
+	// restore hash annotation and revision info
+	if err := restoreRevisionInfo(restoredSidecarSet, revision); err != nil {
+		return nil, err
+	}
+	return restoredSidecarSet, nil
+}
+
+func (r *realControl) getControllerRevision(set *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*apps.ControllerRevision, error) {
+	if revisionInfo == nil {
+		return nil, nil
+	}
+	switch {
+	case revisionInfo.RevisionName != nil:
+		revision := &apps.ControllerRevision{}
+		revisionKey := types.NamespacedName{
+			Namespace: webhookutil.GetNamespace(),
+			Name:      *revisionInfo.RevisionName,
+		}
+		if err := r.Client.Get(context.TODO(), revisionKey, revision); err != nil {
+			klog.Errorf("Failed to get ControllerRevision %v for SidecarSet(%v), err: %v", *revisionInfo.RevisionName, set.Name, err)
+			return nil, err
+		}
+		return revision, nil
+
+	case revisionInfo.CustomVersion != nil:
+		listOpts := []client.ListOption{
+			client.InNamespace(webhookutil.GetNamespace()),
+			&client.ListOptions{LabelSelector: r.GetRevisionSelector(set)},
+			client.MatchingLabels{appsv1alpha1.SidecarSetCustomVersionLabel: *revisionInfo.CustomVersion},
+		}
+		revisionList := &apps.ControllerRevisionList{}
+		if err := r.Client.List(context.TODO(), revisionList, listOpts...); err != nil {
+			klog.Errorf("Failed to get ControllerRevision for SidecarSet(%v), custom version: %v, err: %v", set.Name, *revisionInfo.CustomVersion, err)
+			return nil, err
+		}
+
+		var revisions []*apps.ControllerRevision
+		for i := range revisionList.Items {
+			revisions = append(revisions, &revisionList.Items[i])
+		}
+
+		if len(revisions) == 0 {
+			return nil, generateNotFoundError(set)
+		}
+		history.SortControllerRevisions(revisions)
+		return revisions[len(revisions)-1], nil
+	}
+
+	klog.Error("Failed to get controllerRevision due to both empty RevisionName and CustomVersion")
+	return nil, nil
+}
+
 func copySidecarSetSpecRevision(dst, src map[string]interface{}) {
 	// we will use patch instead of update operation to update pods in the future
 	// dst["$patch"] = "replace"
@@ -171,3 +265,44 @@ func copySidecarSetSpecRevision(dst, src map[string]interface{}) {
 	dst["initContainers"] = src["initContainers"]
 	dst["imagePullSecrets"] = src["imagePullSecrets"]
 }
+
+func restoreRevisionInfo(sidecarSet *appsv1alpha1.SidecarSet, revision *apps.ControllerRevision) error {
+	if sidecarSet.Annotations == nil {
+		sidecarSet.Annotations = map[string]string{}
+	}
+	if revision.Annotations[SidecarSetHashAnnotation] != "" {
+		sidecarSet.Annotations[SidecarSetHashAnnotation] = revision.Annotations[SidecarSetHashAnnotation]
+	} else {
+		hashCodeWithImage, err := SidecarSetHash(sidecarSet)
+		if err != nil {
+			return err
+		}
+		sidecarSet.Annotations[SidecarSetHashAnnotation] = hashCodeWithImage
+	}
+	if revision.Annotations[SidecarSetHashWithoutImageAnnotation] != "" {
+		sidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation] = revision.Annotations[SidecarSetHashWithoutImageAnnotation]
+	} else {
+		hashCodeWithoutImage, err := SidecarSetHashWithoutImage(sidecarSet)
+		if err != nil {
+			return err
+		}
+		sidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation] = hashCodeWithoutImage
+	}
+	sidecarSet.Status.LatestRevision = revision.Name
+	return nil
+}
+
+func MockSidecarSetForRevision(set *appsv1alpha1.SidecarSet) metav1.Object {
+	return &metav1.ObjectMeta{
+		UID:       set.UID,
+		Name:      set.Name,
+		Namespace: webhookutil.GetNamespace(),
+	}
+}
+
+func generateNotFoundError(set *appsv1alpha1.SidecarSet) error {
+	return errors.NewNotFound(schema.GroupResource{
+		Group:    apps.GroupName,
+		Resource: "ControllerRevision",
+	}, set.Name)
+}

pkg/control/sidecarcontrol/util.go

@@ -67,8 +67,8 @@ type SidecarSetUpgradeSpec struct {
 	UpdateTimestamp              metav1.Time `json:"updateTimestamp"`
 	SidecarSetHash               string      `json:"hash"`
 	SidecarSetName               string      `json:"sidecarSetName"`
-	SidecarList                  []string    `json:"sidecarList"`        // sidecarSet container list
-	SidecarSetControllerRevision string      `json:"controllerRevision"` // sidecarSet controllerRevision name
+	SidecarList                  []string    `json:"sidecarList"`                  // sidecarSet container list
+	SidecarSetControllerRevision string      `json:"controllerRevision,omitempty"` // sidecarSet controllerRevision name
 }
 
 // PodMatchSidecarSet determines if pod match Selector of sidecar.

pkg/controller/sidecarset/sidecarset_pod_event_handler.go

@@ -2,6 +2,7 @@ package sidecarset
 
 import (
 	"context"
+	"reflect"
 	"strings"
 	"time"
 
@@ -179,5 +180,12 @@ func isPodConsistentChanged(oldPod, newPod *corev1.Pod, sidecarSet *appsv1alpha1
 		return true, enqueueDelayTime
 	}
 
+	// If the pod's labels changed, and sidecarSet enable selector updateStrategy, should reconcile.
+	if !reflect.DeepEqual(oldPod.Labels, newPod.Labels) && sidecarSet.Spec.UpdateStrategy.Selector != nil {
+		klog.V(3).Infof("pod(%s/%s) Labels changed and sidecarSet (%s) enable selector upgrade strategy, "+
+			"and reconcile sidecarSet", newPod.Namespace, newPod.Name, sidecarSet.Name)
+		return true, 0
+	}
+
 	return false, enqueueDelayTime
 }