sidecarset inject history revision

Signed-off-by: mingzhou.swx <[email protected]>

apis/apps/v1alpha1/sidecarset_types.go

@@ -142,8 +142,34 @@ type SidecarSetInjectionStrategy struct {
 	// but the injected sidecar container remains updating and running.
 	// default is false
 	Paused bool `json:"paused,omitempty"`
+
+	// Revision can help users rolling update SidecarSet safely. If users set
+	// this filed, SidecarSet will try to decide inject update revision according to
+	// different policies.
+	Revision *SidecarSetInjectRevision `json:"revision,omitempty"`
+}
+
+type SidecarSetInjectRevision struct {
+	// ID corresponds to label 'apps.kruise.io/sidecarset-revision-id' of (History) SidecarSet.
+	// SidecarSet will select the specific ControllerRevision via this ID, and then restore the
+	// history SidecarSet to inject specific version of the sidecar to pods.
+	ID string `json:"id"`
+	// Policy describes the behavior of revision injection.
+	// Defaults to Always.
+	Policy SidecarSetInjectRevisionPolicy `json:"policy,omitempty"`
 }
 
+type SidecarSetInjectRevisionPolicy string
+
+const (
+	// AlwaysSidecarSetInjectRevisionPolicy means the SidecarSet will always inject
+	// the specific revision to Pods when pod creating, except matching UpdateStrategy.Selector.
+	AlwaysSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "Always"
+	// AdaptiveSidecarSetInjectRevisionPolicy means the SidecarSet will inject the
+	// specific or the latest revision according to Partition.
+	//AdaptiveSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "Adaptive"
+)
+
 // SidecarSetUpdateStrategy indicates the strategy that the SidecarSet
 // controller will use to perform updates. It includes any additional parameters
 // necessary to perform the update for the indicated strategy.

config/crd/bases/apps.kruise.io_sidecarsets.yaml

@@ -229,6 +229,25 @@ spec:
                       to newly created Pods, but the injected sidecar container remains
                       updating and running. default is false
                     type: boolean
+                  revision:
+                    description: Revision can help users rolling update SidecarSet
+                      safely. If users set this filed, SidecarSet will try to decide
+                      inject update revision according to different policies.
+                    properties:
+                      id:
+                        description: ID corresponds to label 'apps.kruise.io/sidecarset-revision-id'
+                          of (History) SidecarSet. SidecarSet will select the specific
+                          ControllerRevision via this ID, and then restore the history
+                          SidecarSet to inject specific version of the sidecar to
+                          pods.
+                        type: string
+                      policy:
+                        description: Policy describes the behavior of revision injection.
+                          Defaults to Always.
+                        type: string
+                    required:
+                    - id
+                    type: object
                 type: object
               namespace:
                 description: Namespace sidecarSet will only match the pods in the

pkg/webhook/sidecarset/mutating/hash.go → pkg/control/sidecarcontrol/hash.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package mutating
+package sidecarcontrol
 
 import (
 	"crypto/sha256"
@@ -66,3 +66,17 @@ func encodeSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (string, error) {
 func hash(data string) string {
 	return fmt.Sprintf("%x", sha256.Sum256([]byte(data)))
 }
+
+func RecalculateSidecarSetHash(sidecarSet *appsv1alpha1.SidecarSet) error {
+	hashCodeWithImage, err := SidecarSetHash(sidecarSet)
+	if err != nil {
+		return err
+	}
+	hashCodeWithoutImage, err := SidecarSetHashWithoutImage(sidecarSet)
+	if err != nil {
+		return err
+	}
+	sidecarSet.Annotations[SidecarSetHashAnnotation] = hashCodeWithImage
+	sidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation] = hashCodeWithoutImage
+	return nil
+}

pkg/control/sidecarcontrol/history_control.go

@@ -23,13 +23,16 @@ import (
 	"fmt"
 
 	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
-
+	webhookutil "github.com/openkruise/kruise/pkg/webhook/util"
 	apps "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/pkg/controller/history"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -43,6 +46,7 @@ type HistoryControl interface {
 	NewRevision(s *appsv1alpha1.SidecarSet, namespace string, revision int64, collisionCount *int32) (*apps.ControllerRevision, error)
 	NextRevision(revisions []*apps.ControllerRevision) int64
 	GetRevisionLabelSelector(s *appsv1alpha1.SidecarSet) *metav1.LabelSelector
+	GetHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionName string) (*appsv1alpha1.SidecarSet, error)
 }
 
 type realControl struct {
@@ -80,6 +84,9 @@ func (r *realControl) NewRevision(s *appsv1alpha1.SidecarSet, namespace string,
 	if cr.ObjectMeta.Annotations == nil {
 		cr.ObjectMeta.Annotations = make(map[string]string)
 	}
+	if s.Labels[SidecarSetRevisionIDLabel] != "" {
+		cr.Labels[SidecarSetRevisionIDLabel] = s.Labels[SidecarSetRevisionIDLabel]
+	}
 	cr.Labels[SidecarSetKindName] = s.Name
 	for key, value := range s.Annotations {
 		cr.ObjectMeta.Annotations[key] = value
@@ -162,6 +169,56 @@ func (r *realControl) CreateControllerRevision(parent metav1.Object, revision *a
 	}
 }
 
+func (r *realControl) GetHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionID string) (*appsv1alpha1.SidecarSet, error) {
+	if revisionID == "" {
+		return nil, generateNotFoundError(sidecarSet)
+	}
+
+	listOpts := []client.ListOption{
+		client.InNamespace(webhookutil.GetNamespace()),
+		client.MatchingLabels{SidecarSetRevisionIDLabel: revisionID},
+	}
+	revisionList := &apps.ControllerRevisionList{}
+	if err := r.Client.List(context.TODO(), revisionList, listOpts...); err != nil {
+		klog.Errorf("Failed to get ControllerRevision ID: %s, err %v", revisionID, err)
+		return nil, err
+	}
+
+	var revisions []*apps.ControllerRevision
+	for i := range revisionList.Items {
+		revisions = append(revisions, &revisionList.Items[i])
+	}
+
+	if len(revisions) == 0 {
+		return nil, generateNotFoundError(sidecarSet)
+	}
+	history.SortControllerRevisions(revisions)
+	revision := revisions[len(revisions)-1]
+
+	// calculate patch
+	clone := sidecarSet.DeepCopy()
+	cloneBytes, err := runtime.Encode(patchCodec, clone)
+	if err != nil {
+		klog.Errorf("Failed to encode sidecarSet(%v), error: %v", sidecarSet.Name, err)
+		return nil, err
+	}
+	patched, err := strategicpatch.StrategicMergePatch(cloneBytes, revision.Data.Raw, clone)
+	if err != nil {
+		klog.Errorf("Failed to merge sidecarSet(%v) and controllerRevision, ID: %v, error: %v", sidecarSet.Name, revisionID, err)
+		return nil, err
+	}
+	// restore history from patch
+	restoredSidecarSet := &appsv1alpha1.SidecarSet{}
+	if err := json.Unmarshal(patched, restoredSidecarSet); err != nil {
+		return nil, err
+	}
+	// re-calculate sidecarSet hash
+	if err := RecalculateSidecarSetHash(restoredSidecarSet); err != nil {
+		return nil, err
+	}
+	return restoredSidecarSet, nil
+}
+
 func copySidecarSetSpecRevision(dst, src map[string]interface{}) {
 	// we will use patch instead of update operation to update pods in the future
 	// dst["$patch"] = "replace"
@@ -171,3 +228,10 @@ func copySidecarSetSpecRevision(dst, src map[string]interface{}) {
 	dst["initContainers"] = src["initContainers"]
 	dst["imagePullSecrets"] = src["imagePullSecrets"]
 }
+
+func generateNotFoundError(set *appsv1alpha1.SidecarSet) error {
+	return errors.NewNotFound(schema.GroupResource{
+		Group:    apps.GroupName,
+		Resource: "ControllerRevision",
+	}, set.Name)
+}

pkg/control/sidecarcontrol/util.go

@@ -54,6 +54,8 @@ const (
 	// SidecarsetInplaceUpdateStateKey records the state of inplace-update.
 	// The value of annotation is SidecarsetInplaceUpdateStateKey.
 	SidecarsetInplaceUpdateStateKey string = "kruise.io/sidecarset-inplace-update-state"
+
+	SidecarSetRevisionIDLabel = "apps.kruise.io/sidecarset-revision-id"
 )
 
 var (

pkg/controller/sidecarset/sidecarset_pod_event_handler.go

@@ -2,6 +2,7 @@ package sidecarset
 
 import (
 	"context"
+	"reflect"
 	"strings"
 	"time"
 
@@ -179,5 +180,12 @@ func isPodConsistentChanged(oldPod, newPod *corev1.Pod, sidecarSet *appsv1alpha1
 		return true, enqueueDelayTime
 	}
 
+	// If the pod's labels changed, and sidecarSet enable selector updateStrategy, should reconcile.
+	if !reflect.DeepEqual(oldPod.Labels, newPod.Labels) && sidecarSet.Spec.UpdateStrategy.Selector != nil {
+		klog.V(3).Infof("pod(%s/%s) Labels changed and sidecarSet (%s) enable selector upgrade strategy, "+
+			"and reconcile sidecarSet", newPod.Namespace, newPod.Name, sidecarSet.Name)
+		return true, 0
+	}
+
 	return false, enqueueDelayTime
 }

pkg/controller/sidecarset/sidecarset_processor.go

@@ -369,6 +369,18 @@ func (p *Processor) registerLatestRevision(sidecarSet *appsv1alpha1.SidecarSet,
 		revisions = append(revisions, latestRevision)
 	}
 
+	// update revision id for the latest controller revision
+	currentRevisionID := sidecarSet.Labels[sidecarcontrol.SidecarSetRevisionIDLabel]
+	if currentRevisionID != "" && currentRevisionID != latestRevision.Labels[sidecarcontrol.SidecarSetRevisionIDLabel] {
+		latestRevisionCLone := latestRevision.DeepCopy()
+		patchBody := fmt.Sprintf(`{"metadata":{"labels":{"%v":"%v"}}}`, sidecarcontrol.SidecarSetRevisionIDLabel, currentRevisionID)
+		err = p.Client.Patch(context.TODO(), latestRevisionCLone, client.RawPatch(types.StrategicMergePatchType, []byte(patchBody)))
+		if err != nil {
+			klog.Errorf("Failed to patch revision id to latest revision %v, err: %v", latestRevision.Name, err)
+			return nil, collisionCount, err
+		}
+	}
+
 	// only store limited history revisions
 	if err = p.truncateHistory(revisions, sidecarSet, pods); err != nil {
 		klog.Errorf("Failed to truncate history for %s: err: %v", sidecarSet.Name, err)
@@ -395,9 +407,9 @@ func (p *Processor) truncateHistory(revisions []*apps.ControllerRevision, s *app
 	// the number of revisions need to delete
 	deletionCount := revisionCount - limitation
 	// only delete the revisions that no pods use.
-	activeRevisions := filterActiveRevisions(s, pods)
+	activeRevisions := filterActiveRevisions(s, pods, revisions)
 	for i := 0; i < revisionCount-1 && deletionCount > 0; i++ {
-		if !activeRevisions.Has(revisions[i].Name) { // && revision.InjectionStrategy.ControllerRevision != revisions[i].Name
+		if !activeRevisions.Has(revisions[i].Name) {
 			if err := p.historyController.DeleteControllerRevision(revisions[i]); err != nil && !errors.IsNotFound(err) {
 				return err
 			}
@@ -412,13 +424,28 @@ func (p *Processor) truncateHistory(revisions []*apps.ControllerRevision, s *app
 	return nil
 }
 
-func filterActiveRevisions(s *appsv1alpha1.SidecarSet, pods []*corev1.Pod) sets.String {
+func filterActiveRevisions(s *appsv1alpha1.SidecarSet, pods []*corev1.Pod, revisions []*apps.ControllerRevision) sets.String {
 	activeRevisions := sets.NewString()
 	for _, pod := range pods {
 		if revision := sidecarcontrol.GetPodSidecarSetControllerRevision(s.Name, pod); revision != "" {
 			activeRevisions.Insert(revision)
 		}
 	}
+
+	if s.Spec.InjectionStrategy.Revision != nil {
+		equalRevisions := make([]*apps.ControllerRevision, 0)
+		for i := range revisions {
+			revision := revisions[i]
+			if revision.Labels[sidecarcontrol.SidecarSetRevisionIDLabel] == s.Spec.InjectionStrategy.Revision.ID {
+				equalRevisions = append(equalRevisions, revision)
+			}
+		}
+		if len(equalRevisions) > 0 {
+			history.SortControllerRevisions(equalRevisions)
+			activeRevisions.Insert(equalRevisions[len(equalRevisions)-1].Name)
+		}
+	}
+
 	return activeRevisions
 }
 

pkg/webhook/pod/mutating/sidecarset.go

@@ -31,8 +31,10 @@ import (
 	admissionv1 "k8s.io/api/admission/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 )
 
@@ -79,9 +81,14 @@ func (h *PodCreateHandler) sidecarsetMutatingPod(ctx context.Context, req admiss
 		} else if !matched {
 			continue
 		}
+		// get user-specific revision or the latest revision of SidecarSet
+		suitableSidecarSet, err := h.getSuitableRevisionSidecarSet(&sidecarSet, pod)
+		if err != nil {
+			return err
+		}
 		// check whether sidecarSet is active
 		// when sidecarSet is not active, it will not perform injections and upgrades process.
-		control := sidecarcontrol.New(sidecarSet.DeepCopy())
+		control := sidecarcontrol.New(suitableSidecarSet)
 		if !control.IsActiveSidecarSet() {
 			continue
 		}
@@ -139,6 +146,44 @@ func (h *PodCreateHandler) sidecarsetMutatingPod(ctx context.Context, req admiss
 	return nil
 }
 
+func (h *PodCreateHandler) getSuitableRevisionSidecarSet(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) (*appsv1alpha1.SidecarSet, error) {
+	if sidecarSet.Spec.InjectionStrategy.Revision == nil || len(sidecarSet.Spec.InjectionStrategy.Revision.ID) == 0 {
+		return sidecarSet.DeepCopy(), nil
+	}
+
+	if sidecarSet.Spec.UpdateStrategy.Selector != nil {
+		selector, err := metav1.LabelSelectorAsSelector(sidecarSet.Spec.UpdateStrategy.Selector)
+		if err != nil {
+			return nil, err
+		}
+
+		// inject new revision of sidecarSet if it matches selector labels
+		if selector.Matches(labels.Set(pod.Labels)) {
+			klog.Infof("Pod %v matches sidecarSet %s upgrade selector", client.ObjectKeyFromObject(pod), sidecarSet.Name)
+			return sidecarSet.DeepCopy(), nil
+		}
+	}
+
+	// TODO: support 'Adaptive' policy to inject old/new revision according to Partition
+	switch sidecarSet.Spec.InjectionStrategy.Revision.Policy {
+	case "", appsv1alpha1.AlwaysSidecarSetInjectRevisionPolicy:
+		return h.getSpecificHistorySidecarSet(sidecarSet)
+	}
+	return h.getSpecificHistorySidecarSet(sidecarSet)
+}
+
+func (h *PodCreateHandler) getSpecificHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (*appsv1alpha1.SidecarSet, error) {
+	// else return its corresponding history revision
+	hc := sidecarcontrol.NewHistoryControl(h.Client)
+	history, err := hc.GetHistorySidecarSet(sidecarSet, sidecarSet.Spec.InjectionStrategy.Revision.ID)
+	if err != nil || history == nil {
+		klog.Warningf("Failed to restore history revision for SidecarSet %v, ControllerRevision name %v:, error: %v",
+			sidecarSet.Name, sidecarSet.Spec.InjectionStrategy.Revision, err)
+		return nil, err
+	}
+	return history, nil
+}
+
 func mergeSidecarSecrets(secretsInPod, secretsInSidecar []corev1.LocalObjectReference) (allSecrets []corev1.LocalObjectReference) {
 	secretFilter := make(map[string]bool)
 	for _, podSecret := range secretsInPod {