diff --git a/apis/apps/defaults/v1alpha1.go b/apis/apps/defaults/v1alpha1.go index 9f267689e5..818d8b9b2e 100644 --- a/apis/apps/defaults/v1alpha1.go +++ b/apis/apps/defaults/v1alpha1.go @@ -43,6 +43,15 @@ func SetDefaultsSidecarSet(obj *v1alpha1.SidecarSet) { //default setting history revision limitation SetDefaultRevisionHistoryLimit(&obj.Spec.RevisionHistoryLimit) + + //default setting injectRevisionStrategy + SetDefaultInjectRevision(&obj.Spec.InjectionStrategy) +} + +func SetDefaultInjectRevision(strategy *v1alpha1.SidecarSetInjectionStrategy) { + if strategy.Revision != nil && strategy.Revision.Policy == "" { + strategy.Revision.Policy = v1alpha1.AlwaysSidecarSetInjectRevisionPolicy + } } func SetDefaultRevisionHistoryLimit(revisionHistoryLimit **int32) { diff --git a/apis/apps/v1alpha1/sidecarset_types.go b/apis/apps/v1alpha1/sidecarset_types.go index b150e6dfe4..652aefffc6 100644 --- a/apis/apps/v1alpha1/sidecarset_types.go +++ b/apis/apps/v1alpha1/sidecarset_types.go @@ -22,6 +22,16 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) +const ( + // SidecarSetCustomVersionLabel is designed to record and label the controllerRevision of sidecarSet. + // This label will be passed from SidecarSet to its corresponding ControllerRevision, users can use + // this label to selector the ControllerRevision they want. + // For example, users can update the label from "version-1" to "version-2" when they upgrade the + // sidecarSet to "version-2", and they write the "version-2" to InjectionStrategy.Revision.CustomVersion + // when they decided to promote the "version-2", to avoid some risks about gray deployment of SidecarSet. + SidecarSetCustomVersionLabel = "apps.kruise.io/sidecarset-custom-version" +) + // SidecarSetSpec defines the desired state of SidecarSet type SidecarSetSpec struct { // selector is a label query over pods that should be injected @@ -142,8 +152,38 @@ type SidecarSetInjectionStrategy struct { // but the injected sidecar container remains updating and running. // default is false Paused bool `json:"paused,omitempty"` + + // Revision can help users rolling update SidecarSet safely. If users set + // this filed, SidecarSet will try to inject specific revision according to + // different policies. + Revision *SidecarSetInjectRevision `json:"revision,omitempty"` +} + +type SidecarSetInjectRevision struct { + // CustomVersion corresponds to label 'apps.kruise.io/sidecarset-custom-version' of (History) SidecarSet. + // SidecarSet will select the specific ControllerRevision via this CustomVersion, and then restore the + // history SidecarSet to inject specific version of the sidecar to pods. + // + optional + CustomVersion *string `json:"customVersion,omitempty"` + // RevisionName corresponds to a specific ControllerRevision name of SidecarSet that you want to inject to Pods. + // + optional + RevisionName *string `json:"revisionName,omitempty"` + // Policy describes the behavior of revision injection. + // Defaults to Always. + Policy SidecarSetInjectRevisionPolicy `json:"policy,omitempty"` } +type SidecarSetInjectRevisionPolicy string + +const ( + // AlwaysSidecarSetInjectRevisionPolicy means the SidecarSet will always inject + // the specific revision to Pods when pod creating, except matching UpdateStrategy.Selector. + AlwaysSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "Always" + // PartitionBasedSidecarSetInjectRevisionPolicy means the SidecarSet will inject the + // specific or the latest revision according to Partition. + //PartitionBasedSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "PartitionBased" +) + // SidecarSetUpdateStrategy indicates the strategy that the SidecarSet // controller will use to perform updates. It includes any additional parameters // necessary to perform the update for the indicated strategy. diff --git a/apis/apps/v1alpha1/zz_generated.deepcopy.go b/apis/apps/v1alpha1/zz_generated.deepcopy.go index 72cd8b292d..85535bd7b7 100644 --- a/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -2183,9 +2183,39 @@ func (in *SidecarSet) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SidecarSetInjectRevision) DeepCopyInto(out *SidecarSetInjectRevision) { + *out = *in + if in.CustomVersion != nil { + in, out := &in.CustomVersion, &out.CustomVersion + *out = new(string) + **out = **in + } + if in.RevisionName != nil { + in, out := &in.RevisionName, &out.RevisionName + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SidecarSetInjectRevision. +func (in *SidecarSetInjectRevision) DeepCopy() *SidecarSetInjectRevision { + if in == nil { + return nil + } + out := new(SidecarSetInjectRevision) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SidecarSetInjectionStrategy) DeepCopyInto(out *SidecarSetInjectionStrategy) { *out = *in + if in.Revision != nil { + in, out := &in.Revision, &out.Revision + *out = new(SidecarSetInjectRevision) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SidecarSetInjectionStrategy. @@ -2260,7 +2290,7 @@ func (in *SidecarSetSpec) DeepCopyInto(out *SidecarSetSpec) { } } in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy) - out.InjectionStrategy = in.InjectionStrategy + in.InjectionStrategy.DeepCopyInto(&out.InjectionStrategy) if in.ImagePullSecrets != nil { in, out := &in.ImagePullSecrets, &out.ImagePullSecrets *out = make([]v1.LocalObjectReference, len(*in)) diff --git a/config/crd/bases/apps.kruise.io_sidecarsets.yaml b/config/crd/bases/apps.kruise.io_sidecarsets.yaml index db5bc300ed..9fbe66e8d5 100644 --- a/config/crd/bases/apps.kruise.io_sidecarsets.yaml +++ b/config/crd/bases/apps.kruise.io_sidecarsets.yaml @@ -229,6 +229,27 @@ spec: to newly created Pods, but the injected sidecar container remains updating and running. default is false type: boolean + revision: + description: Revision can help users rolling update SidecarSet + safely. If users set this filed, SidecarSet will try to inject + specific revision according to different policies. + properties: + customVersion: + description: CustomVersion corresponds to label 'apps.kruise.io/sidecarset-custom-version' + of (History) SidecarSet. SidecarSet will select the specific + ControllerRevision via this CustomVersion, and then restore + the history SidecarSet to inject specific version of the + sidecar to pods. + type: string + policy: + description: Policy describes the behavior of revision injection. + Defaults to Always. + type: string + revisionName: + description: RevisionName corresponds to a specific ControllerRevision + name of SidecarSet that you want to inject to Pods. + type: string + type: object type: object namespace: description: Namespace sidecarSet will only match the pods in the diff --git a/pkg/webhook/sidecarset/mutating/hash.go b/pkg/control/sidecarcontrol/hash.go similarity index 98% rename from pkg/webhook/sidecarset/mutating/hash.go rename to pkg/control/sidecarcontrol/hash.go index 35f58c37c8..79a6dcb2d8 100644 --- a/pkg/webhook/sidecarset/mutating/hash.go +++ b/pkg/control/sidecarcontrol/hash.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mutating +package sidecarcontrol import ( "crypto/sha256" diff --git a/pkg/control/sidecarcontrol/history_control.go b/pkg/control/sidecarcontrol/history_control.go index 2323e13756..be4d8ff0df 100644 --- a/pkg/control/sidecarcontrol/history_control.go +++ b/pkg/control/sidecarcontrol/history_control.go @@ -23,13 +23,18 @@ import ( "fmt" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - + "github.com/openkruise/kruise/pkg/util" + webhookutil "github.com/openkruise/kruise/pkg/webhook/util" apps "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/controller/history" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -42,7 +47,8 @@ type HistoryControl interface { CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) NewRevision(s *appsv1alpha1.SidecarSet, namespace string, revision int64, collisionCount *int32) (*apps.ControllerRevision, error) NextRevision(revisions []*apps.ControllerRevision) int64 - GetRevisionLabelSelector(s *appsv1alpha1.SidecarSet) *metav1.LabelSelector + GetRevisionSelector(s *appsv1alpha1.SidecarSet) labels.Selector + GetHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*appsv1alpha1.SidecarSet, error) } type realControl struct { @@ -80,6 +86,15 @@ func (r *realControl) NewRevision(s *appsv1alpha1.SidecarSet, namespace string, if cr.ObjectMeta.Annotations == nil { cr.ObjectMeta.Annotations = make(map[string]string) } + if s.Annotations[SidecarSetHashAnnotation] != "" { + cr.Annotations[SidecarSetHashAnnotation] = s.Annotations[SidecarSetHashAnnotation] + } + if s.Annotations[SidecarSetHashWithoutImageAnnotation] != "" { + cr.Annotations[SidecarSetHashWithoutImageAnnotation] = s.Annotations[SidecarSetHashWithoutImageAnnotation] + } + if s.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] != "" { + cr.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] = s.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] + } cr.Labels[SidecarSetKindName] = s.Name for key, value := range s.Annotations { cr.ObjectMeta.Annotations[key] = value @@ -120,12 +135,18 @@ func (r *realControl) NextRevision(revisions []*apps.ControllerRevision) int64 { return revisions[count-1].Revision + 1 } -func (r *realControl) GetRevisionLabelSelector(s *appsv1alpha1.SidecarSet) *metav1.LabelSelector { - return &metav1.LabelSelector{ +func (r *realControl) GetRevisionSelector(s *appsv1alpha1.SidecarSet) labels.Selector { + labelSelector := &metav1.LabelSelector{ MatchLabels: map[string]string{ SidecarSetKindName: s.GetName(), }, } + selector, err := util.GetFastLabelSelector(labelSelector) + if err != nil { + // static error, just panic + panic("Incorrect label selector for ControllerRevision of SidecarSet.") + } + return selector } func (r *realControl) CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) { @@ -162,6 +183,79 @@ func (r *realControl) CreateControllerRevision(parent metav1.Object, revision *a } } +func (r *realControl) GetHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*appsv1alpha1.SidecarSet, error) { + revision, err := r.getControllerRevision(sidecarSet, revisionInfo) + if err != nil || revision == nil { + return nil, err + } + clone := sidecarSet.DeepCopy() + cloneBytes, err := runtime.Encode(patchCodec, clone) + if err != nil { + klog.Errorf("Failed to encode sidecarSet(%v), error: %v", sidecarSet.Name, err) + return nil, err + } + patched, err := strategicpatch.StrategicMergePatch(cloneBytes, revision.Data.Raw, clone) + if err != nil { + klog.Errorf("Failed to merge sidecarSet(%v) and controllerRevision(%v): %v, error: %v", sidecarSet.Name, revision.Name, err) + return nil, err + } + // restore history from patch + restoredSidecarSet := &appsv1alpha1.SidecarSet{} + if err := json.Unmarshal(patched, restoredSidecarSet); err != nil { + return nil, err + } + // restore hash annotation and revision info + if err := restoreRevisionInfo(restoredSidecarSet, revision); err != nil { + return nil, err + } + return restoredSidecarSet, nil +} + +func (r *realControl) getControllerRevision(set *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*apps.ControllerRevision, error) { + if revisionInfo == nil { + return nil, nil + } + switch { + case revisionInfo.RevisionName != nil: + revision := &apps.ControllerRevision{} + revisionKey := types.NamespacedName{ + Namespace: webhookutil.GetNamespace(), + Name: *revisionInfo.RevisionName, + } + if err := r.Client.Get(context.TODO(), revisionKey, revision); err != nil { + klog.Errorf("Failed to get ControllerRevision %v for SidecarSet(%v), err: %v", *revisionInfo.RevisionName, set.Name, err) + return nil, err + } + return revision, nil + + case revisionInfo.CustomVersion != nil: + listOpts := []client.ListOption{ + client.InNamespace(webhookutil.GetNamespace()), + &client.ListOptions{LabelSelector: r.GetRevisionSelector(set)}, + client.MatchingLabels{appsv1alpha1.SidecarSetCustomVersionLabel: *revisionInfo.CustomVersion}, + } + revisionList := &apps.ControllerRevisionList{} + if err := r.Client.List(context.TODO(), revisionList, listOpts...); err != nil { + klog.Errorf("Failed to get ControllerRevision for SidecarSet(%v), custom version: %v, err: %v", set.Name, *revisionInfo.CustomVersion, err) + return nil, err + } + + var revisions []*apps.ControllerRevision + for i := range revisionList.Items { + revisions = append(revisions, &revisionList.Items[i]) + } + + if len(revisions) == 0 { + return nil, generateNotFoundError(set) + } + history.SortControllerRevisions(revisions) + return revisions[len(revisions)-1], nil + } + + klog.Error("Failed to get controllerRevision due to both empty RevisionName and CustomVersion") + return nil, nil +} + func copySidecarSetSpecRevision(dst, src map[string]interface{}) { // we will use patch instead of update operation to update pods in the future // dst["$patch"] = "replace" @@ -171,3 +265,44 @@ func copySidecarSetSpecRevision(dst, src map[string]interface{}) { dst["initContainers"] = src["initContainers"] dst["imagePullSecrets"] = src["imagePullSecrets"] } + +func restoreRevisionInfo(sidecarSet *appsv1alpha1.SidecarSet, revision *apps.ControllerRevision) error { + if sidecarSet.Annotations == nil { + sidecarSet.Annotations = map[string]string{} + } + if revision.Annotations[SidecarSetHashAnnotation] != "" { + sidecarSet.Annotations[SidecarSetHashAnnotation] = revision.Annotations[SidecarSetHashAnnotation] + } else { + hashCodeWithImage, err := SidecarSetHash(sidecarSet) + if err != nil { + return err + } + sidecarSet.Annotations[SidecarSetHashAnnotation] = hashCodeWithImage + } + if revision.Annotations[SidecarSetHashWithoutImageAnnotation] != "" { + sidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation] = revision.Annotations[SidecarSetHashWithoutImageAnnotation] + } else { + hashCodeWithoutImage, err := SidecarSetHashWithoutImage(sidecarSet) + if err != nil { + return err + } + sidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation] = hashCodeWithoutImage + } + sidecarSet.Status.LatestRevision = revision.Name + return nil +} + +func MockSidecarSetForRevision(set *appsv1alpha1.SidecarSet) metav1.Object { + return &metav1.ObjectMeta{ + UID: set.UID, + Name: set.Name, + Namespace: webhookutil.GetNamespace(), + } +} + +func generateNotFoundError(set *appsv1alpha1.SidecarSet) error { + return errors.NewNotFound(schema.GroupResource{ + Group: apps.GroupName, + Resource: "ControllerRevision", + }, set.Name) +} diff --git a/pkg/control/sidecarcontrol/util.go b/pkg/control/sidecarcontrol/util.go index 4250432b05..c91c15eebb 100644 --- a/pkg/control/sidecarcontrol/util.go +++ b/pkg/control/sidecarcontrol/util.go @@ -67,8 +67,8 @@ type SidecarSetUpgradeSpec struct { UpdateTimestamp metav1.Time `json:"updateTimestamp"` SidecarSetHash string `json:"hash"` SidecarSetName string `json:"sidecarSetName"` - SidecarList []string `json:"sidecarList"` // sidecarSet container list - SidecarSetControllerRevision string `json:"controllerRevision"` // sidecarSet controllerRevision name + SidecarList []string `json:"sidecarList"` // sidecarSet container list + SidecarSetControllerRevision string `json:"controllerRevision,omitempty"` // sidecarSet controllerRevision name } // PodMatchSidecarSet determines if pod match Selector of sidecar. diff --git a/pkg/controller/sidecarset/sidecarset_pod_event_handler.go b/pkg/controller/sidecarset/sidecarset_pod_event_handler.go index 9c218eeeaa..5a9c36fea7 100644 --- a/pkg/controller/sidecarset/sidecarset_pod_event_handler.go +++ b/pkg/controller/sidecarset/sidecarset_pod_event_handler.go @@ -2,6 +2,7 @@ package sidecarset import ( "context" + "reflect" "strings" "time" @@ -179,5 +180,12 @@ func isPodConsistentChanged(oldPod, newPod *corev1.Pod, sidecarSet *appsv1alpha1 return true, enqueueDelayTime } + // If the pod's labels changed, and sidecarSet enable selector updateStrategy, should reconcile. + if !reflect.DeepEqual(oldPod.Labels, newPod.Labels) && sidecarSet.Spec.UpdateStrategy.Selector != nil { + klog.V(3).Infof("pod(%s/%s) Labels changed and sidecarSet (%s) enable selector upgrade strategy, "+ + "and reconcile sidecarSet", newPod.Namespace, newPod.Name, sidecarSet.Name) + return true, 0 + } + return false, enqueueDelayTime } diff --git a/pkg/controller/sidecarset/sidecarset_processor.go b/pkg/controller/sidecarset/sidecarset_processor.go index 0080be9e4f..12b16eccb2 100644 --- a/pkg/controller/sidecarset/sidecarset_processor.go +++ b/pkg/controller/sidecarset/sidecarset_processor.go @@ -305,21 +305,14 @@ func (p *Processor) getSelectedPods(namespaces []string, selector labels.Selecto return } -func (p *Processor) registerLatestRevision(sidecarSet *appsv1alpha1.SidecarSet, pods []*corev1.Pod) ( +func (p *Processor) registerLatestRevision(set *appsv1alpha1.SidecarSet, pods []*corev1.Pod) ( latestRevision *apps.ControllerRevision, collisionCount int32, err error, ) { + sidecarSet := set.DeepCopy() // get revision selector of this sidecarSet hc := sidecarcontrol.NewHistoryControl(p.Client) - selector, err := util.GetFastLabelSelector( - hc.GetRevisionLabelSelector(sidecarSet), - ) - if err != nil { - klog.Errorf("Failed to convert labels to selector, err %v, name %v", err, sidecarSet.Name) - return nil, collisionCount, nil - } - // list all revisions - revisions, err := p.historyController.ListControllerRevisions(sidecarSet, selector) + revisions, err := p.historyController.ListControllerRevisions(sidecarcontrol.MockSidecarSetForRevision(set), hc.GetRevisionSelector(sidecarSet)) if err != nil { klog.Errorf("Failed to list history controllerRevisions, err %v, name %v", err, sidecarSet.Name) return nil, collisionCount, err @@ -369,6 +362,11 @@ func (p *Processor) registerLatestRevision(sidecarSet *appsv1alpha1.SidecarSet, revisions = append(revisions, latestRevision) } + // update custom revision for the latest controller revision + if err = p.updateCustomVersionLabel(latestRevision, sidecarSet.Labels[appsv1alpha1.SidecarSetCustomVersionLabel]); err != nil { + return nil, collisionCount, err + } + // only store limited history revisions if err = p.truncateHistory(revisions, sidecarSet, pods); err != nil { klog.Errorf("Failed to truncate history for %s: err: %v", sidecarSet.Name, err) @@ -377,6 +375,19 @@ func (p *Processor) registerLatestRevision(sidecarSet *appsv1alpha1.SidecarSet, return latestRevision, collisionCount, nil } +func (p *Processor) updateCustomVersionLabel(revision *apps.ControllerRevision, customVersion string) error { + if customVersion != "" && customVersion != revision.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] { + revisionClone := revision.DeepCopy() + patchBody := fmt.Sprintf(`{"metadata":{"labels":{"%v":"%v"}}}`, appsv1alpha1.SidecarSetCustomVersionLabel, customVersion) + err := p.Client.Patch(context.TODO(), revisionClone, client.RawPatch(types.StrategicMergePatchType, []byte(patchBody))) + if err != nil { + klog.Errorf(`Failed to patch custom revision label "%v" to latest revision %v, err: %v`, revision.Name, customVersion, err) + return err + } + } + return nil +} + func (p *Processor) truncateHistory(revisions []*apps.ControllerRevision, s *appsv1alpha1.SidecarSet, pods []*corev1.Pod) error { // We do not delete the latest revision because we are using it. // Thus, we must ensure the limitation is bounded, minimum value is 1. @@ -395,9 +406,9 @@ func (p *Processor) truncateHistory(revisions []*apps.ControllerRevision, s *app // the number of revisions need to delete deletionCount := revisionCount - limitation // only delete the revisions that no pods use. - activeRevisions := filterActiveRevisions(s, pods) + activeRevisions := filterActiveRevisions(s, pods, revisions) for i := 0; i < revisionCount-1 && deletionCount > 0; i++ { - if !activeRevisions.Has(revisions[i].Name) { // && revision.InjectionStrategy.ControllerRevision != revisions[i].Name + if !activeRevisions.Has(revisions[i].Name) { if err := p.historyController.DeleteControllerRevision(revisions[i]); err != nil && !errors.IsNotFound(err) { return err } @@ -412,13 +423,34 @@ func (p *Processor) truncateHistory(revisions []*apps.ControllerRevision, s *app return nil } -func filterActiveRevisions(s *appsv1alpha1.SidecarSet, pods []*corev1.Pod) sets.String { +func filterActiveRevisions(s *appsv1alpha1.SidecarSet, pods []*corev1.Pod, revisions []*apps.ControllerRevision) sets.String { activeRevisions := sets.NewString() for _, pod := range pods { if revision := sidecarcontrol.GetPodSidecarSetControllerRevision(s.Name, pod); revision != "" { activeRevisions.Insert(revision) } } + + if s.Spec.InjectionStrategy.Revision != nil { + equalRevisions := make([]*apps.ControllerRevision, 0) + if s.Spec.InjectionStrategy.Revision.RevisionName != nil { + activeRevisions.Insert(*s.Spec.InjectionStrategy.Revision.RevisionName) + } + + if s.Spec.InjectionStrategy.Revision.CustomVersion != nil { + for i := range revisions { + revision := revisions[i] + if revision.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] == *s.Spec.InjectionStrategy.Revision.CustomVersion { + equalRevisions = append(equalRevisions, revision) + } + } + if len(equalRevisions) > 0 { + history.SortControllerRevisions(equalRevisions) + activeRevisions.Insert(equalRevisions[len(equalRevisions)-1].Name) + } + } + } + return activeRevisions } diff --git a/pkg/webhook/pod/mutating/sidecarset.go b/pkg/webhook/pod/mutating/sidecarset.go index b4bb2e8f95..ed5f30aa1d 100644 --- a/pkg/webhook/pod/mutating/sidecarset.go +++ b/pkg/webhook/pod/mutating/sidecarset.go @@ -27,8 +27,10 @@ import ( "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" utilclient "github.com/openkruise/kruise/pkg/util/client" + "github.com/openkruise/kruise/pkg/util/history" admissionv1 "k8s.io/api/admission/v1" + apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -79,9 +81,14 @@ func (h *PodCreateHandler) sidecarsetMutatingPod(ctx context.Context, req admiss } else if !matched { continue } + // get user-specific revision or the latest revision of SidecarSet + suitableSidecarSet, err := h.getSuitableRevisionSidecarSet(&sidecarSet, oldPod, pod, req.AdmissionRequest.Operation) + if err != nil { + return false, err + } // check whether sidecarSet is active // when sidecarSet is not active, it will not perform injections and upgrades process. - control := sidecarcontrol.New(sidecarSet.DeepCopy()) + control := sidecarcontrol.New(suitableSidecarSet) if !control.IsActiveSidecarSet() { continue } @@ -139,6 +146,81 @@ func (h *PodCreateHandler) sidecarsetMutatingPod(ctx context.Context, req admiss return false, nil } +func (h *PodCreateHandler) getSuitableRevisionSidecarSet(sidecarSet *appsv1alpha1.SidecarSet, oldPod, newPod *corev1.Pod, operation admissionv1.Operation) (*appsv1alpha1.SidecarSet, error) { + switch operation { + case admissionv1.Update: + // optimization: quickly return if newPod matched the latest sidecarSet + if sidecarcontrol.GetPodSidecarSetRevision(sidecarSet.Name, newPod) == sidecarcontrol.GetSidecarSetRevision(sidecarSet) { + return sidecarSet.DeepCopy(), nil + } + + hc := sidecarcontrol.NewHistoryControl(h.Client) + revisions, err := history.NewHistory(h.Client).ListControllerRevisions(sidecarcontrol.MockSidecarSetForRevision(sidecarSet), hc.GetRevisionSelector(sidecarSet)) + if err != nil { + klog.Errorf("Failed to list history controllerRevisions, err %v, name %v", err, sidecarSet.Name) + return nil, err + } + + suitableSidecarSet, err := h.getSpecificRevisionSidecarSetForPod(sidecarSet, revisions, newPod) + if err != nil { + return nil, err + } else if suitableSidecarSet != nil { + return suitableSidecarSet, nil + } + + suitableSidecarSet, err = h.getSpecificRevisionSidecarSetForPod(sidecarSet, revisions, oldPod) + if err != nil { + return nil, err + } else if suitableSidecarSet != nil { + return suitableSidecarSet, nil + } + + return sidecarSet.DeepCopy(), nil + + default: + revisionInfo := sidecarSet.Spec.InjectionStrategy.Revision + if revisionInfo == nil || (revisionInfo.RevisionName == nil && revisionInfo.CustomVersion == nil) { + return sidecarSet.DeepCopy(), nil + } + + // TODO: support 'PartitionBased' policy to inject old/new revision according to Partition + switch sidecarSet.Spec.InjectionStrategy.Revision.Policy { + case "", appsv1alpha1.AlwaysSidecarSetInjectRevisionPolicy: + return h.getSpecificHistorySidecarSet(sidecarSet, revisionInfo) + } + + return h.getSpecificHistorySidecarSet(sidecarSet, revisionInfo) + } +} + +func (h *PodCreateHandler) getSpecificRevisionSidecarSetForPod(sidecarSet *appsv1alpha1.SidecarSet, revisions []*apps.ControllerRevision, pod *corev1.Pod) (*appsv1alpha1.SidecarSet, error) { + var err error + var matchedSidecarSet *appsv1alpha1.SidecarSet + for _, revision := range revisions { + revisionHash := revision.Annotations[sidecarcontrol.SidecarSetHashAnnotation] + if revisionHash != "" && sidecarcontrol.GetPodSidecarSetRevision(sidecarSet.Name, pod) == revisionHash { + matchedSidecarSet, err = h.getSpecificHistorySidecarSet(sidecarSet, &appsv1alpha1.SidecarSetInjectRevision{RevisionName: &revision.Name}) + if err != nil { + return nil, err + } + break + } + } + return matchedSidecarSet, nil +} + +func (h *PodCreateHandler) getSpecificHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*appsv1alpha1.SidecarSet, error) { + // else return its corresponding history revision + hc := sidecarcontrol.NewHistoryControl(h.Client) + historySidecarSet, err := hc.GetHistorySidecarSet(sidecarSet, revisionInfo) + if err != nil || historySidecarSet == nil { + klog.Warningf("Failed to restore history revision for SidecarSet %v, ControllerRevision name %v:, error: %v", + sidecarSet.Name, sidecarSet.Spec.InjectionStrategy.Revision, err) + return nil, err + } + return historySidecarSet, nil +} + func mergeSidecarSecrets(secretsInPod, secretsInSidecar []corev1.LocalObjectReference) (allSecrets []corev1.LocalObjectReference) { secretFilter := make(map[string]bool) for _, podSecret := range secretsInPod { @@ -241,9 +323,10 @@ func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedS volumesMap := getVolumesMapInSidecarSet(sidecarSet) // process sidecarset hash setUpgrade1 := sidecarcontrol.SidecarSetUpgradeSpec{ - UpdateTimestamp: metav1.Now(), - SidecarSetHash: sidecarcontrol.GetSidecarSetRevision(sidecarSet), - SidecarSetName: sidecarSet.Name, + UpdateTimestamp: metav1.Now(), + SidecarSetHash: sidecarcontrol.GetSidecarSetRevision(sidecarSet), + SidecarSetName: sidecarSet.Name, + SidecarSetControllerRevision: sidecarSet.Status.LatestRevision, } setUpgrade2 := sidecarcontrol.SidecarSetUpgradeSpec{ UpdateTimestamp: metav1.Now(), diff --git a/pkg/webhook/pod/mutating/sidecarset_test.go b/pkg/webhook/pod/mutating/sidecarset_test.go index 077740a32f..c3518b94d3 100644 --- a/pkg/webhook/pod/mutating/sidecarset_test.go +++ b/pkg/webhook/pod/mutating/sidecarset_test.go @@ -19,6 +19,7 @@ package mutating import ( "context" "encoding/json" + "fmt" "os" "path/filepath" "testing" @@ -27,12 +28,15 @@ import ( appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" + webhookutil "github.com/openkruise/kruise/pkg/webhook/util" admissionv1 "k8s.io/api/admission/v1" + apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -463,6 +467,74 @@ func testInjectionStrategyPaused(t *testing.T, sidecarIn *appsv1alpha1.SidecarSe } } +func TestInjectionStrategyRevision(t *testing.T) { + spec := map[string]interface{}{ + "spec": map[string]interface{}{ + "initContainers": []appsv1alpha1.SidecarContainer{ + { + Container: corev1.Container{ + Name: "init-2", + Image: "busybox:1.0.0", + }, + }, + }, + "containers": []appsv1alpha1.SidecarContainer{ + { + Container: corev1.Container{ + Name: "dns-f", + Image: "dns-f-image:1.0", + }, + PodInjectPolicy: appsv1alpha1.BeforeAppContainerType, + ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ + Type: appsv1alpha1.ShareVolumePolicyDisabled, + }, + }, + }, + }, + } + + raw, _ := json.Marshal(spec) + revisionID := fmt.Sprintf("%s-12345", sidecarSet1.Name) + sidecarSetIn := sidecarSet1.DeepCopy() + sidecarSetIn.Spec.InjectionStrategy.Revision = &appsv1alpha1.SidecarSetInjectRevision{ + CustomVersion: &revisionID, + Policy: appsv1alpha1.AlwaysSidecarSetInjectRevisionPolicy, + } + historyInjection := []client.Object{ + sidecarSetIn, + &apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: webhookutil.GetNamespace(), + Name: revisionID, + Labels: map[string]string{ + appsv1alpha1.SidecarSetCustomVersionLabel: revisionID, + }, + }, + Data: runtime.RawExtension{ + Raw: raw, + }, + }, + } + testInjectionStrategyRevision(t, historyInjection) +} + +func testInjectionStrategyRevision(t *testing.T, env []client.Object) { + podIn := pod1.DeepCopy() + podOut := podIn.DeepCopy() + decoder, _ := admission.NewDecoder(scheme.Scheme) + client := fake.NewClientBuilder().WithObjects(env...).Build() + podHandler := &PodCreateHandler{Decoder: decoder, Client: client} + req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") + _, err := podHandler.sidecarsetMutatingPod(context.Background(), req, podOut) + if err != nil { + t.Fatalf("failed to mutating pod, err: %v", err) + } + + if len(podIn.Spec.Containers)+len(podIn.Spec.InitContainers)+2 != len(podOut.Spec.Containers)+len(podOut.Spec.InitContainers) { + t.Fatalf("expect %v containers but got %v", len(podIn.Spec.Containers)+2, len(podOut.Spec.Containers)) + } +} + func TestSidecarSetPodInjectPolicy(t *testing.T) { sidecarSetIn := sidecarSet1.DeepCopy() testSidecarSetPodInjectPolicy(t, sidecarSetIn) diff --git a/pkg/webhook/sidecarset/mutating/sidecarset_create_update_handler.go b/pkg/webhook/sidecarset/mutating/sidecarset_create_update_handler.go index 04cf2106ce..8f2ef3ccb0 100644 --- a/pkg/webhook/sidecarset/mutating/sidecarset_create_update_handler.go +++ b/pkg/webhook/sidecarset/mutating/sidecarset_create_update_handler.go @@ -49,13 +49,13 @@ func setHashSidecarSet(sidecarset *appsv1alpha1.SidecarSet) error { sidecarset.Annotations = make(map[string]string) } - hash, err := SidecarSetHash(sidecarset) + hash, err := sidecarcontrol.SidecarSetHash(sidecarset) if err != nil { return err } sidecarset.Annotations[sidecarcontrol.SidecarSetHashAnnotation] = hash - hash, err = SidecarSetHashWithoutImage(sidecarset) + hash, err = sidecarcontrol.SidecarSetHashWithoutImage(sidecarset) if err != nil { return err } diff --git a/pkg/webhook/sidecarset/mutating/sidecarset_mutating_test.go b/pkg/webhook/sidecarset/mutating/sidecarset_mutating_test.go index 55980bf706..fb1b6e77ce 100644 --- a/pkg/webhook/sidecarset/mutating/sidecarset_mutating_test.go +++ b/pkg/webhook/sidecarset/mutating/sidecarset_mutating_test.go @@ -10,6 +10,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" ) func TestMutatingSidecarSetFn(t *testing.T) { @@ -27,6 +28,11 @@ func TestMutatingSidecarSetFn(t *testing.T) { }, }, }, + InjectionStrategy: appsv1alpha1.SidecarSetInjectionStrategy{ + Revision: &appsv1alpha1.SidecarSetInjectRevision{ + CustomVersion: pointer.String("1"), + }, + }, }, } defaults.SetDefaultsSidecarSet(sidecarSet) @@ -64,4 +70,7 @@ func TestMutatingSidecarSetFn(t *testing.T) { if sidecarSet.Annotations[sidecarcontrol.SidecarSetHashAnnotation] != "6wbd76bd7984x24fb4f44fv9222cw9v9bcf85x766744wddd4zwx927zzz2zb684" { t.Fatalf("sidecarset %v hash initialized incorrectly, got %v", sidecarSet.Name, sidecarSet.Annotations[sidecarcontrol.SidecarSetHashAnnotation]) } + if sidecarSet.Spec.InjectionStrategy.Revision.Policy != appsv1alpha1.AlwaysSidecarSetInjectRevisionPolicy { + t.Fatalf("sidecarset %v InjectionStrategy inilize incorrectly, got %v", sidecarSet.Name, sidecarSet.Spec.InjectionStrategy.Revision.Policy) + } } diff --git a/pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go b/pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go index ec3a9e0b4e..a6f2293dfa 100644 --- a/pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go +++ b/pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go @@ -86,7 +86,7 @@ func (h *SidecarSetCreateUpdateHandler) validateSidecarSet(obj *appsv1alpha1.Sid // validating ObjectMeta allErrs := genericvalidation.ValidateObjectMeta(&obj.ObjectMeta, false, validateSidecarSetName, field.NewPath("metadata")) // validating spec - allErrs = append(allErrs, validateSidecarSetSpec(obj, field.NewPath("spec"))...) + allErrs = append(allErrs, h.validateSidecarSetSpec(obj, field.NewPath("spec"))...) // when operation is update, older isn't empty, and validating whether old and new containers conflict if older != nil { allErrs = append(allErrs, validateSidecarContainerConflict(obj.Spec.Containers, older.Spec.Containers, field.NewPath("spec.containers"))...) @@ -110,7 +110,7 @@ func validateSidecarSetName(name string, prefix bool) (allErrs []string) { return allErrs } -func validateSidecarSetSpec(obj *appsv1alpha1.SidecarSet, fldPath *field.Path) field.ErrorList { +func (h *SidecarSetCreateUpdateHandler) validateSidecarSetSpec(obj *appsv1alpha1.SidecarSet, fldPath *field.Path) field.ErrorList { spec := &obj.Spec allErrs := field.ErrorList{} @@ -120,8 +120,10 @@ func validateSidecarSetSpec(obj *appsv1alpha1.SidecarSet, fldPath *field.Path) f } else { allErrs = append(allErrs, validateSelector(spec.Selector, fldPath.Child("selector"))...) } + //validating SidecarSetInjectionStrategy + allErrs = append(allErrs, h.validateSidecarSetInjectionStrategy(obj, fldPath.Child("injectionStrategy"))...) //validating SidecarSetUpdateStrategy - allErrs = append(allErrs, validateSidecarSetUpdateStrategy(&spec.UpdateStrategy, fldPath.Child("strategy"))...) + allErrs = append(allErrs, validateSidecarSetUpdateStrategy(&spec.UpdateStrategy, fldPath.Child("updateStrategy"))...) //validating volumes vols, vErrs := getCoreVolumes(spec.Volumes, fldPath.Child("volumes")) allErrs = append(allErrs, vErrs...) @@ -149,6 +151,31 @@ func validateSelector(selector *metav1.LabelSelector, fldPath *field.Path) field return allErrs } +func (h *SidecarSetCreateUpdateHandler) validateSidecarSetInjectionStrategy(obj *appsv1alpha1.SidecarSet, fldPath *field.Path) field.ErrorList { + errList := field.ErrorList{} + revisionInfo := obj.Spec.InjectionStrategy.Revision + + if revisionInfo != nil { + switch { + case revisionInfo.RevisionName == nil && revisionInfo.CustomVersion == nil: + errList = append(errList, field.Invalid(field.NewPath("revision"), revisionInfo, "revisionName and customVersion cannot be empty simultaneously")) + default: + revision, err := sidecarcontrol.NewHistoryControl(h.Client).GetHistorySidecarSet(obj, revisionInfo) + if err != nil || revision == nil { + errList = append(errList, field.Invalid(field.NewPath("revision"), revision, fmt.Sprintf("Cannot find specific ControllerRevision, err: %v", err))) + } + } + + switch revisionInfo.Policy { + case "", appsv1alpha1.AlwaysSidecarSetInjectRevisionPolicy: + default: + errList = append(errList, field.Invalid(field.NewPath("revision").Child("policy"), revisionInfo, fmt.Sprintf("Invalid policy %v, only support 'Always' currently", revisionInfo.Policy))) + } + + } + return errList +} + func validateSidecarSetUpdateStrategy(strategy *appsv1alpha1.SidecarSetUpdateStrategy, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} // if SidecarSet update strategy is RollingUpdate diff --git a/pkg/webhook/sidecarset/validating/sidecarset_validating_test.go b/pkg/webhook/sidecarset/validating/sidecarset_validating_test.go index 9b505e7cdb..e39cb4c72b 100644 --- a/pkg/webhook/sidecarset/validating/sidecarset_validating_test.go +++ b/pkg/webhook/sidecarset/validating/sidecarset_validating_test.go @@ -7,11 +7,26 @@ import ( appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" "github.com/openkruise/kruise/pkg/util" + apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +var ( + testScheme *runtime.Scheme + handler = &SidecarSetCreateUpdateHandler{} +) + +func init() { + testScheme = runtime.NewScheme() + apps.AddToScheme(testScheme) +} + func TestValidateSidecarSet(t *testing.T) { errorCases := map[string]appsv1alpha1.SidecarSet{ "missing-selector": { @@ -199,10 +214,92 @@ func TestValidateSidecarSet(t *testing.T) { }, }, }, + "wrong-name-injectionStrategy": { + ObjectMeta: metav1.ObjectMeta{Name: "test-sidecarset"}, + Spec: appsv1alpha1.SidecarSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"a": "b"}, + }, + InjectionStrategy: appsv1alpha1.SidecarSetInjectionStrategy{ + Revision: &appsv1alpha1.SidecarSetInjectRevision{ + CustomVersion: pointer.String("normal-sidecarset-01234"), + }, + }, + UpdateStrategy: appsv1alpha1.SidecarSetUpdateStrategy{ + Type: appsv1alpha1.NotUpdateSidecarSetStrategyType, + }, + Containers: []appsv1alpha1.SidecarContainer{ + { + PodInjectPolicy: appsv1alpha1.BeforeAppContainerType, + ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ + Type: appsv1alpha1.ShareVolumePolicyDisabled, + }, + UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{ + UpgradeType: appsv1alpha1.SidecarContainerColdUpgrade, + }, + Container: corev1.Container{ + Name: "test-sidecar", + Image: "test-image", + ImagePullPolicy: corev1.PullIfNotPresent, + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + }, + }, + }, + }, + }, + "not-existing-injectionStrategy": { + ObjectMeta: metav1.ObjectMeta{Name: "test-sidecarset"}, + Spec: appsv1alpha1.SidecarSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"a": "b"}, + }, + InjectionStrategy: appsv1alpha1.SidecarSetInjectionStrategy{ + Revision: &appsv1alpha1.SidecarSetInjectRevision{ + RevisionName: pointer.String("test-sidecarset-678235"), + Policy: appsv1alpha1.AlwaysSidecarSetInjectRevisionPolicy, + }, + }, + UpdateStrategy: appsv1alpha1.SidecarSetUpdateStrategy{ + Type: appsv1alpha1.NotUpdateSidecarSetStrategyType, + }, + Containers: []appsv1alpha1.SidecarContainer{ + { + PodInjectPolicy: appsv1alpha1.BeforeAppContainerType, + ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ + Type: appsv1alpha1.ShareVolumePolicyDisabled, + }, + UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{ + UpgradeType: appsv1alpha1.SidecarContainerColdUpgrade, + }, + Container: corev1.Container{ + Name: "test-sidecar", + Image: "test-image", + ImagePullPolicy: corev1.PullIfNotPresent, + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + }, + }, + }, + }, + }, + } + + SidecarSetRevisions := []client.Object{ + &apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sidecarset-01234", + }, + }, + &apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sidecarset-56789", + }, + }, } for name, sidecarSet := range errorCases { - allErrs := validateSidecarSetSpec(&sidecarSet, field.NewPath("spec")) + fakeClient := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(SidecarSetRevisions...).Build() + handler.Client = fakeClient + allErrs := handler.validateSidecarSetSpec(&sidecarSet, field.NewPath("spec")) if len(allErrs) != 1 { t.Errorf("%v: expect errors len 1, but got: %v", name, allErrs) } else { diff --git a/test/e2e/apps/sidecarset.go b/test/e2e/apps/sidecarset.go index 6a01796d00..38ceced906 100644 --- a/test/e2e/apps/sidecarset.go +++ b/test/e2e/apps/sidecarset.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "reflect" + "strconv" "time" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" @@ -1035,5 +1036,90 @@ var _ = SIGDescribe("SidecarSet", func() { revisionChecker(sidecarSetIn, 10, expectedOrder) ginkgo.By(fmt.Sprintf("sidecarSet history revision check done")) }) + + framework.ConformanceIt("sidecarSet InjectionStrategy.Revision checker", func() { + // create sidecarSet + nginxName := func(tag string) string { + return fmt.Sprintf("nginx:%s", tag) + } + tags := []string{ + "latest", "1.21.1", "1.21", "1.20.1", "1.20", "1.19.10", + } + sidecarSetIn := tester.NewBaseSidecarSet(ns) + if sidecarSetIn.Labels == nil { + sidecarSetIn.Labels = map[string]string{ + appsv1alpha1.SidecarSetCustomVersionLabel: "0", + } + } + sidecarSetIn.SetName("e2e-test-for-injection-strategy-revision") + sidecarSetIn.Spec.UpdateStrategy.Paused = true + sidecarSetIn.Spec.Containers[0].Image = nginxName(tags[0]) + ginkgo.By(fmt.Sprintf("Creating SidecarSet %s", sidecarSetIn.Name)) + sidecarSetIn = tester.CreateSidecarSet(sidecarSetIn) + time.Sleep(time.Second) + for i := 1; i < 6; i++ { + // update sidecarSet and stored revisions + sidecarSetIn.Spec.Containers[0].Image = nginxName(tags[i]) + sidecarSetIn.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] = strconv.Itoa(i) + tester.UpdateSidecarSet(sidecarSetIn) + gomega.Eventually(func() int { + rv := tester.ListControllerRevisions(sidecarSetIn) + return len(rv) + }, 5*time.Second, time.Second).Should(gomega.Equal(i + 1)) + } + + // pick a history revision to inject + pick := 3 + list := tester.ListControllerRevisions(sidecarSetIn) + gomega.Expect(list).To(gomega.HaveLen(6)) + history.SortControllerRevisions(list) + sidecarSetIn.Spec.InjectionStrategy.Revision = &appsv1alpha1.SidecarSetInjectRevision{ + CustomVersion: utilpointer.String(strconv.Itoa(pick)), + Policy: appsv1alpha1.AlwaysSidecarSetInjectRevisionPolicy, + } + tester.UpdateSidecarSet(sidecarSetIn) + time.Sleep(time.Second) + + // create deployment + deploymentIn := tester.NewBaseDeployment(ns) + deploymentIn.Spec.Replicas = utilpointer.Int32Ptr(1) + ginkgo.By(fmt.Sprintf("Creating Deployment(%s.%s)", deploymentIn.Namespace, deploymentIn.Name)) + tester.CreateDeployment(deploymentIn) + + // check sidecarSet revision + pods, err := tester.GetSelectorPods(ns, deploymentIn.Spec.Selector) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(pods).To(gomega.HaveLen(1)) + gomega.Expect(pods[0].Spec.Containers[0].Image).To(gomega.Equal(nginxName(tags[pick]))) + + // check pod sidecarSetHash + gomega.Expect(len(pods[0].Annotations[sidecarcontrol.SidecarSetHashAnnotation]) > 0).To(gomega.BeTrue()) + hash := make(map[string]sidecarcontrol.SidecarSetUpgradeSpec) + err = json.Unmarshal([]byte(pods[0].Annotations[sidecarcontrol.SidecarSetHashAnnotation]), &hash) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(hash[sidecarSetIn.Name].SidecarSetControllerRevision).To(gomega.Equal(list[pick].Name)) + + // check again after sidecarSet upgrade + sidecarSetIn.Spec.UpdateStrategy.Paused = false + tester.UpdateSidecarSet(sidecarSetIn) + except := &appsv1alpha1.SidecarSetStatus{ + MatchedPods: 1, + UpdatedPods: 1, + UpdatedReadyPods: 1, + ReadyPods: 1, + } + tester.WaitForSidecarSetUpgradeComplete(sidecarSetIn, except) + + pods, err = tester.GetSelectorPods(ns, deploymentIn.Spec.Selector) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(pods).To(gomega.HaveLen(1)) + gomega.Expect(pods[0].Spec.Containers[0].Image).To(gomega.Equal(nginxName(tags[5]))) + gomega.Expect(len(pods[0].Annotations[sidecarcontrol.SidecarSetHashAnnotation]) > 0).To(gomega.BeTrue()) + hash = make(map[string]sidecarcontrol.SidecarSetUpgradeSpec) + err = json.Unmarshal([]byte(pods[0].Annotations[sidecarcontrol.SidecarSetHashAnnotation]), &hash) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(hash[sidecarSetIn.Name].SidecarSetControllerRevision).To(gomega.Equal(list[5].Name)) + ginkgo.By(fmt.Sprintf("sidecarSet InjectionStrategy.Revision check done")) + }) }) })