From 0fff9e349a12c5577b302f9cd9bd69289b6c0378 Mon Sep 17 00:00:00 2001 From: "mingzhou.swx" Date: Thu, 14 Jul 2022 11:48:40 +0800 Subject: [PATCH] sidecarset inject history revision Signed-off-by: mingzhou.swx --- apis/apps/v1alpha1/sidecarset_types.go | 26 +++++ apis/apps/v1alpha1/zz_generated.deepcopy.go | 22 ++++- .../crd/bases/apps.kruise.io_sidecarsets.yaml | 19 ++++ .../sidecarcontrol}/hash.go | 16 ++- pkg/control/sidecarcontrol/history_control.go | 66 ++++++++++++- pkg/control/sidecarcontrol/util.go | 2 + .../sidecarset_pod_event_handler.go | 8 ++ .../sidecarset/sidecarset_processor.go | 33 ++++++- pkg/webhook/pod/mutating/sidecarset.go | 47 ++++++++- pkg/webhook/pod/mutating/sidecarset_test.go | 68 +++++++++++++ .../sidecarset_create_update_handler.go | 4 +- .../sidecarset_create_update_handler.go | 29 +++++- .../validating/sidecarset_validating_test.go | 97 ++++++++++++++++++- test/e2e/apps/sidecarset.go | 81 ++++++++++++++++ 14 files changed, 505 insertions(+), 13 deletions(-) rename pkg/{webhook/sidecarset/mutating => control/sidecarcontrol}/hash.go (81%) diff --git a/apis/apps/v1alpha1/sidecarset_types.go b/apis/apps/v1alpha1/sidecarset_types.go index b150e6dfe4..1ced90511f 100644 --- a/apis/apps/v1alpha1/sidecarset_types.go +++ b/apis/apps/v1alpha1/sidecarset_types.go @@ -142,8 +142,34 @@ type SidecarSetInjectionStrategy struct { // but the injected sidecar container remains updating and running. // default is false Paused bool `json:"paused,omitempty"` + + // Revision can help users rolling update SidecarSet safely. If users set + // this filed, SidecarSet will try to decide inject update revision according to + // different policies. + Revision *SidecarSetInjectRevision `json:"revision,omitempty"` +} + +type SidecarSetInjectRevision struct { + // ID corresponds to label 'apps.kruise.io/sidecarset-revision-id' of (History) SidecarSet. + // SidecarSet will select the specific ControllerRevision via this ID, and then restore the + // history SidecarSet to inject specific version of the sidecar to pods. + ID string `json:"id"` + // Policy describes the behavior of revision injection. + // Defaults to Always. + Policy SidecarSetInjectRevisionPolicy `json:"policy,omitempty"` } +type SidecarSetInjectRevisionPolicy string + +const ( + // AlwaysSidecarSetInjectRevisionPolicy means the SidecarSet will always inject + // the specific revision to Pods when pod creating, except matching UpdateStrategy.Selector. + AlwaysSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "Always" + // AdaptiveSidecarSetInjectRevisionPolicy means the SidecarSet will inject the + // specific or the latest revision according to Partition. + //AdaptiveSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "Adaptive" +) + // SidecarSetUpdateStrategy indicates the strategy that the SidecarSet // controller will use to perform updates. It includes any additional parameters // necessary to perform the update for the indicated strategy. diff --git a/apis/apps/v1alpha1/zz_generated.deepcopy.go b/apis/apps/v1alpha1/zz_generated.deepcopy.go index 72cd8b292d..59bd8a0665 100644 --- a/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -2183,9 +2183,29 @@ func (in *SidecarSet) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SidecarSetInjectRevision) DeepCopyInto(out *SidecarSetInjectRevision) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SidecarSetInjectRevision. +func (in *SidecarSetInjectRevision) DeepCopy() *SidecarSetInjectRevision { + if in == nil { + return nil + } + out := new(SidecarSetInjectRevision) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SidecarSetInjectionStrategy) DeepCopyInto(out *SidecarSetInjectionStrategy) { *out = *in + if in.Revision != nil { + in, out := &in.Revision, &out.Revision + *out = new(SidecarSetInjectRevision) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SidecarSetInjectionStrategy. @@ -2260,7 +2280,7 @@ func (in *SidecarSetSpec) DeepCopyInto(out *SidecarSetSpec) { } } in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy) - out.InjectionStrategy = in.InjectionStrategy + in.InjectionStrategy.DeepCopyInto(&out.InjectionStrategy) if in.ImagePullSecrets != nil { in, out := &in.ImagePullSecrets, &out.ImagePullSecrets *out = make([]v1.LocalObjectReference, len(*in)) diff --git a/config/crd/bases/apps.kruise.io_sidecarsets.yaml b/config/crd/bases/apps.kruise.io_sidecarsets.yaml index db5bc300ed..2b923b7883 100644 --- a/config/crd/bases/apps.kruise.io_sidecarsets.yaml +++ b/config/crd/bases/apps.kruise.io_sidecarsets.yaml @@ -229,6 +229,25 @@ spec: to newly created Pods, but the injected sidecar container remains updating and running. default is false type: boolean + revision: + description: Revision can help users rolling update SidecarSet + safely. If users set this filed, SidecarSet will try to decide + inject update revision according to different policies. + properties: + id: + description: ID corresponds to label 'apps.kruise.io/sidecarset-revision-id' + of (History) SidecarSet. SidecarSet will select the specific + ControllerRevision via this ID, and then restore the history + SidecarSet to inject specific version of the sidecar to + pods. + type: string + policy: + description: Policy describes the behavior of revision injection. + Defaults to Always. + type: string + required: + - id + type: object type: object namespace: description: Namespace sidecarSet will only match the pods in the diff --git a/pkg/webhook/sidecarset/mutating/hash.go b/pkg/control/sidecarcontrol/hash.go similarity index 81% rename from pkg/webhook/sidecarset/mutating/hash.go rename to pkg/control/sidecarcontrol/hash.go index 35f58c37c8..9479479be2 100644 --- a/pkg/webhook/sidecarset/mutating/hash.go +++ b/pkg/control/sidecarcontrol/hash.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mutating +package sidecarcontrol import ( "crypto/sha256" @@ -66,3 +66,17 @@ func encodeSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (string, error) { func hash(data string) string { return fmt.Sprintf("%x", sha256.Sum256([]byte(data))) } + +func RecalculateSidecarSetHash(sidecarSet *appsv1alpha1.SidecarSet) error { + hashCodeWithImage, err := SidecarSetHash(sidecarSet) + if err != nil { + return err + } + hashCodeWithoutImage, err := SidecarSetHashWithoutImage(sidecarSet) + if err != nil { + return err + } + sidecarSet.Annotations[SidecarSetHashAnnotation] = hashCodeWithImage + sidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation] = hashCodeWithoutImage + return nil +} diff --git a/pkg/control/sidecarcontrol/history_control.go b/pkg/control/sidecarcontrol/history_control.go index 2323e13756..31cf282431 100644 --- a/pkg/control/sidecarcontrol/history_control.go +++ b/pkg/control/sidecarcontrol/history_control.go @@ -23,13 +23,16 @@ import ( "fmt" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - + webhookutil "github.com/openkruise/kruise/pkg/webhook/util" apps "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/controller/history" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -43,6 +46,7 @@ type HistoryControl interface { NewRevision(s *appsv1alpha1.SidecarSet, namespace string, revision int64, collisionCount *int32) (*apps.ControllerRevision, error) NextRevision(revisions []*apps.ControllerRevision) int64 GetRevisionLabelSelector(s *appsv1alpha1.SidecarSet) *metav1.LabelSelector + GetHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionName string) (*appsv1alpha1.SidecarSet, error) } type realControl struct { @@ -80,6 +84,9 @@ func (r *realControl) NewRevision(s *appsv1alpha1.SidecarSet, namespace string, if cr.ObjectMeta.Annotations == nil { cr.ObjectMeta.Annotations = make(map[string]string) } + if s.Labels[SidecarSetRevisionIDLabel] != "" { + cr.Labels[SidecarSetRevisionIDLabel] = s.Labels[SidecarSetRevisionIDLabel] + } cr.Labels[SidecarSetKindName] = s.Name for key, value := range s.Annotations { cr.ObjectMeta.Annotations[key] = value @@ -162,6 +169,56 @@ func (r *realControl) CreateControllerRevision(parent metav1.Object, revision *a } } +func (r *realControl) GetHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionID string) (*appsv1alpha1.SidecarSet, error) { + if revisionID == "" { + return nil, generateNotFoundError(sidecarSet) + } + + listOpts := []client.ListOption{ + client.InNamespace(webhookutil.GetNamespace()), + client.MatchingLabels{SidecarSetRevisionIDLabel: revisionID}, + } + revisionList := &apps.ControllerRevisionList{} + if err := r.Client.List(context.TODO(), revisionList, listOpts...); err != nil { + klog.Errorf("Failed to get ControllerRevision ID: %s, err %v", revisionID, err) + return nil, err + } + + var revisions []*apps.ControllerRevision + for i := range revisionList.Items { + revisions = append(revisions, &revisionList.Items[i]) + } + + if len(revisions) == 0 { + return nil, generateNotFoundError(sidecarSet) + } + history.SortControllerRevisions(revisions) + revision := revisions[len(revisions)-1] + + // calculate patch + clone := sidecarSet.DeepCopy() + cloneBytes, err := runtime.Encode(patchCodec, clone) + if err != nil { + klog.Errorf("Failed to encode sidecarSet(%v), error: %v", sidecarSet.Name, err) + return nil, err + } + patched, err := strategicpatch.StrategicMergePatch(cloneBytes, revision.Data.Raw, clone) + if err != nil { + klog.Errorf("Failed to merge sidecarSet(%v) and controllerRevision, ID: %v, error: %v", sidecarSet.Name, revisionID, err) + return nil, err + } + // restore history from patch + restoredSidecarSet := &appsv1alpha1.SidecarSet{} + if err := json.Unmarshal(patched, restoredSidecarSet); err != nil { + return nil, err + } + // re-calculate sidecarSet hash + if err := RecalculateSidecarSetHash(restoredSidecarSet); err != nil { + return nil, err + } + return restoredSidecarSet, nil +} + func copySidecarSetSpecRevision(dst, src map[string]interface{}) { // we will use patch instead of update operation to update pods in the future // dst["$patch"] = "replace" @@ -171,3 +228,10 @@ func copySidecarSetSpecRevision(dst, src map[string]interface{}) { dst["initContainers"] = src["initContainers"] dst["imagePullSecrets"] = src["imagePullSecrets"] } + +func generateNotFoundError(set *appsv1alpha1.SidecarSet) error { + return errors.NewNotFound(schema.GroupResource{ + Group: apps.GroupName, + Resource: "ControllerRevision", + }, set.Name) +} diff --git a/pkg/control/sidecarcontrol/util.go b/pkg/control/sidecarcontrol/util.go index 4250432b05..c4fe4e5227 100644 --- a/pkg/control/sidecarcontrol/util.go +++ b/pkg/control/sidecarcontrol/util.go @@ -54,6 +54,8 @@ const ( // SidecarsetInplaceUpdateStateKey records the state of inplace-update. // The value of annotation is SidecarsetInplaceUpdateStateKey. SidecarsetInplaceUpdateStateKey string = "kruise.io/sidecarset-inplace-update-state" + + SidecarSetRevisionIDLabel = "apps.kruise.io/sidecarset-revision-id" ) var ( diff --git a/pkg/controller/sidecarset/sidecarset_pod_event_handler.go b/pkg/controller/sidecarset/sidecarset_pod_event_handler.go index 9c218eeeaa..5a9c36fea7 100644 --- a/pkg/controller/sidecarset/sidecarset_pod_event_handler.go +++ b/pkg/controller/sidecarset/sidecarset_pod_event_handler.go @@ -2,6 +2,7 @@ package sidecarset import ( "context" + "reflect" "strings" "time" @@ -179,5 +180,12 @@ func isPodConsistentChanged(oldPod, newPod *corev1.Pod, sidecarSet *appsv1alpha1 return true, enqueueDelayTime } + // If the pod's labels changed, and sidecarSet enable selector updateStrategy, should reconcile. + if !reflect.DeepEqual(oldPod.Labels, newPod.Labels) && sidecarSet.Spec.UpdateStrategy.Selector != nil { + klog.V(3).Infof("pod(%s/%s) Labels changed and sidecarSet (%s) enable selector upgrade strategy, "+ + "and reconcile sidecarSet", newPod.Namespace, newPod.Name, sidecarSet.Name) + return true, 0 + } + return false, enqueueDelayTime } diff --git a/pkg/controller/sidecarset/sidecarset_processor.go b/pkg/controller/sidecarset/sidecarset_processor.go index 0080be9e4f..ac4df65cc0 100644 --- a/pkg/controller/sidecarset/sidecarset_processor.go +++ b/pkg/controller/sidecarset/sidecarset_processor.go @@ -369,6 +369,18 @@ func (p *Processor) registerLatestRevision(sidecarSet *appsv1alpha1.SidecarSet, revisions = append(revisions, latestRevision) } + // update revision id for the latest controller revision + currentRevisionID := sidecarSet.Labels[sidecarcontrol.SidecarSetRevisionIDLabel] + if currentRevisionID != "" && currentRevisionID != latestRevision.Labels[sidecarcontrol.SidecarSetRevisionIDLabel] { + latestRevisionCLone := latestRevision.DeepCopy() + patchBody := fmt.Sprintf(`{"metadata":{"labels":{"%v":"%v"}}}`, sidecarcontrol.SidecarSetRevisionIDLabel, currentRevisionID) + err = p.Client.Patch(context.TODO(), latestRevisionCLone, client.RawPatch(types.StrategicMergePatchType, []byte(patchBody))) + if err != nil { + klog.Errorf("Failed to patch revision id to latest revision %v, err: %v", latestRevision.Name, err) + return nil, collisionCount, err + } + } + // only store limited history revisions if err = p.truncateHistory(revisions, sidecarSet, pods); err != nil { klog.Errorf("Failed to truncate history for %s: err: %v", sidecarSet.Name, err) @@ -395,9 +407,9 @@ func (p *Processor) truncateHistory(revisions []*apps.ControllerRevision, s *app // the number of revisions need to delete deletionCount := revisionCount - limitation // only delete the revisions that no pods use. - activeRevisions := filterActiveRevisions(s, pods) + activeRevisions := filterActiveRevisions(s, pods, revisions) for i := 0; i < revisionCount-1 && deletionCount > 0; i++ { - if !activeRevisions.Has(revisions[i].Name) { // && revision.InjectionStrategy.ControllerRevision != revisions[i].Name + if !activeRevisions.Has(revisions[i].Name) { if err := p.historyController.DeleteControllerRevision(revisions[i]); err != nil && !errors.IsNotFound(err) { return err } @@ -412,13 +424,28 @@ func (p *Processor) truncateHistory(revisions []*apps.ControllerRevision, s *app return nil } -func filterActiveRevisions(s *appsv1alpha1.SidecarSet, pods []*corev1.Pod) sets.String { +func filterActiveRevisions(s *appsv1alpha1.SidecarSet, pods []*corev1.Pod, revisions []*apps.ControllerRevision) sets.String { activeRevisions := sets.NewString() for _, pod := range pods { if revision := sidecarcontrol.GetPodSidecarSetControllerRevision(s.Name, pod); revision != "" { activeRevisions.Insert(revision) } } + + if s.Spec.InjectionStrategy.Revision != nil { + equalRevisions := make([]*apps.ControllerRevision, 0) + for i := range revisions { + revision := revisions[i] + if revision.Labels[sidecarcontrol.SidecarSetRevisionIDLabel] == s.Spec.InjectionStrategy.Revision.ID { + equalRevisions = append(equalRevisions, revision) + } + } + if len(equalRevisions) > 0 { + history.SortControllerRevisions(equalRevisions) + activeRevisions.Insert(equalRevisions[len(equalRevisions)-1].Name) + } + } + return activeRevisions } diff --git a/pkg/webhook/pod/mutating/sidecarset.go b/pkg/webhook/pod/mutating/sidecarset.go index 78e092dd75..5484924de1 100644 --- a/pkg/webhook/pod/mutating/sidecarset.go +++ b/pkg/webhook/pod/mutating/sidecarset.go @@ -31,8 +31,10 @@ import ( admissionv1 "k8s.io/api/admission/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) @@ -79,9 +81,14 @@ func (h *PodCreateHandler) sidecarsetMutatingPod(ctx context.Context, req admiss } else if !matched { continue } + // get user-specific revision or the latest revision of SidecarSet + suitableSidecarSet, err := h.getSuitableRevisionSidecarSet(&sidecarSet, pod) + if err != nil { + return err + } // check whether sidecarSet is active // when sidecarSet is not active, it will not perform injections and upgrades process. - control := sidecarcontrol.New(sidecarSet.DeepCopy()) + control := sidecarcontrol.New(suitableSidecarSet) if !control.IsActiveSidecarSet() { continue } @@ -139,6 +146,44 @@ func (h *PodCreateHandler) sidecarsetMutatingPod(ctx context.Context, req admiss return nil } +func (h *PodCreateHandler) getSuitableRevisionSidecarSet(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) (*appsv1alpha1.SidecarSet, error) { + if sidecarSet.Spec.InjectionStrategy.Revision == nil || len(sidecarSet.Spec.InjectionStrategy.Revision.ID) == 0 { + return sidecarSet.DeepCopy(), nil + } + + if sidecarSet.Spec.UpdateStrategy.Selector != nil { + selector, err := metav1.LabelSelectorAsSelector(sidecarSet.Spec.UpdateStrategy.Selector) + if err != nil { + return nil, err + } + + // inject new revision of sidecarSet if it matches selector labels + if selector.Matches(labels.Set(pod.Labels)) { + klog.Infof("Pod %v matches sidecarSet %s upgrade selector", client.ObjectKeyFromObject(pod), sidecarSet.Name) + return sidecarSet.DeepCopy(), nil + } + } + + // TODO: support 'Adaptive' policy to inject old/new revision according to Partition + switch sidecarSet.Spec.InjectionStrategy.Revision.Policy { + case "", appsv1alpha1.AlwaysSidecarSetInjectRevisionPolicy: + return h.getSpecificHistorySidecarSet(sidecarSet) + } + return h.getSpecificHistorySidecarSet(sidecarSet) +} + +func (h *PodCreateHandler) getSpecificHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (*appsv1alpha1.SidecarSet, error) { + // else return its corresponding history revision + hc := sidecarcontrol.NewHistoryControl(h.Client) + history, err := hc.GetHistorySidecarSet(sidecarSet, sidecarSet.Spec.InjectionStrategy.Revision.ID) + if err != nil || history == nil { + klog.Warningf("Failed to restore history revision for SidecarSet %v, ControllerRevision name %v:, error: %v", + sidecarSet.Name, sidecarSet.Spec.InjectionStrategy.Revision, err) + return nil, err + } + return history, nil +} + func mergeSidecarSecrets(secretsInPod, secretsInSidecar []corev1.LocalObjectReference) (allSecrets []corev1.LocalObjectReference) { secretFilter := make(map[string]bool) for _, podSecret := range secretsInPod { diff --git a/pkg/webhook/pod/mutating/sidecarset_test.go b/pkg/webhook/pod/mutating/sidecarset_test.go index 6b1110ebd4..f8b89a38bc 100644 --- a/pkg/webhook/pod/mutating/sidecarset_test.go +++ b/pkg/webhook/pod/mutating/sidecarset_test.go @@ -19,6 +19,7 @@ package mutating import ( "context" "encoding/json" + "fmt" "os" "path/filepath" "testing" @@ -27,8 +28,10 @@ import ( appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" + webhookutil "github.com/openkruise/kruise/pkg/webhook/util" admissionv1 "k8s.io/api/admission/v1" + apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -463,6 +466,71 @@ func testInjectionStrategyPaused(t *testing.T, sidecarIn *appsv1alpha1.SidecarSe } } +func TestInjectionStrategyRevision(t *testing.T) { + spec := map[string]interface{}{ + "spec": map[string]interface{}{ + "initContainers": []appsv1alpha1.SidecarContainer{ + { + Container: corev1.Container{ + Name: "init-2", + Image: "busybox:1.0.0", + }, + }, + }, + "containers": []appsv1alpha1.SidecarContainer{ + { + Container: corev1.Container{ + Name: "dns-f", + Image: "dns-f-image:1.0", + }, + PodInjectPolicy: appsv1alpha1.BeforeAppContainerType, + ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ + Type: appsv1alpha1.ShareVolumePolicyDisabled, + }, + }, + }, + }, + } + + raw, _ := json.Marshal(spec) + revisionID := fmt.Sprintf("%s-12345", sidecarSet1.Name) + sidecarSetIn := sidecarSet1.DeepCopy() + sidecarSetIn.Spec.InjectionStrategy.Revision = &appsv1alpha1.SidecarSetInjectRevision{ + ID: revisionID, + Policy: appsv1alpha1.AlwaysSidecarSetInjectRevisionPolicy, + } + historyInjection := []runtime.Object{ + sidecarSetIn, + &apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: webhookutil.GetNamespace(), + Name: revisionID, + Labels: map[string]string{ + sidecarcontrol.SidecarSetRevisionIDLabel: revisionID, + }, + }, + Data: runtime.RawExtension{ + Raw: raw, + }, + }, + } + testInjectionStrategyRevision(t, historyInjection) +} + +func testInjectionStrategyRevision(t *testing.T, env []runtime.Object) { + podIn := pod1.DeepCopy() + podOut := podIn.DeepCopy() + decoder, _ := admission.NewDecoder(scheme.Scheme) + client := fake.NewFakeClient(env...) + podHandler := &PodCreateHandler{Decoder: decoder, Client: client} + req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") + _ = podHandler.sidecarsetMutatingPod(context.Background(), req, podOut) + + if len(podIn.Spec.Containers)+len(podIn.Spec.InitContainers)+2 != len(podOut.Spec.Containers)+len(podOut.Spec.InitContainers) { + t.Fatalf("expect %v containers but got %v", len(podIn.Spec.Containers)+2, len(podOut.Spec.Containers)) + } +} + func TestSidecarSetPodInjectPolicy(t *testing.T) { sidecarSetIn := sidecarSet1.DeepCopy() testSidecarSetPodInjectPolicy(t, sidecarSetIn) diff --git a/pkg/webhook/sidecarset/mutating/sidecarset_create_update_handler.go b/pkg/webhook/sidecarset/mutating/sidecarset_create_update_handler.go index 04cf2106ce..8f2ef3ccb0 100644 --- a/pkg/webhook/sidecarset/mutating/sidecarset_create_update_handler.go +++ b/pkg/webhook/sidecarset/mutating/sidecarset_create_update_handler.go @@ -49,13 +49,13 @@ func setHashSidecarSet(sidecarset *appsv1alpha1.SidecarSet) error { sidecarset.Annotations = make(map[string]string) } - hash, err := SidecarSetHash(sidecarset) + hash, err := sidecarcontrol.SidecarSetHash(sidecarset) if err != nil { return err } sidecarset.Annotations[sidecarcontrol.SidecarSetHashAnnotation] = hash - hash, err = SidecarSetHashWithoutImage(sidecarset) + hash, err = sidecarcontrol.SidecarSetHashWithoutImage(sidecarset) if err != nil { return err } diff --git a/pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go b/pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go index ec3a9e0b4e..b96257348a 100644 --- a/pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go +++ b/pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go @@ -30,6 +30,7 @@ import ( webhookutil "github.com/openkruise/kruise/pkg/webhook/util" admissionv1 "k8s.io/api/admission/v1" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" genericvalidation "k8s.io/apimachinery/pkg/api/validation" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -86,7 +87,7 @@ func (h *SidecarSetCreateUpdateHandler) validateSidecarSet(obj *appsv1alpha1.Sid // validating ObjectMeta allErrs := genericvalidation.ValidateObjectMeta(&obj.ObjectMeta, false, validateSidecarSetName, field.NewPath("metadata")) // validating spec - allErrs = append(allErrs, validateSidecarSetSpec(obj, field.NewPath("spec"))...) + allErrs = append(allErrs, h.validateSidecarSetSpec(obj, field.NewPath("spec"))...) // when operation is update, older isn't empty, and validating whether old and new containers conflict if older != nil { allErrs = append(allErrs, validateSidecarContainerConflict(obj.Spec.Containers, older.Spec.Containers, field.NewPath("spec.containers"))...) @@ -110,7 +111,7 @@ func validateSidecarSetName(name string, prefix bool) (allErrs []string) { return allErrs } -func validateSidecarSetSpec(obj *appsv1alpha1.SidecarSet, fldPath *field.Path) field.ErrorList { +func (h *SidecarSetCreateUpdateHandler) validateSidecarSetSpec(obj *appsv1alpha1.SidecarSet, fldPath *field.Path) field.ErrorList { spec := &obj.Spec allErrs := field.ErrorList{} @@ -120,8 +121,10 @@ func validateSidecarSetSpec(obj *appsv1alpha1.SidecarSet, fldPath *field.Path) f } else { allErrs = append(allErrs, validateSelector(spec.Selector, fldPath.Child("selector"))...) } + //validating SidecarSetInjectionStrategy + allErrs = append(allErrs, h.validateSidecarSetInjectionStrategy(obj, fldPath.Child("InjectionStrategy"))...) //validating SidecarSetUpdateStrategy - allErrs = append(allErrs, validateSidecarSetUpdateStrategy(&spec.UpdateStrategy, fldPath.Child("strategy"))...) + allErrs = append(allErrs, validateSidecarSetUpdateStrategy(&spec.UpdateStrategy, fldPath.Child("updateStrategy"))...) //validating volumes vols, vErrs := getCoreVolumes(spec.Volumes, fldPath.Child("volumes")) allErrs = append(allErrs, vErrs...) @@ -149,6 +152,26 @@ func validateSelector(selector *metav1.LabelSelector, fldPath *field.Path) field return allErrs } +func (h *SidecarSetCreateUpdateHandler) validateSidecarSetInjectionStrategy(obj *appsv1alpha1.SidecarSet, fldPath *field.Path) field.ErrorList { + errList := field.ErrorList{} + revision := obj.Spec.InjectionStrategy.Revision + + if revision != nil && len(revision.ID) > 0 { + listOpts := []client.ListOption{ + client.InNamespace(webhookutil.GetNamespace()), + client.MatchingLabels{sidecarcontrol.SidecarSetRevisionIDLabel: revision.ID}, + } + revisionList := &appsv1.ControllerRevisionList{} + if err := h.Client.List(context.TODO(), revisionList, listOpts...); err != nil { + errList = append(errList, field.InternalError(field.NewPath("stableRevision"), err)) + } + if len(revisionList.Items) == 0 { + errList = append(errList, field.Invalid(field.NewPath("stableRevision"), revision, fmt.Sprintf("Cannot find specific stable revision id %v", revision))) + } + } + return errList +} + func validateSidecarSetUpdateStrategy(strategy *appsv1alpha1.SidecarSetUpdateStrategy, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} // if SidecarSet update strategy is RollingUpdate diff --git a/pkg/webhook/sidecarset/validating/sidecarset_validating_test.go b/pkg/webhook/sidecarset/validating/sidecarset_validating_test.go index 9b505e7cdb..2587f469ec 100644 --- a/pkg/webhook/sidecarset/validating/sidecarset_validating_test.go +++ b/pkg/webhook/sidecarset/validating/sidecarset_validating_test.go @@ -7,11 +7,24 @@ import ( appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" "github.com/openkruise/kruise/pkg/util" + apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +var ( + testScheme *runtime.Scheme + handler = &SidecarSetCreateUpdateHandler{} +) + +func init() { + testScheme = runtime.NewScheme() + apps.AddToScheme(testScheme) +} + func TestValidateSidecarSet(t *testing.T) { errorCases := map[string]appsv1alpha1.SidecarSet{ "missing-selector": { @@ -199,10 +212,92 @@ func TestValidateSidecarSet(t *testing.T) { }, }, }, + "wrong-name-injectionStrategy": { + ObjectMeta: metav1.ObjectMeta{Name: "test-sidecarset"}, + Spec: appsv1alpha1.SidecarSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"a": "b"}, + }, + InjectionStrategy: appsv1alpha1.SidecarSetInjectionStrategy{ + Revision: &appsv1alpha1.SidecarSetInjectRevision{ + ID: "normal-sidecarset-01234", + }, + }, + UpdateStrategy: appsv1alpha1.SidecarSetUpdateStrategy{ + Type: appsv1alpha1.NotUpdateSidecarSetStrategyType, + }, + Containers: []appsv1alpha1.SidecarContainer{ + { + PodInjectPolicy: appsv1alpha1.BeforeAppContainerType, + ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ + Type: appsv1alpha1.ShareVolumePolicyDisabled, + }, + UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{ + UpgradeType: appsv1alpha1.SidecarContainerColdUpgrade, + }, + Container: corev1.Container{ + Name: "test-sidecar", + Image: "test-image", + ImagePullPolicy: corev1.PullIfNotPresent, + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + }, + }, + }, + }, + }, + "not-existing-injectionStrategy": { + ObjectMeta: metav1.ObjectMeta{Name: "test-sidecarset"}, + Spec: appsv1alpha1.SidecarSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"a": "b"}, + }, + InjectionStrategy: appsv1alpha1.SidecarSetInjectionStrategy{ + Revision: &appsv1alpha1.SidecarSetInjectRevision{ + ID: "test-sidecarset-678235", + Policy: appsv1alpha1.AlwaysSidecarSetInjectRevisionPolicy, + }, + }, + UpdateStrategy: appsv1alpha1.SidecarSetUpdateStrategy{ + Type: appsv1alpha1.NotUpdateSidecarSetStrategyType, + }, + Containers: []appsv1alpha1.SidecarContainer{ + { + PodInjectPolicy: appsv1alpha1.BeforeAppContainerType, + ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ + Type: appsv1alpha1.ShareVolumePolicyDisabled, + }, + UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{ + UpgradeType: appsv1alpha1.SidecarContainerColdUpgrade, + }, + Container: corev1.Container{ + Name: "test-sidecar", + Image: "test-image", + ImagePullPolicy: corev1.PullIfNotPresent, + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + }, + }, + }, + }, + }, + } + + SidecarSetRevisions := []runtime.Object{ + &apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sidecarset-01234", + }, + }, + &apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sidecarset-56789", + }, + }, } for name, sidecarSet := range errorCases { - allErrs := validateSidecarSetSpec(&sidecarSet, field.NewPath("spec")) + fakeClient := fake.NewFakeClientWithScheme(testScheme, SidecarSetRevisions...) + handler.Client = fakeClient + allErrs := handler.validateSidecarSetSpec(&sidecarSet, field.NewPath("spec")) if len(allErrs) != 1 { t.Errorf("%v: expect errors len 1, but got: %v", name, allErrs) } else { diff --git a/test/e2e/apps/sidecarset.go b/test/e2e/apps/sidecarset.go index 6a01796d00..96356aa61a 100644 --- a/test/e2e/apps/sidecarset.go +++ b/test/e2e/apps/sidecarset.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "reflect" + "strconv" "time" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" @@ -1035,5 +1036,85 @@ var _ = SIGDescribe("SidecarSet", func() { revisionChecker(sidecarSetIn, 10, expectedOrder) ginkgo.By(fmt.Sprintf("sidecarSet history revision check done")) }) + + ginkgo.It("sidecarSet InjectionStrategy.Revision checker", func() { + // create sidecarSet + nginxName := func(tag string) string { + return fmt.Sprintf("nginx:%s", tag) + } + tags := []string{ + "latest", "1.21.1", "1.21", "1.20.1", "1.20", "1.19.10", + } + sidecarSetIn := tester.NewBaseSidecarSet(ns) + sidecarSetIn.SetName("e2e-test-for-injection-strategy-revision") + sidecarSetIn.Spec.UpdateStrategy.Paused = true + sidecarSetIn.Spec.Containers[0].Image = nginxName(tags[0]) + ginkgo.By(fmt.Sprintf("Creating SidecarSet %s", sidecarSetIn.Name)) + sidecarSetIn = tester.CreateSidecarSet(sidecarSetIn) + // update sidecarSet and stored revisions + time.Sleep(time.Second) + for i := 1; i < 6; i++ { + sidecarSetIn.Spec.Containers[0].Image = nginxName(tags[i]) + sidecarSetIn.Labels[sidecarcontrol.SidecarSetRevisionIDLabel] = strconv.Itoa(i) + tester.UpdateSidecarSet(sidecarSetIn) + gomega.Eventually(func() int { + rv := tester.ListControllerRevisions(sidecarSetIn) + return len(rv) + }, 5*time.Second, time.Second).Should(gomega.Equal(i + 1)) + } + + // pick a history revision to inject + pick := 3 + list := tester.ListControllerRevisions(sidecarSetIn) + gomega.Expect(list).To(gomega.HaveLen(6)) + history.SortControllerRevisions(list) + sidecarSetIn.Spec.InjectionStrategy.Revision = &appsv1alpha1.SidecarSetInjectRevision{ + ID: strconv.Itoa(pick), + Policy: appsv1alpha1.AlwaysSidecarSetInjectRevisionPolicy, + } + tester.UpdateSidecarSet(sidecarSetIn) + time.Sleep(time.Second) + + // create deployment + deploymentIn := tester.NewBaseDeployment(ns) + deploymentIn.Spec.Replicas = utilpointer.Int32Ptr(1) + ginkgo.By(fmt.Sprintf("Creating Deployment(%s.%s)", deploymentIn.Namespace, deploymentIn.Name)) + tester.CreateDeployment(deploymentIn) + + // check sidecarSet revision + pods, err := tester.GetSelectorPods(ns, deploymentIn.Spec.Selector) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(pods).To(gomega.HaveLen(1)) + gomega.Expect(pods[0].Spec.Containers[0].Image).To(gomega.Equal(nginxName(tags[pick]))) + + // check pod sidecarSetHash + gomega.Expect(len(pods[0].Annotations[sidecarcontrol.SidecarSetHashAnnotation]) > 0).To(gomega.BeTrue()) + hash := make(map[string]sidecarcontrol.SidecarSetUpgradeSpec) + err = json.Unmarshal([]byte(pods[0].Annotations[sidecarcontrol.SidecarSetHashAnnotation]), &hash) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(hash[sidecarSetIn.Name].SidecarSetControllerRevision).To(gomega.Equal(list[pick].Name)) + + // check again after sidecarSet upgrade + sidecarSetIn.Spec.UpdateStrategy.Paused = false + tester.UpdateSidecarSet(sidecarSetIn) + except := &appsv1alpha1.SidecarSetStatus{ + MatchedPods: 1, + UpdatedPods: 1, + UpdatedReadyPods: 1, + ReadyPods: 1, + } + tester.WaitForSidecarSetUpgradeComplete(sidecarSetIn, except) + + pods, err = tester.GetSelectorPods(ns, deploymentIn.Spec.Selector) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(pods).To(gomega.HaveLen(1)) + gomega.Expect(pods[0].Spec.Containers[0].Image).To(gomega.Equal(nginxName(tags[5]))) + gomega.Expect(len(pods[0].Annotations[sidecarcontrol.SidecarSetHashAnnotation]) > 0).To(gomega.BeTrue()) + hash = make(map[string]sidecarcontrol.SidecarSetUpgradeSpec) + err = json.Unmarshal([]byte(pods[0].Annotations[sidecarcontrol.SidecarSetHashAnnotation]), &hash) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(hash[sidecarSetIn.Name].SidecarSetControllerRevision).To(gomega.Equal(list[5].Name)) + ginkgo.By(fmt.Sprintf("sidecarSet InjectionStrategy.Revision check done")) + }) }) })