Skip to content

Commit

Permalink
store history revisions for sidecarset
Browse files Browse the repository at this point in the history
Signed-off-by: veophi <[email protected]>
  • Loading branch information
veophi committed Dec 20, 2021
1 parent 0417db8 commit b95ead0
Show file tree
Hide file tree
Showing 14 changed files with 718 additions and 16 deletions.
13 changes: 11 additions & 2 deletions apis/apps/defaults/v1alpha1.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import (

// SetDefaults_SidecarSet set default values for SidecarSet.
func SetDefaultsSidecarSet(obj *v1alpha1.SidecarSet) {
setSidecarSetUpdateStratety(&obj.Spec.UpdateStrategy)
setSidecarSetUpdateStrategy(&obj.Spec.UpdateStrategy)

for i := range obj.Spec.InitContainers {
setSidecarDefaultContainer(&obj.Spec.InitContainers[i])
Expand All @@ -40,6 +40,15 @@ func SetDefaultsSidecarSet(obj *v1alpha1.SidecarSet) {

//default setting volumes
SetDefaultPodVolumes(obj.Spec.Volumes)

//default setting history revision limitation
SetDefaultRevisionHistoryLimit(&obj.Spec.RevisionHistoryLimit)
}

func SetDefaultRevisionHistoryLimit(revisionHistoryLimit **int32) {
if *revisionHistoryLimit == nil {
*revisionHistoryLimit = utilpointer.Int32Ptr(10)
}
}

func setDefaultSidecarContainer(sidecarContainer *v1alpha1.SidecarContainer) {
Expand All @@ -56,7 +65,7 @@ func setDefaultSidecarContainer(sidecarContainer *v1alpha1.SidecarContainer) {
setSidecarDefaultContainer(sidecarContainer)
}

func setSidecarSetUpdateStratety(strategy *v1alpha1.SidecarSetUpdateStrategy) {
func setSidecarSetUpdateStrategy(strategy *v1alpha1.SidecarSetUpdateStrategy) {
if strategy.Type == "" {
strategy.Type = v1alpha1.RollingUpdateSidecarSetStrategyType
}
Expand Down
12 changes: 12 additions & 0 deletions apis/apps/v1alpha1/sidecarset_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ type SidecarSetSpec struct {

// List of the names of secrets required by pulling sidecar container images
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`

// RevisionHistoryLimit indicates the maximum quantity of stored revisions about the SidecarSet.
// default value is 10
RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"`
}

// SidecarContainer defines the container of Sidecar
Expand Down Expand Up @@ -202,6 +206,14 @@ type SidecarSetStatus struct {

// updatedReadyPods is the number of matched pods that updated and ready
UpdatedReadyPods int32 `json:"updatedReadyPods,omitempty"`

// LatestRevision, if not empty, indicates the latest controllerRevision name of the SidecarSet.
LatestRevision string `json:"latestRevision,omitempty"`

// CollisionCount is the count of hash collisions for the SidecarSet. The SidecarSet controller
// uses this field as a collision avoidance mechanism when it needs to create the name for the
// newest ControllerRevision.
CollisionCount *int32 `json:"collisionCount,omitempty"`
}

// +genclient
Expand Down
12 changes: 11 additions & 1 deletion apis/apps/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 16 additions & 0 deletions config/crd/bases/apps.kruise.io_sidecarsets.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,11 @@ spec:
description: Namespace sidecarSet will only match the pods in the
namespace otherwise, match pods in all namespaces(in cluster)
type: string
revisionHistoryLimit:
description: RevisionHistoryLimit indicates the maximum quantity of
stored revisions about the SidecarSet. default value is 10
format: int32
type: integer
selector:
description: selector is a label query over pods that should be injected
properties:
Expand Down Expand Up @@ -387,6 +392,17 @@ spec:
status:
description: SidecarSetStatus defines the observed state of SidecarSet
properties:
collisionCount:
description: CollisionCount is the count of hash collisions for the
SidecarSet. The SidecarSet controller uses this field as a collision
avoidance mechanism when it needs to create the name for the newest
ControllerRevision.
format: int32
type: integer
latestRevision:
description: LatestRevision, if not empty, indicates the latest controllerRevision
name of the SidecarSet.
type: string
matchedPods:
description: matchedPods is the number of Pods whose labels are matched
with this SidecarSet's selector and are created after sidecarset
Expand Down
173 changes: 173 additions & 0 deletions pkg/control/sidecarcontrol/history_control.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
/*
Copyright 2021 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sidecarcontrol

import (
"bytes"
"context"
"encoding/json"
"fmt"

appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"

apps "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/kubernetes/pkg/controller/history"
"sigs.k8s.io/controller-runtime/pkg/client"
)

var (
patchCodec = scheme.Codecs.LegacyCodec(appsv1alpha1.SchemeGroupVersion)
)

type HistoryControl interface {
CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error)
NewRevision(s *appsv1alpha1.SidecarSet, namespace string, revision int64, collisionCount *int32) (*apps.ControllerRevision, error)
NextRevision(revisions []*apps.ControllerRevision) int64
GetRevisionLabelSelector(s *appsv1alpha1.SidecarSet) *metav1.LabelSelector
}

type realControl struct {
Client client.Client
}

func NewHistoryControl(client client.Client) HistoryControl {
return &realControl{
Client: client,
}
}

func (r *realControl) NewRevision(s *appsv1alpha1.SidecarSet, namespace string, revision int64, collisionCount *int32) (
*apps.ControllerRevision, error,
) {
patch, err := r.getPatch(s)
if err != nil {
return nil, err
}

cr, err := history.NewControllerRevision(s,
s.GetObjectKind().GroupVersionKind(),
s.Labels,
runtime.RawExtension{Raw: patch},
revision,
collisionCount)
if err != nil {
return nil, err
}

cr.SetNamespace(namespace)
if cr.Labels == nil {
cr.Labels = make(map[string]string)
}
if cr.ObjectMeta.Annotations == nil {
cr.ObjectMeta.Annotations = make(map[string]string)
}
cr.Labels[SidecarSetKindName] = s.Name
for key, value := range s.Annotations {
cr.ObjectMeta.Annotations[key] = value
}
return cr, nil
}

// getPatch returns a strategic merge patch that can be applied to restore a SidecarSet to a
// previous version. If the returned error is nil the patch is valid. The current state that we save is just the
// PodSpecTemplate. We can modify this later to encompass more state (or less) and remain compatible with previously
// recorded patches.
func (r *realControl) getPatch(s *appsv1alpha1.SidecarSet) ([]byte, error) {
str, err := runtime.Encode(patchCodec, s)
if err != nil {
return nil, err
}
var raw map[string]interface{}
_ = json.Unmarshal(str, &raw)

objCopy := make(map[string]interface{})
specCopy := make(map[string]interface{})
// only copy some specified fields of s.Spec to specCopy
spec := raw["spec"].(map[string]interface{})
copySidecarSetSpecRevision(specCopy, spec)

objCopy["spec"] = specCopy
return json.Marshal(objCopy)
}

// NextRevision finds the next valid revision number based on revisions. If the length of revisions
// is 0 this is 1. Otherwise, it is 1 greater than the largest revision's Revision. This method
// assumes that revisions has been sorted by Revision.
func (r *realControl) NextRevision(revisions []*apps.ControllerRevision) int64 {
count := len(revisions)
if count <= 0 {
return 1
}
return revisions[count-1].Revision + 1
}

func (r *realControl) GetRevisionLabelSelector(s *appsv1alpha1.SidecarSet) *metav1.LabelSelector {
return &metav1.LabelSelector{
MatchLabels: map[string]string{
SidecarSetKindName: s.GetName(),
},
}
}

func (r *realControl) CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) {
if collisionCount == nil {
return nil, fmt.Errorf("collisionCount should not be nil")
}

// Clone the input
clone := revision.DeepCopy()

// Continue to attempt to create the revision updating the name with a new hash on each iteration
for {
hash := history.HashControllerRevision(revision, collisionCount)
// Update the revisions name
clone.Name = history.ControllerRevisionName(parent.GetName(), hash)
err := r.Client.Create(context.TODO(), clone)
if errors.IsAlreadyExists(err) {
exists := &apps.ControllerRevision{}
key := types.NamespacedName{
Namespace: clone.Namespace,
Name: clone.Name,
}
err = r.Client.Get(context.TODO(), key, exists)
if err != nil {
return nil, err
}
if bytes.Equal(exists.Data.Raw, clone.Data.Raw) {
return exists, nil
}
*collisionCount++
continue
}
return clone, err
}
}

func copySidecarSetSpecRevision(dst, src map[string]interface{}) {
// we will use patch instead of update operation to update pods in the future
// dst["$patch"] = "replace"
// only record these revisions
dst["volumes"] = src["volumes"]
dst["containers"] = src["containers"]
dst["initContainers"] = src["initContainers"]
dst["imagePullSecrets"] = src["imagePullSecrets"]
}
25 changes: 17 additions & 8 deletions pkg/control/sidecarcontrol/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ import (
)

const (
SidecarSetKindName = "kruise.io/sidecarset-name"

// SidecarSetHashAnnotation represents the key of a sidecarSet hash
SidecarSetHashAnnotation = "kruise.io/sidecarset-hash"
// SidecarSetHashWithoutImageAnnotation represents the key of a sidecarset hash without images of sidecar
Expand All @@ -62,10 +64,11 @@ var (
)

type SidecarSetUpgradeSpec struct {
UpdateTimestamp metav1.Time `json:"updateTimestamp"`
SidecarSetHash string `json:"hash"`
SidecarSetName string `json:"sidecarSetName"`
SidecarList []string `json:"sidecarList"`
UpdateTimestamp metav1.Time `json:"updateTimestamp"`
SidecarSetHash string `json:"hash"`
SidecarSetName string `json:"sidecarSetName"`
SidecarList []string `json:"sidecarList"` // sidecarSet container list
SidecarSetControllerRevision string `json:"controllerRevision"` // sidecarSet controllerRevision name
}

// PodMatchSidecarSet determines if pod match Selector of sidecar.
Expand Down Expand Up @@ -109,6 +112,11 @@ func GetPodSidecarSetRevision(sidecarSetName string, pod metav1.Object) string {
return upgradeSpec.SidecarSetHash
}

func GetPodSidecarSetControllerRevision(sidecarSetName string, pod metav1.Object) string {
upgradeSpec := GetPodSidecarSetUpgradeSpecInAnnotations(sidecarSetName, SidecarSetHashAnnotation, pod)
return upgradeSpec.SidecarSetControllerRevision
}

func GetPodSidecarSetUpgradeSpecInAnnotations(sidecarSetName, annotationKey string, pod metav1.Object) SidecarSetUpgradeSpec {
annotations := pod.GetAnnotations()
hashKey := annotationKey
Expand Down Expand Up @@ -183,10 +191,11 @@ func updatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSe
}

sidecarSetHash[sidecarSet.Name] = SidecarSetUpgradeSpec{
UpdateTimestamp: metav1.Now(),
SidecarSetHash: GetSidecarSetRevision(sidecarSet),
SidecarSetName: sidecarSet.Name,
SidecarList: sidecarList.List(),
UpdateTimestamp: metav1.Now(),
SidecarSetHash: GetSidecarSetRevision(sidecarSet),
SidecarSetName: sidecarSet.Name,
SidecarList: sidecarList.List(),
SidecarSetControllerRevision: sidecarSet.Status.LatestRevision,
}
newHash, _ := json.Marshal(sidecarSetHash)
pod.Annotations[hashKey] = string(newHash)
Expand Down
1 change: 1 addition & 0 deletions pkg/controller/sidecarset/sidecarset_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ type ReconcileSidecarSet struct {

// +kubebuilder:rbac:groups=apps.kruise.io,resources=sidecarsets,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=apps.kruise.io,resources=sidecarsets/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=apps,resources=controllerrevisions,verbs=get;list;watch;create;update;patch;delete

// Reconcile reads that state of the cluster for a SidecarSet object and makes changes based on the state read
// and what is in the SidecarSet.Spec
Expand Down
7 changes: 7 additions & 0 deletions pkg/controller/sidecarset/sidecarset_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,15 @@ import (
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
"github.com/openkruise/kruise/pkg/util/expectations"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/record"
utilpointer "k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
Expand Down Expand Up @@ -51,6 +54,7 @@ var (
//Partition: &partition,
//MaxUnavailable: &maxUnavailable,
},
RevisionHistoryLimit: utilpointer.Int32Ptr(10),
},
}

Expand Down Expand Up @@ -127,6 +131,9 @@ var (

func init() {
scheme = runtime.NewScheme()
_ = clientgoscheme.AddToScheme(scheme)
_ = appsv1alpha1.AddToScheme(clientgoscheme.Scheme)
_ = appsv1.AddToScheme(scheme)
_ = appsv1alpha1.AddToScheme(scheme)
_ = corev1.AddToScheme(scheme)
}
Expand Down
Loading

0 comments on commit b95ead0

Please sign in to comment.