Skip to content

Commit

Permalink
make some logic changes
Browse files Browse the repository at this point in the history 
Signed-off-by: Kuromesi <[email protected]>
  • Loading branch information
@Kuromesi
Kuromesi committed Sep 12, 2023
1 parent 509341f commit 520ae57
Show file tree
Hide file tree
Showing 5 changed files with 312 additions and 70 deletions.
126 changes: 107 additions & 19 deletions pkg/trafficrouting/network/custom/custom.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,10 @@ func NewCustomController(client client.Client, conf Config) (network.NetworkProv
return r, nil
}

// when initializing, first check lua and get all custom providers, then store custom providers
// once fails to store a custom provider, roll back custom providers previously (assume error should not occur)
func (r *customController) Initialize(ctx context.Context) error {
customNetworkRefList := make([]*unstructured.Unstructured, 0)
for _, ref := range r.conf.TrafficConf {
obj := &unstructured.Unstructured{}
obj.SetAPIVersion(ref.APIVersion)
Expand All @@ -100,16 +103,52 @@ func (r *customController) Initialize(ctx context.Context) error {
klog.Errorf("failed to get lua script for custom network provider %s(%s/%s): %s", ref.Kind, r.conf.RolloutNs, ref.Name, err.Error())
return err
}
if err := r.storeObject(obj); err != nil {
customNetworkRefList = append(customNetworkRefList, obj)
}
for i := 0; i < len(customNetworkRefList); i++ {
nObj := customNetworkRefList[i].DeepCopy()
err := r.storeObject(nObj)
if err != nil {
klog.Errorf("failed to store custom network provider %s(%s/%s): %s", customNetworkRefList[i].GetKind(), r.conf.RolloutNs, customNetworkRefList[i].GetName(), err.Error())
klog.Errorf("roll back custom network providers previously")
r.rollBack(customNetworkRefList, i)
return err
}
customNetworkRefList[i] = nObj
}
return nil
}

// when ensuring routes, first execute lua for all custom providers, then update
// once fails to update a custom provider, roll back custom providers previously (assume error should not occur)
func (r *customController) EnsureRoutes(ctx context.Context, strategy *rolloutv1alpha1.TrafficRoutingStrategy) (bool, error) {
done := true
// *strategy.Weight == 0 indicates traffic routing is doing finalising and tries to route whole traffic to stable service
// then restore custom network provider directly
if strategy.Weight != nil && *strategy.Weight == 0 {
for _, ref := range r.conf.TrafficConf {
obj := &unstructured.Unstructured{}
obj.SetAPIVersion(ref.APIVersion)
obj.SetKind(ref.Kind)
if err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: ref.Name}, obj); err != nil {
klog.Errorf("failed to get %s(%s/%s) when routing whole traffic to stable service", ref.Kind, r.conf.RolloutNs, ref.Name)
return false, err
}
changed, err := r.restoreObject(obj, true)
if err != nil {
klog.Errorf("failed to restore %s(%s/%s) when routing whole traffic to stable service", ref.Kind, r.conf.RolloutNs, ref.Name)
return false, err
}
if changed {
done = false
}
}
return done, nil
}
var err error
var done = true
nSpecList := make([]Data, 0)
customNetworkRefList := make([]*unstructured.Unstructured, 0)
// first execute lua for new spec
for _, ref := range r.conf.TrafficConf {
obj := &unstructured.Unstructured{}
obj.SetAPIVersion(ref.APIVersion)
Expand All @@ -133,21 +172,32 @@ func (r *customController) EnsureRoutes(ctx context.Context, strategy *rolloutv1
klog.Errorf("failed to execute lua for %s(%s/%s): %s", ref.Kind, r.conf.RolloutNs, ref.Name, err.Error())
return false, err
}
nObj := obj.DeepCopy()
nSpecList = append(nSpecList, nSpec)
customNetworkRefList = append(customNetworkRefList, obj)
}
// update CustomNetworkRefs then
for i := 0; i < len(nSpecList); i++ {
nObj := customNetworkRefList[i].DeepCopy()
nSpec := nSpecList[i]
if compareAndSetObject(nSpec, nObj) {
continue
}
if err = r.Update(context.TODO(), nObj); err != nil {
klog.Errorf("failed to update custom network provider %s(%s/%s) from (%s) to (%s)", ref.Kind, r.conf.RolloutNs, ref.Name, util.DumpJSON(obj), util.DumpJSON(nObj))
klog.Errorf("failed to update custom network provider %s(%s/%s) from (%s) to (%s)", nObj.GetKind(), r.conf.RolloutNs, nObj.GetName(), util.DumpJSON(customNetworkRefList[i]), util.DumpJSON(nObj))
// if fails to update, restore the previous CustomNetworkRefs
klog.Errorf("roll back custom network providers previously")
r.rollBack(customNetworkRefList, i)
return false, err
}
klog.Infof("update custom network provider %s(%s/%s) from (%s) to (%s) success", ref.Kind, r.conf.RolloutNs, ref.Name, util.DumpJSON(obj), util.DumpJSON(nObj))
klog.Infof("update custom network provider %s(%s/%s) from (%s) to (%s) success", nObj.GetKind(), r.conf.RolloutNs, nObj.GetName(), util.DumpJSON(customNetworkRefList[i]), util.DumpJSON(nObj))
customNetworkRefList[i] = nObj
done = false
}
return done, nil
}

func (r *customController) Finalise(ctx context.Context) error {
done := true
for _, ref := range r.conf.TrafficConf {
obj := &unstructured.Unstructured{}
obj.SetAPIVersion(ref.APIVersion)
Expand All @@ -157,11 +207,29 @@ func (r *customController) Finalise(ctx context.Context) error {
klog.Infof("custom network provider %s(%s/%s) not found when finalising", ref.Kind, r.conf.RolloutNs, ref.Name)
continue
}
return err
klog.Errorf("failed to get %s(%s/%s) when finalising, proccess next first", ref.Kind, r.conf.RolloutNs, ref.Name)
done = false
continue
}
if err := r.restoreObject(obj); err != nil {
return err
if _, err := r.restoreObject(obj, false); err != nil {
done = false
klog.Errorf("failed to restore %s(%s/%s) when finalising: %s", ref.Kind, r.conf.RolloutNs, ref.Name, err.Error())
}
}
if !done {
return fmt.Errorf("finalising work is not done")
}
return nil
}

// roll back custom network provider previous to failedIdx
func (r *customController) rollBack(customNetworkRefList []*unstructured.Unstructured, failedIdx int) error {
for j := 0; j < failedIdx; j++ {
if _, err := r.restoreObject(customNetworkRefList[j].DeepCopy(), true); err != nil {
klog.Errorf("failed to roll back custom network provider %s(%s/%s): %s", customNetworkRefList[j].GetKind(), r.conf.RolloutNs, customNetworkRefList[j].GetName(), err.Error())
continue
}
klog.Infof("roll back custom network provider %s(%s/%s) success", customNetworkRefList[j].GetKind(), r.conf.RolloutNs, customNetworkRefList[j].GetName())
}
return nil
}
Expand All @@ -173,18 +241,18 @@ func (r *customController) storeObject(obj *unstructured.Unstructured) error {
annotations = make(map[string]string)
}
labels := obj.GetLabels()
oSpec := annotations[OriginalSpecAnnotation]
oSpecStr := annotations[OriginalSpecAnnotation]
delete(annotations, OriginalSpecAnnotation)
data := Data{
Spec: obj.Object["spec"],
Labels: labels,
Annotations: annotations,
}
cSpec := util.DumpJSON(data)
if oSpec == cSpec {
cSpecStr := util.DumpJSON(data)
if oSpecStr == cSpecStr {
return nil
}
annotations[OriginalSpecAnnotation] = cSpec
annotations[OriginalSpecAnnotation] = cSpecStr
obj.SetAnnotations(annotations)
if err := r.Update(context.TODO(), obj); err != nil {
klog.Errorf("failed to store custom network provider %s(%s/%s): %s", obj.GetKind(), r.conf.RolloutNs, obj.GetName(), err.Error())
Expand All @@ -194,24 +262,44 @@ func (r *customController) storeObject(obj *unstructured.Unstructured) error {
return nil
}

// restore an object from spec stored in OriginalSpecAnnotation
func (r *customController) restoreObject(obj *unstructured.Unstructured) error {
// restore an object from spec stored in OriginalSpecAnnotation, needStoreOriginalSpec indicates whether the original spec should be stored
// if needStoreOriginalSpec is false, indicates traffic routing is doing finalising
// if needStoreOriginalSpec is true, indicates traffic routing failed when ensuring routes and is rolling back or in finalising
func (r *customController) restoreObject(obj *unstructured.Unstructured, needStoreOriginalSpec bool) (bool, error) {
changed := false
annotations := obj.GetAnnotations()
if annotations == nil || annotations[OriginalSpecAnnotation] == "" {
return nil
klog.Infof("OriginalSpecAnnotation not found in custom network provider %s(%s/%s)")
return changed, nil
}
specStr := annotations[OriginalSpecAnnotation]
oSpecStr := annotations[OriginalSpecAnnotation]
var oSpec Data
_ = json.Unmarshal([]byte(specStr), &oSpec)
_ = json.Unmarshal([]byte(oSpecStr), &oSpec)
if needStoreOriginalSpec {
// when the traffic routing is rolling back, first check whether current spec equals to original spec, if equals, then just return
// this is not a concern when traffic routing is doing finalising, since the OriginalSpecAnnotation should be removed and network provider always need to be updated
delete(annotations, OriginalSpecAnnotation)
data := Data{
Spec: obj.Object["spec"],
Labels: obj.GetLabels(),
Annotations: annotations,
}
cSpecStr := util.DumpJSON(data)
if oSpecStr == cSpecStr {
return changed, nil
}
oSpec.Annotations[OriginalSpecAnnotation] = oSpecStr
}
obj.Object["spec"] = oSpec.Spec
obj.SetAnnotations(oSpec.Annotations)
obj.SetLabels(oSpec.Labels)
if err := r.Update(context.TODO(), obj); err != nil {
klog.Errorf("failed to restore object %s(%s/%s) from annotation(%s): %s", obj.GetKind(), r.conf.RolloutNs, obj.GetName(), OriginalSpecAnnotation, err.Error())
return err
return changed, err
}
changed = true
klog.Infof("restore custom network provider %s(%s/%s) from annotation(%s) success", obj.GetKind(), obj.GetNamespace(), obj.GetName(), OriginalSpecAnnotation)
return nil
return changed, nil
}

func (r *customController) executeLuaForCanary(spec Data, strategy *rolloutv1alpha1.TrafficRoutingStrategy, luaScript string) (Data, error) {
Expand Down
Loading

0 comments on commit 520ae57

Please sign in to comment.