diff --git a/api/v1alpha1/trafficrouting_types.go b/api/v1alpha1/trafficrouting_types.go index 0386a877..01d25e65 100644 --- a/api/v1alpha1/trafficrouting_types.go +++ b/api/v1alpha1/trafficrouting_types.go @@ -36,6 +36,8 @@ type TrafficRoutingRef struct { // Gateway holds Gateway specific configuration to route traffic // Gateway configuration only supports >= v0.4.0 (v1alpha2). Gateway *GatewayTrafficRouting `json:"gateway,omitempty"` + // CustomNetworkRefs hold a list of custom providers to route traffic + CustomNetworkRefs []CustomNetworkRef `json:"customNetworkRefs,omitempty"` } // IngressTrafficRouting configuration for ingress controller to control traffic routing @@ -149,6 +151,12 @@ type TrafficRoutingList struct { Items []TrafficRouting `json:"items"` } +type CustomNetworkRef struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Name string `json:"name"` +} + func init() { SchemeBuilder.Register(&TrafficRouting{}, &TrafficRoutingList{}) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 29d6019a..4a78d09e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -256,6 +256,21 @@ func (in *CanaryStrategy) DeepCopy() *CanaryStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomNetworkRef) DeepCopyInto(out *CustomNetworkRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomNetworkRef. +func (in *CustomNetworkRef) DeepCopy() *CustomNetworkRef { + if in == nil { + return nil + } + out := new(CustomNetworkRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeploymentExtraStatus) DeepCopyInto(out *DeploymentExtraStatus) { *out = *in @@ -901,6 +916,11 @@ func (in *TrafficRoutingRef) DeepCopyInto(out *TrafficRoutingRef) { *out = new(GatewayTrafficRouting) (*in).DeepCopyInto(*out) } + if in.CustomNetworkRefs != nil { + in, out := &in.CustomNetworkRefs, &out.CustomNetworkRefs + *out = make([]CustomNetworkRef, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TrafficRoutingRef. diff --git a/config/crd/bases/rollouts.kruise.io_rollouts.yaml b/config/crd/bases/rollouts.kruise.io_rollouts.yaml index 5e189d67..84626f25 100644 --- a/config/crd/bases/rollouts.kruise.io_rollouts.yaml +++ b/config/crd/bases/rollouts.kruise.io_rollouts.yaml @@ -340,6 +340,23 @@ spec: for supported service meshes to enable more fine-grained traffic routing properties: + customNetworkRefs: + description: CustomNetworkRefs hold a list of custom + providers to route traffic + items: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - apiVersion + - kind + - name + type: object + type: array gateway: description: Gateway holds Gateway specific configuration to route traffic Gateway configuration only supports diff --git a/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml b/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml index 68775c5f..a3d0ad39 100644 --- a/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml +++ b/config/crd/bases/rollouts.kruise.io_trafficroutings.yaml @@ -54,6 +54,23 @@ spec: for supported service meshes to enable more fine-grained traffic routing properties: + customNetworkRefs: + description: CustomNetworkRefs hold a list of custom providers + to route traffic + items: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - apiVersion + - kind + - name + type: object + type: array gateway: description: Gateway holds Gateway specific configuration to route traffic Gateway configuration only supports >= v0.4.0 diff --git a/lua_configuration/convert_test_case_to_lua_object.go b/lua_configuration/convert_test_case_to_lua_object.go new file mode 100644 index 00000000..c58bdac5 --- /dev/null +++ b/lua_configuration/convert_test_case_to_lua_object.go @@ -0,0 +1,228 @@ +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/openkruise/rollouts/api/v1alpha1" + custom "github.com/openkruise/rollouts/pkg/trafficrouting/network/customNetworkProvider" + "github.com/openkruise/rollouts/pkg/util/luamanager" + lua "github.com/yuin/gopher-lua" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/yaml" + + utilpointer "k8s.io/utils/pointer" +) + +type TestCase struct { + Rollout *v1alpha1.Rollout `json:"rollout,omitempty"` + TrafficRouting *v1alpha1.TrafficRouting `json:"trafficRouting,omitempty"` + Original *unstructured.Unstructured `json:"original,omitempty"` + Expected []*unstructured.Unstructured `json:"expected,omitempty"` +} + +// this function aims to convert testdata to lua object for debugging +// run `go run lua.go`, then this program will get all testdata and convert them into lua objects +// copy the generated objects to lua scripts and then you can start debugging your lua scripts +func main() { + err := convertTestCaseToLuaObject() + if err != nil { + fmt.Println(err) + } +} + +func convertTestCaseToLuaObject() error { + err := filepath.Walk("./", func(path string, f os.FileInfo, err error) error { + if !strings.Contains(path, "trafficRouting.lua") { + return nil + } + if err != nil { + return fmt.Errorf("failed to walk path: %s", err.Error()) + } + dir := filepath.Dir(path) + if _, err := os.Stat(filepath.Join(dir, "testdata")); err != nil { + fmt.Printf("testdata not found in %s\n", dir) + return nil + } + err = filepath.Walk(filepath.Join(dir, "testdata"), func(path string, info os.FileInfo, err error) error { + if !info.IsDir() && filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".yml" { + fmt.Printf("--- walking path: %s ---\n", path) + err = objectToTable(path) + if err != nil { + return fmt.Errorf("failed to convert object to table: %s", err) + } + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to walk path: %s", err.Error()) + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to walk path: %s", err) + } + return nil +} + +// convert a testcase object to lua table for debug +func objectToTable(path string) error { + dir, file := filepath.Split(path) + testCase, err := getLuaTestCase(path) + if err != nil { + return fmt.Errorf("failed to get lua testcase: %s", err) + } + uList := make(map[string]interface{}) + rollout := testCase.Rollout + trafficRouting := testCase.TrafficRouting + if rollout != nil { + steps := rollout.Spec.Strategy.Canary.Steps + for i, step := range steps { + weight := step.TrafficRoutingStrategy.Weight + if step.TrafficRoutingStrategy.Weight == nil { + weight = utilpointer.Int32(-1) + } + var canaryService string + stableService := rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service + canaryService = fmt.Sprintf("%s-canary", stableService) + data := &custom.LuaData{ + Data: custom.Data{ + Labels: testCase.Original.GetLabels(), + Annotations: testCase.Original.GetAnnotations(), + Spec: testCase.Original.Object["spec"], + }, + Matches: step.TrafficRoutingStrategy.Matches, + CanaryWeight: *weight, + StableWeight: 100 - *weight, + CanaryService: canaryService, + StableService: stableService, + } + uList[fmt.Sprintf("step_%d", i)] = data + } + } else if trafficRouting != nil { + weight := trafficRouting.Spec.Strategy.Weight + if weight == nil { + weight = utilpointer.Int32(-1) + } + var canaryService string + stableService := trafficRouting.Spec.ObjectRef[0].Service + canaryService = stableService + data := &custom.LuaData{ + Data: custom.Data{ + Labels: testCase.Original.GetLabels(), + Annotations: testCase.Original.GetAnnotations(), + Spec: testCase.Original.Object["spec"], + }, + Matches: trafficRouting.Spec.Strategy.Matches, + CanaryWeight: *weight, + StableWeight: 100 - *weight, + CanaryService: canaryService, + StableService: stableService, + } + uList["steps_0"] = data + } else { + return fmt.Errorf("neither rollout nor trafficRouting defined in test case: %s", path) + } + + objStr, err := executeLua(uList) + if err != nil { + return fmt.Errorf("failed to execute lua: %s", err.Error()) + } + filePath := fmt.Sprintf("%s%s_obj.lua", dir, strings.Split(file, ".")[0]) + fileStream, err := os.OpenFile(filePath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) + if err != nil { + return fmt.Errorf("failed to open file: %s", err) + } + defer fileStream.Close() + header := "-- THIS IS GENERATED BY LUA.GO FOR DEBUGGING --\n" + _, err = io.WriteString(fileStream, header+objStr) + if err != nil { + return fmt.Errorf("failed to WriteString %s", err) + } + return nil +} + +func getLuaTestCase(path string) (*TestCase, error) { + yamlFile, err := os.ReadFile(path) + if err != nil { + return nil, err + } + luaTestCase := &TestCase{} + err = yaml.Unmarshal(yamlFile, luaTestCase) + if err != nil { + return nil, err + } + return luaTestCase, nil +} + +func executeLua(steps map[string]interface{}) (string, error) { + luaManager := &luamanager.LuaManager{} + unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&steps) + if err != nil { + return "", fmt.Errorf("failed to convert to unstructured: %s", err) + } + u := &unstructured.Unstructured{Object: unObj} + script := ` + function serialize(obj, isKey) + local lua = "" + local t = type(obj) + if t == "number" then + lua = lua .. obj + elseif t == "boolean" then + lua = lua .. tostring(obj) + elseif t == "string" then + if isKey then + lua = lua .. string.format("%s", obj) + else + lua = lua .. string.format("%q", obj) + end + elseif t == "table" then + lua = lua .. "{" + for k, v in pairs(obj) do + if type(k) == "string" then + lua = lua .. serialize(k, true) .. "=" .. serialize(v, false) .. "," + else + lua = lua .. serialize(v, false) .. "," + end + end + local metatable = getmetatable(obj) + if metatable ~= nil and type(metatable.__index) == "table" then + for k, v in pairs(metatable.__index) do + if type(k) == "string" then + lua = lua .. serialize(k, true) .. "=" .. serialize(v, false) .. "," + else + lua = lua .. serialize(v, false) .. "," + end + end + end + lua = lua .. "}" + elseif t == "nil" then + return nil + else + error("can not serialize a " .. t .. " type.") + end + return lua + end + + function table2string(tablevalue) + local stringtable = "steps=" .. serialize(tablevalue) + print(stringtable) + return stringtable + end + return table2string(obj) + ` + l, err := luaManager.RunLuaScript(u, script) + if err != nil { + return "", fmt.Errorf("failed to run lua script: %s", err) + } + returnValue := l.Get(-1) + if returnValue.Type() == lua.LTString { + return returnValue.String(), nil + } else { + return "", fmt.Errorf("unexpected lua output type") + } +} diff --git a/pkg/controller/trafficrouting/trafficrouting_controller.go b/pkg/controller/trafficrouting/trafficrouting_controller.go index eb62d3c2..88ad5123 100644 --- a/pkg/controller/trafficrouting/trafficrouting_controller.go +++ b/pkg/controller/trafficrouting/trafficrouting_controller.go @@ -97,6 +97,7 @@ func (r *TrafficRoutingReconciler) Reconcile(ctx context.Context, req ctrl.Reque newStatus := tr.Status.DeepCopy() if newStatus.Phase == "" { newStatus.Phase = v1alpha1.TrafficRoutingPhaseInitial + newStatus.Message = "TrafficRouting is Initializing" } if !tr.DeletionTimestamp.IsZero() { newStatus.Phase = v1alpha1.TrafficRoutingPhaseTerminating diff --git a/pkg/trafficrouting/manager.go b/pkg/trafficrouting/manager.go index 390b4cd0..7fc6251c 100644 --- a/pkg/trafficrouting/manager.go +++ b/pkg/trafficrouting/manager.go @@ -23,6 +23,7 @@ import ( "github.com/openkruise/rollouts/api/v1alpha1" "github.com/openkruise/rollouts/pkg/trafficrouting/network" + custom "github.com/openkruise/rollouts/pkg/trafficrouting/network/customNetworkProvider" "github.com/openkruise/rollouts/pkg/trafficrouting/network/gateway" "github.com/openkruise/rollouts/pkg/trafficrouting/network/ingress" "github.com/openkruise/rollouts/pkg/util" @@ -263,6 +264,16 @@ func (m *Manager) FinalisingTrafficRouting(c *TrafficRoutingContext, onlyRestore func newNetworkProvider(c client.Client, con *TrafficRoutingContext, sService, cService string) (network.NetworkProvider, error) { trafficRouting := con.ObjectRef[0] + if trafficRouting.CustomNetworkRefs != nil { + return custom.NewCustomController(c, custom.Config{ + Key: con.Key, + RolloutNs: con.Namespace, + CanaryService: cService, + StableService: sService, + TrafficConf: trafficRouting.CustomNetworkRefs, + OwnerRef: con.OwnerRef, + }) + } if trafficRouting.Ingress != nil { return ingress.NewIngressTrafficRouting(c, ingress.Config{ Key: con.Key, diff --git a/pkg/trafficrouting/network/customNetworkProvider/custom_network_provider.go b/pkg/trafficrouting/network/customNetworkProvider/custom_network_provider.go new file mode 100644 index 00000000..07be66e6 --- /dev/null +++ b/pkg/trafficrouting/network/customNetworkProvider/custom_network_provider.go @@ -0,0 +1,352 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package custom + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/klog/v2" + + rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/trafficrouting/network" + "github.com/openkruise/rollouts/pkg/util" + "github.com/openkruise/rollouts/pkg/util/configuration" + "github.com/openkruise/rollouts/pkg/util/luamanager" + lua "github.com/yuin/gopher-lua" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + OriginalSpecAnnotation = "rollouts.kruise.io/original-spec-configuration" + LuaConfigMap = "kruise-rollout-configuration" +) + +type LuaData struct { + Data Data + CanaryWeight int32 + StableWeight int32 + Matches []rolloutv1alpha1.HttpRouteMatch + CanaryService string + StableService string +} +type Data struct { + Spec interface{} `json:"spec,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +type customController struct { + client.Client + conf Config + luaManager *luamanager.LuaManager +} + +type Config struct { + Key string + RolloutNs string + CanaryService string + StableService string + // network providers need to be created + TrafficConf []rolloutv1alpha1.CustomNetworkRef + OwnerRef metav1.OwnerReference +} + +func NewCustomController(client client.Client, conf Config) (network.NetworkProvider, error) { + r := &customController{ + Client: client, + conf: conf, + luaManager: &luamanager.LuaManager{}, + } + return r, nil +} + +// when initializing, first check lua and get all custom providers, then store custom providers +func (r *customController) Initialize(ctx context.Context) error { + for _, ref := range r.conf.TrafficConf { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(ref.APIVersion) + obj.SetKind(ref.Kind) + if err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: ref.Name}, obj); err != nil { + klog.Errorf("failed to get custom network provider %s(%s/%s): %s", ref.Kind, r.conf.RolloutNs, ref.Name, err.Error()) + return err + } + + // check if lua script exists + _, err := r.getLuaScript(ctx, ref) + if err != nil { + klog.Errorf("failed to get lua script for custom network provider %s(%s/%s): %s", ref.Kind, r.conf.RolloutNs, ref.Name, err.Error()) + return err + } + } + return nil +} + +// when ensuring routes, first execute lua for all custom providers, then update +func (r *customController) EnsureRoutes(ctx context.Context, strategy *rolloutv1alpha1.TrafficRoutingStrategy) (bool, error) { + done := true + // *strategy.Weight == 0 indicates traffic routing is doing finalising and tries to route whole traffic to stable service + // then directly do finalising + if strategy.Weight != nil && *strategy.Weight == 0 { + return true, nil + } + var err error + customNetworkRefList := make([]*unstructured.Unstructured, len(r.conf.TrafficConf)) + + // first get all custom network provider object + for i, ref := range r.conf.TrafficConf { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(ref.APIVersion) + obj.SetKind(ref.Kind) + if err = r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: ref.Name}, obj); err != nil { + return false, err + } + customNetworkRefList[i] = obj + } + + // check if original configuration is stored in annotation, store it if not. + for i := 0; i < len(customNetworkRefList); i++ { + obj := customNetworkRefList[i] + if _, ok := obj.GetAnnotations()[OriginalSpecAnnotation]; !ok { + err := r.storeObject(obj) + if err != nil { + klog.Errorf("failed to store custom network provider %s(%s/%s): %s", customNetworkRefList[i].GetKind(), r.conf.RolloutNs, customNetworkRefList[i].GetName(), err.Error()) + return false, err + } + } + } + + // first execute lua for new spec + nSpecList := make([]Data, len(r.conf.TrafficConf)) + for i := 0; i < len(customNetworkRefList); i++ { + obj := customNetworkRefList[i] + ref := r.conf.TrafficConf[i] + specStr := obj.GetAnnotations()[OriginalSpecAnnotation] + if specStr == "" { + return false, fmt.Errorf("failed to get original spec from annotation for %s(%s/%s)", ref.Kind, r.conf.RolloutNs, ref.Name) + } + var oSpec Data + _ = json.Unmarshal([]byte(specStr), &oSpec) + luaScript, err := r.getLuaScript(ctx, ref) + if err != nil { + klog.Errorf("failed to get lua script for %s(%s/%s): %s", ref.Kind, r.conf.RolloutNs, ref.Name, err.Error()) + return false, err + } + nSpec, err := r.executeLuaForCanary(oSpec, strategy, luaScript) + if err != nil { + klog.Errorf("failed to execute lua for %s(%s/%s): %s", ref.Kind, r.conf.RolloutNs, ref.Name, err.Error()) + return false, err + } + nSpecList[i] = nSpec + } + + // update CustomNetworkRefs then + for i := 0; i < len(nSpecList); i++ { + nSpec := nSpecList[i] + updated, err := r.compareAndUpdateObject(nSpec, customNetworkRefList[i]) + if err != nil { + klog.Errorf("failed to update object %s(%s/%s) when ensure routes: %s", customNetworkRefList[i].GetKind(), r.conf.RolloutNs, customNetworkRefList[i].GetName(), err.Error()) + return false, err + } + if updated { + done = false + } + } + return done, nil +} + +func (r *customController) Finalise(ctx context.Context) error { + done := true + for _, ref := range r.conf.TrafficConf { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(ref.APIVersion) + obj.SetKind(ref.Kind) + if err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: ref.Name}, obj); err != nil { + if errors.IsNotFound(err) { + klog.Infof("custom network provider %s(%s/%s) not found when finalising", ref.Kind, r.conf.RolloutNs, ref.Name) + continue + } + klog.Errorf("failed to get %s(%s/%s) when finalising, process next first", ref.Kind, r.conf.RolloutNs, ref.Name) + done = false + continue + } + if err := r.restoreObject(obj); err != nil { + done = false + klog.Errorf("failed to restore %s(%s/%s) when finalising: %s", ref.Kind, r.conf.RolloutNs, ref.Name, err.Error()) + } + } + if !done { + return fmt.Errorf("finalising work for %s is not done", r.conf.Key) + } + return nil +} + +// store spec of an object in OriginalSpecAnnotation +func (r *customController) storeObject(obj *unstructured.Unstructured) error { + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + labels := obj.GetLabels() + oSpecStr := annotations[OriginalSpecAnnotation] + delete(annotations, OriginalSpecAnnotation) + data := Data{ + Spec: obj.Object["spec"], + Labels: labels, + Annotations: annotations, + } + cSpecStr := util.DumpJSON(data) + if oSpecStr == cSpecStr { + return nil + } + annotations[OriginalSpecAnnotation] = cSpecStr + obj.SetAnnotations(annotations) + if err := r.Update(context.TODO(), obj); err != nil { + klog.Errorf("failed to store custom network provider %s(%s/%s): %s", obj.GetKind(), r.conf.RolloutNs, obj.GetName(), err.Error()) + return err + } + klog.Infof("store old configuration of custom network provider %s(%s/%s) in annotation(%s) success", obj.GetKind(), r.conf.RolloutNs, obj.GetName(), OriginalSpecAnnotation) + return nil +} + +// restore an object from spec stored in OriginalSpecAnnotation +func (r *customController) restoreObject(obj *unstructured.Unstructured) error { + annotations := obj.GetAnnotations() + if annotations == nil || annotations[OriginalSpecAnnotation] == "" { + klog.Infof("OriginalSpecAnnotation not found in custom network provider %s(%s/%s)", obj.GetKind(), r.conf.RolloutNs, obj.GetName()) + return nil + } + oSpecStr := annotations[OriginalSpecAnnotation] + var oSpec Data + _ = json.Unmarshal([]byte(oSpecStr), &oSpec) + obj.Object["spec"] = oSpec.Spec + obj.SetAnnotations(oSpec.Annotations) + obj.SetLabels(oSpec.Labels) + if err := r.Update(context.TODO(), obj); err != nil { + klog.Errorf("failed to restore object %s(%s/%s) from annotation(%s): %s", obj.GetKind(), r.conf.RolloutNs, obj.GetName(), OriginalSpecAnnotation, err.Error()) + return err + } + klog.Infof("restore custom network provider %s(%s/%s) from annotation(%s) success", obj.GetKind(), obj.GetNamespace(), obj.GetName(), OriginalSpecAnnotation) + return nil +} + +func (r *customController) executeLuaForCanary(spec Data, strategy *rolloutv1alpha1.TrafficRoutingStrategy, luaScript string) (Data, error) { + weight := strategy.Weight + matches := strategy.Matches + if weight == nil { + // the lua script does not have a pointer type, + // so we need to pass weight=-1 to indicate the case where weight is nil. + weight = utilpointer.Int32(-1) + } + data := &LuaData{ + Data: spec, + CanaryWeight: *weight, + StableWeight: 100 - *weight, + Matches: matches, + CanaryService: r.conf.CanaryService, + StableService: r.conf.StableService, + } + + unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(data) + if err != nil { + return Data{}, err + } + u := &unstructured.Unstructured{Object: unObj} + l, err := r.luaManager.RunLuaScript(u, luaScript) + if err != nil { + return Data{}, err + } + returnValue := l.Get(-1) + if returnValue.Type() == lua.LTTable { + jsonBytes, err := luamanager.Encode(returnValue) + if err != nil { + return Data{}, err + } + var obj Data + err = json.Unmarshal(jsonBytes, &obj) + if err != nil { + return Data{}, err + } + return obj, nil + } + return Data{}, fmt.Errorf("expect table output from Lua script, not %s", returnValue.Type().String()) +} + +func (r *customController) getLuaScript(ctx context.Context, ref rolloutv1alpha1.CustomNetworkRef) (string, error) { + // get local lua script + // luaScript.Provider: CRDGroupt/Kind + group := strings.Split(ref.APIVersion, "/")[0] + key := fmt.Sprintf("lua_configuration/%s/trafficRouting.lua", fmt.Sprintf("%s/%s", group, ref.Kind)) + script := util.GetLuaConfigurationContent(key) + if script != "" { + return script, nil + } + + // if lua script is not found locally, then try ConfigMap + nameSpace := util.GetRolloutNamespace() // kruise-rollout + name := LuaConfigMap + configMap := &corev1.ConfigMap{} + err := r.Get(ctx, types.NamespacedName{Namespace: nameSpace, Name: name}, configMap) + if err != nil { + return "", fmt.Errorf("failed to get ConfigMap(%s/%s)", nameSpace, name) + } else { + // in format like "lua.traffic.routing.ingress.aliyun-alb" + key = fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingCustomTypePrefix, ref.Kind, group) + if script, ok := configMap.Data[key]; ok { + return script, nil + } else if !ok { + return "", fmt.Errorf("expected script not found neither locally nor in ConfigMap") + } + } + return "", nil +} + +// compare and update obj, return whether the obj is updated +func (r *customController) compareAndUpdateObject(data Data, obj *unstructured.Unstructured) (bool, error) { + spec := data.Spec + annotations := data.Annotations + if annotations == nil { + annotations = make(map[string]string) + } + annotations[OriginalSpecAnnotation] = obj.GetAnnotations()[OriginalSpecAnnotation] + labels := data.Labels + if util.DumpJSON(obj.Object["spec"]) == util.DumpJSON(spec) && + reflect.DeepEqual(obj.GetAnnotations(), annotations) && + reflect.DeepEqual(obj.GetLabels(), labels) { + return false, nil + } + nObj := obj.DeepCopy() + nObj.Object["spec"] = spec + nObj.SetAnnotations(annotations) + nObj.SetLabels(labels) + if err := r.Update(context.TODO(), nObj); err != nil { + klog.Errorf("failed to update custom network provider %s(%s/%s) from (%s) to (%s)", nObj.GetKind(), r.conf.RolloutNs, nObj.GetName(), util.DumpJSON(obj), util.DumpJSON(nObj)) + return false, err + } + klog.Infof("update custom network provider %s(%s/%s) from (%s) to (%s) success", nObj.GetKind(), r.conf.RolloutNs, nObj.GetName(), util.DumpJSON(obj), util.DumpJSON(nObj)) + return true, nil +} diff --git a/pkg/trafficrouting/network/customNetworkProvider/custom_network_provider_test.go b/pkg/trafficrouting/network/customNetworkProvider/custom_network_provider_test.go new file mode 100644 index 00000000..01bf63f9 --- /dev/null +++ b/pkg/trafficrouting/network/customNetworkProvider/custom_network_provider_test.go @@ -0,0 +1,649 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package custom + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" + "github.com/openkruise/rollouts/pkg/util/configuration" + "github.com/openkruise/rollouts/pkg/util/luamanager" + lua "github.com/yuin/gopher-lua" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + utilpointer "k8s.io/utils/pointer" + luajson "layeh.com/gopher-json" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/yaml" +) + +var ( + scheme *runtime.Scheme + virtualServiceDemo = ` + { + "apiVersion": "networking.istio.io/v1alpha3", + "kind": "VirtualService", + "metadata": { + "name": "echoserver", + "annotations": { + "virtual": "test" + } + }, + "spec": { + "hosts": [ + "echoserver.example.com" + ], + "http": [ + { + "route": [ + { + "destination": { + "host": "echoserver" + } + } + ] + } + ] + } + } + ` + destinationRuleDemo = ` + { + "apiVersion": "networking.istio.io/v1alpha3", + "kind": "DestinationRule", + "metadata": { + "name": "dr-demo" + }, + "spec": { + "host": "mockb", + "subsets": [ + { + "labels": { + "version": "base" + }, + "name": "version-base" + } + ], + "trafficPolicy": { + "loadBalancer": { + "simple": "ROUND_ROBIN" + } + } + } + } + ` + // lua script for this resource contains error and cannot be executed + luaErrorDemo = ` + { + "apiVersion": "networking.error.io/v1alpha3", + "kind": "LuaError", + "metadata": { + "name": "error-demo" + }, + "spec": { + "error": true + } + } + ` +) + +func init() { + scheme = runtime.NewScheme() + _ = clientgoscheme.AddToScheme(scheme) + _ = rolloutsv1alpha1.AddToScheme(scheme) +} + +func TestInitialize(t *testing.T) { + cases := []struct { + name string + getUnstructured func() *unstructured.Unstructured + getConfig func() Config + getConfigMap func() *corev1.ConfigMap + }{ + { + name: "test1, find lua script locally", + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(virtualServiceDemo)) + return u + }, + getConfig: func() Config { + return Config{ + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{ + { + APIVersion: "networking.istio.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + }, + getConfigMap: func() *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: LuaConfigMap, + Namespace: util.GetRolloutNamespace(), + }, + Data: map[string]string{ + fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingCustomTypePrefix, "VirtualService", "networking.istio.io"): "ExpectedLuaScript", + }, + } + }, + }, + { + name: "test2, find lua script in ConfigMap", + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(virtualServiceDemo)) + u.SetAPIVersion("networking.test.io/v1alpha3") + return u + }, + getConfig: func() Config { + return Config{ + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{ + { + APIVersion: "networking.test.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + }, + getConfigMap: func() *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: LuaConfigMap, + Namespace: util.GetRolloutNamespace(), + }, + Data: map[string]string{ + fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingCustomTypePrefix, "VirtualService", "networking.test.io"): "ExpectedLuaScript", + }, + } + }, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + err := fakeCli.Create(context.TODO(), cs.getUnstructured()) + if err != nil { + klog.Errorf(err.Error()) + return + } + if err := fakeCli.Create(context.TODO(), cs.getConfigMap()); err != nil { + klog.Errorf(err.Error()) + } + c, _ := NewCustomController(fakeCli, cs.getConfig()) + err = c.Initialize(context.TODO()) + if err != nil { + t.Fatalf("Initialize failed: %s", err.Error()) + } + }) + } +} + +func checkEqual(cli client.Client, t *testing.T, expect *unstructured.Unstructured) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(expect.GetAPIVersion()) + obj.SetKind(expect.GetKind()) + if err := cli.Get(context.TODO(), types.NamespacedName{Namespace: expect.GetNamespace(), Name: expect.GetName()}, obj); err != nil { + t.Fatalf("Get object failed: %s", err.Error()) + } + if !reflect.DeepEqual(obj.GetAnnotations(), expect.GetAnnotations()) { + fmt.Println(util.DumpJSON(obj.GetAnnotations()), util.DumpJSON(expect.GetAnnotations())) + t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect.GetAnnotations()), util.DumpJSON(obj.GetAnnotations())) + } + if util.DumpJSON(expect.Object["spec"]) != util.DumpJSON(obj.Object["spec"]) { + t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect.Object["spec"]), util.DumpJSON(obj.Object["spec"])) + } +} + +func TestEnsureRoutes(t *testing.T) { + cases := []struct { + name string + getLua func() map[string]string + getRoutes func() *rolloutsv1alpha1.TrafficRoutingStrategy + getUnstructureds func() []*unstructured.Unstructured + getConfig func() Config + expectState func() (bool, bool) + expectUnstructureds func() []*unstructured.Unstructured + }{ + { + name: "test1, do traffic routing for VirtualService and DestinationRule", + getRoutes: func() *rolloutsv1alpha1.TrafficRoutingStrategy { + return &rolloutsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(5), + } + }, + getUnstructureds: func() []*unstructured.Unstructured { + objects := make([]*unstructured.Unstructured, 0) + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(virtualServiceDemo)) + u.SetAPIVersion("networking.istio.io/v1alpha3") + objects = append(objects, u) + + u = &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(destinationRuleDemo)) + u.SetAPIVersion("networking.istio.io/v1alpha3") + objects = append(objects, u) + return objects + }, + getConfig: func() Config { + return Config{ + Key: "rollout-demo", + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{ + { + APIVersion: "networking.istio.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + { + APIVersion: "networking.istio.io/v1alpha3", + Kind: "DestinationRule", + Name: "dr-demo", + }, + }, + } + }, + expectUnstructureds: func() []*unstructured.Unstructured { + objects := make([]*unstructured.Unstructured, 0) + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(virtualServiceDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":95},{"destination":{"host":"echoserver-canary"},"weight":5}]}]}` + var spec interface{} + _ = json.Unmarshal([]byte(specStr), &spec) + u.Object["spec"] = spec + objects = append(objects, u) + + u = &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(destinationRuleDemo)) + annotations = map[string]string{ + OriginalSpecAnnotation: `{"spec":{"host":"mockb","subsets":[{"labels":{"version":"base"},"name":"version-base"}],"trafficPolicy":{"loadBalancer":{"simple":"ROUND_ROBIN"}}}}`, + } + u.SetAnnotations(annotations) + specStr = `{"host":"mockb","subsets":[{"labels":{"version":"base"},"name":"version-base"},{"labels":{"istio.service.tag":"gray"},"name":"canary"}],"trafficPolicy":{"loadBalancer":{"simple":"ROUND_ROBIN"}}}` + _ = json.Unmarshal([]byte(specStr), &spec) + u.Object["spec"] = spec + objects = append(objects, u) + return objects + }, + expectState: func() (bool, bool) { + done := false + hasError := false + return done, hasError + }, + }, + { + name: "test2, do traffic routing but failed to execute lua", + getRoutes: func() *rolloutsv1alpha1.TrafficRoutingStrategy { + return &rolloutsv1alpha1.TrafficRoutingStrategy{ + Weight: utilpointer.Int32(5), + } + }, + getUnstructureds: func() []*unstructured.Unstructured { + objects := make([]*unstructured.Unstructured, 0) + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(virtualServiceDemo)) + u.SetAPIVersion("networking.istio.io/v1alpha3") + objects = append(objects, u) + + u = &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(luaErrorDemo)) + u.SetAPIVersion("networking.error.io/v1alpha3") + objects = append(objects, u) + return objects + }, + getConfig: func() Config { + return Config{ + Key: "rollout-demo", + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{ + { + APIVersion: "networking.istio.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + { + APIVersion: "networking.error.io/v1alpha3", + Kind: "LuaError", + Name: "error-demo", + }, + }, + } + }, + expectUnstructureds: func() []*unstructured.Unstructured { + objects := make([]*unstructured.Unstructured, 0) + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(virtualServiceDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}` + var spec interface{} + _ = json.Unmarshal([]byte(specStr), &spec) + u.Object["spec"] = spec + objects = append(objects, u) + + u = &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(luaErrorDemo)) + annotations = map[string]string{ + OriginalSpecAnnotation: `{"spec":{"error":true}}`, + } + u.SetAnnotations(annotations) + specStr = `{"error":true}` + _ = json.Unmarshal([]byte(specStr), &spec) + u.Object["spec"] = spec + objects = append(objects, u) + return objects + }, + expectState: func() (bool, bool) { + done := false + hasError := true + return done, hasError + }, + }, + } + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + for _, obj := range cs.getUnstructureds() { + err := fakeCli.Create(context.TODO(), obj) + if err != nil { + t.Fatalf("failed to create objects: %s", err.Error()) + } + } + c, _ := NewCustomController(fakeCli, cs.getConfig()) + strategy := cs.getRoutes() + expectDone, expectHasError := cs.expectState() + err := c.Initialize(context.TODO()) + if err != nil { + t.Fatalf("failed to initialize custom controller") + } + done, err := c.EnsureRoutes(context.TODO(), strategy) + if !expectHasError && err != nil { + t.Fatalf("EnsureRoutes failed: %s", err.Error()) + } else if expectHasError && err == nil { + t.Fatalf("expect error occurred but not") + } else if done != expectDone { + t.Fatalf("expect(%v), but get(%v)", expectDone, done) + } + for _, expectUnstructured := range cs.expectUnstructureds() { + checkEqual(fakeCli, t, expectUnstructured) + } + }) + } +} + +func TestFinalise(t *testing.T) { + cases := []struct { + name string + getUnstructured func() *unstructured.Unstructured + getConfig func() Config + expectUnstructured func() *unstructured.Unstructured + }{ + { + name: "test1, finalise VirtualService", + getUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(virtualServiceDemo)) + annotations := map[string]string{ + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, + "virtual": "test", + } + u.SetAnnotations(annotations) + specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":95},{"destination":{"host":"echoserver-canary"},"weight":5}}]}]}` + var spec interface{} + _ = json.Unmarshal([]byte(specStr), &spec) + u.Object["spec"] = spec + return u + }, + getConfig: func() Config { + return Config{ + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{ + { + APIVersion: "networking.istio.io/v1alpha3", + Kind: "VirtualService", + Name: "echoserver", + }, + }, + } + }, + expectUnstructured: func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + _ = u.UnmarshalJSON([]byte(virtualServiceDemo)) + return u + }, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + err := fakeCli.Create(context.TODO(), cs.getUnstructured()) + if err != nil { + klog.Errorf(err.Error()) + return + } + c, _ := NewCustomController(fakeCli, cs.getConfig()) + err = c.Finalise(context.TODO()) + if err != nil { + t.Fatalf("Initialize failed: %s", err.Error()) + } + checkEqual(fakeCli, t, cs.expectUnstructured()) + }) + } +} + +type TestCase struct { + Rollout *rolloutsv1alpha1.Rollout `json:"rollout,omitempty"` + TrafficRouting *rolloutsv1alpha1.TrafficRouting `json:"trafficRouting,omitempty"` + Original *unstructured.Unstructured `json:"original,omitempty"` + Expected []*unstructured.Unstructured `json:"expected,omitempty"` +} + +// test if the lua script of a network provider run as expected +func TestLuaScript(t *testing.T) { + err := filepath.Walk("../../../../lua_configuration", func(path string, f os.FileInfo, err error) error { + if !strings.Contains(path, "trafficRouting.lua") { + return nil + } + if err != nil { + t.Errorf("failed to walk lua script dir") + return err + } + script, err := readScript(t, path) + if err != nil { + t.Errorf("failed to read lua script from: %s", path) + return err + } + dir := filepath.Dir(path) + err = filepath.Walk(filepath.Join(dir, "testdata"), func(path string, info os.FileInfo, err error) error { + klog.Infof("testing lua script: %s", path) + if err != nil { + t.Errorf("fail to walk testdata dir") + return err + } + + if !info.IsDir() && filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".yml" { + testCase, err := getLuaTestCase(t, path) + if err != nil { + t.Errorf("faied to get lua test case: %s", path) + return err + } + rollout := testCase.Rollout + trafficRouting := testCase.TrafficRouting + if rollout != nil { + steps := rollout.Spec.Strategy.Canary.Steps + for i, step := range steps { + weight := step.TrafficRoutingStrategy.Weight + if weight == nil { + weight = utilpointer.Int32(-1) + } + var canaryService string + stableService := rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service + canaryService = fmt.Sprintf("%s-canary", stableService) + data := &LuaData{ + Data: Data{ + Labels: testCase.Original.GetLabels(), + Annotations: testCase.Original.GetAnnotations(), + Spec: testCase.Original.Object["spec"], + }, + Matches: step.TrafficRoutingStrategy.Matches, + CanaryWeight: *weight, + StableWeight: 100 - *weight, + CanaryService: canaryService, + StableService: stableService, + } + nSpec, err := executeLua(data, script) + if err != nil { + t.Errorf("failed to execute lua for test case: %s", path) + return err + } + eSpec := Data{ + Spec: testCase.Expected[i].Object["spec"], + Annotations: testCase.Expected[i].GetAnnotations(), + Labels: testCase.Expected[i].GetLabels(), + } + if util.DumpJSON(eSpec) != util.DumpJSON(nSpec) { + return fmt.Errorf("expect %s, but get %s for test case: %s", util.DumpJSON(eSpec), util.DumpJSON(nSpec), path) + } + } + } else if trafficRouting != nil { + weight := trafficRouting.Spec.Strategy.Weight + if weight == nil { + weight = utilpointer.Int32(-1) + } + var canaryService string + stableService := trafficRouting.Spec.ObjectRef[0].Service + canaryService = stableService + data := &LuaData{ + Data: Data{ + Labels: testCase.Original.GetLabels(), + Annotations: testCase.Original.GetAnnotations(), + Spec: testCase.Original.Object["spec"], + }, + Matches: trafficRouting.Spec.Strategy.Matches, + CanaryWeight: *weight, + StableWeight: 100 - *weight, + CanaryService: canaryService, + StableService: stableService, + } + nSpec, err := executeLua(data, script) + if err != nil { + t.Errorf("failed to execute lua for test case: %s", path) + return err + } + eSpec := Data{ + Spec: testCase.Expected[0].Object["spec"], + Annotations: testCase.Expected[0].GetAnnotations(), + Labels: testCase.Expected[0].GetLabels(), + } + if util.DumpJSON(eSpec) != util.DumpJSON(nSpec) { + return fmt.Errorf("expect %s, but get %s for test case: %s", util.DumpJSON(eSpec), util.DumpJSON(nSpec), path) + } + } else { + return fmt.Errorf("neither rollout nor trafficRouting defined in test case: %s", path) + } + } + return nil + }) + return err + }) + if err != nil { + t.Fatalf("failed to test lua scripts: %s", err.Error()) + } +} + +func readScript(t *testing.T, path string) (string, error) { + data, err := os.ReadFile(filepath.Clean(path)) + if err != nil { + return "", err + } + return string(data), err +} + +func getLuaTestCase(t *testing.T, path string) (*TestCase, error) { + yamlFile, err := os.ReadFile(path) + if err != nil { + t.Errorf("failed to read file %s", path) + return nil, err + } + luaTestCase := &TestCase{} + err = yaml.Unmarshal(yamlFile, luaTestCase) + if err != nil { + t.Errorf("test case %s format error", path) + return nil, err + } + return luaTestCase, nil +} + +func executeLua(data *LuaData, script string) (Data, error) { + luaManager := &luamanager.LuaManager{} + unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(data) + if err != nil { + return Data{}, err + } + u := &unstructured.Unstructured{Object: unObj} + l, err := luaManager.RunLuaScript(u, script) + if err != nil { + return Data{}, err + } + returnValue := l.Get(-1) + var nSpec Data + if returnValue.Type() == lua.LTTable { + jsonBytes, err := luajson.Encode(returnValue) + if err != nil { + return Data{}, err + } + err = json.Unmarshal(jsonBytes, &nSpec) + if err != nil { + return Data{}, err + } + } + return nSpec, nil +} diff --git a/pkg/trafficrouting/network/customNetworkProvider/lua_configuration/networking.error.io/LuaError/trafficRouting.lua b/pkg/trafficrouting/network/customNetworkProvider/lua_configuration/networking.error.io/LuaError/trafficRouting.lua new file mode 100644 index 00000000..59282070 --- /dev/null +++ b/pkg/trafficrouting/network/customNetworkProvider/lua_configuration/networking.error.io/LuaError/trafficRouting.lua @@ -0,0 +1,3 @@ +-- the lua script contains error +local spec = obj.error +return sepc.error \ No newline at end of file diff --git a/pkg/trafficrouting/network/customNetworkProvider/lua_configuration/networking.istio.io/DestinationRule/trafficRouting.lua b/pkg/trafficrouting/network/customNetworkProvider/lua_configuration/networking.istio.io/DestinationRule/trafficRouting.lua new file mode 100644 index 00000000..3df3aace --- /dev/null +++ b/pkg/trafficrouting/network/customNetworkProvider/lua_configuration/networking.istio.io/DestinationRule/trafficRouting.lua @@ -0,0 +1,8 @@ +local spec = obj.data.spec +local canary = {} +canary.labels = {} +canary.name = "canary" +local podLabelKey = "istio.service.tag" +canary.labels[podLabelKey] = "gray" +table.insert(spec.subsets, canary) +return obj.data \ No newline at end of file diff --git a/pkg/trafficrouting/network/customNetworkProvider/lua_configuration/networking.istio.io/VirtualService/trafficRouting.lua b/pkg/trafficrouting/network/customNetworkProvider/lua_configuration/networking.istio.io/VirtualService/trafficRouting.lua new file mode 100644 index 00000000..d4d52c23 --- /dev/null +++ b/pkg/trafficrouting/network/customNetworkProvider/lua_configuration/networking.istio.io/VirtualService/trafficRouting.lua @@ -0,0 +1,216 @@ +spec = obj.data.spec + +if obj.canaryWeight == -1 then + obj.canaryWeight = 100 + obj.stableWeight = 0 +end + +function FindRules(spec, protocol) + local rules = {} + if (protocol == "http") then + if (spec.http ~= nil) then + for _, http in ipairs(spec.http) do + table.insert(rules, http) + end + end + elseif (protocol == "tcp") then + if (spec.tcp ~= nil) then + for _, http in ipairs(spec.tcp) do + table.insert(rules, http) + end + end + elseif (protocol == "tls") then + if (spec.tls ~= nil) then + for _, http in ipairs(spec.tls) do + table.insert(rules, http) + end + end + end + return rules +end + +-- find matched route of VirtualService spec with stable svc +function FindMatchedRules(spec, stableService, protocol) + local matchedRoutes = {} + local rules = FindRules(spec, protocol) + -- a rule contains 'match' and 'route' + for _, rule in ipairs(rules) do + for _, route in ipairs(rule.route) do + if route.destination.host == stableService then + table.insert(matchedRoutes, rule) + break + end + end + end + return matchedRoutes +end + +function FindStableServiceSubsets(spec, stableService, protocol) + local stableSubsets = {} + local rules = FindRules(spec, protocol) + local hasRule = false + -- a rule contains 'match' and 'route' + for _, rule in ipairs(rules) do + for _, route in ipairs(rule.route) do + if route.destination.host == stableService then + hasRule = true + local contains = false + for _, v in ipairs(stableSubsets) do + if v == route.destination.subset then + contains = true + break + end + end + if not contains and route.destination.subset ~= nil then + table.insert(stableSubsets, route.destination.subset) + end + end + end + end + return hasRule, stableSubsets +end + +function DeepCopy(original) + local copy + if type(original) == 'table' then + copy = {} + for key, value in pairs(original) do + copy[key] = DeepCopy(value) + end + else + copy = original + end + return copy +end + +function CalculateWeight(route, stableWeight, n) + local weight + if (route.weight) then + weight = math.floor(route.weight * stableWeight / 100) + else + weight = math.floor(stableWeight / n) + end + return weight +end + +-- generate routes with matches, insert a rule before other rules +function GenerateMatchedRoutes(spec, matches, stableService, canaryService, stableWeight, canaryWeight, protocol) + local hasRule, stableServiceSubsets = FindStableServiceSubsets(spec, stableService, protocol) + if (not hasRule) then + return + end + for _, match in ipairs(matches) do + local route = {} + route["match"] = {} + + for key, value in pairs(match) do + local vsMatch = {} + vsMatch[key] = {} + for _, rule in ipairs(value) do + if rule["type"] == "RegularExpression" then + matchType = "regex" + elseif rule["type"] == "Exact" then + matchType = "exact" + elseif rule["type"] == "Prefix" then + matchType = "prefix" + end + if key == "headers" then + vsMatch[key][rule["name"]] = {} + vsMatch[key][rule["name"]][matchType] = rule.value + else + vsMatch[key][matchType] = rule.value + end + end + table.insert(route["match"], vsMatch) + end + route.route = { + { + destination = {} + } + } + -- if stableWeight != 0, then add stable service destinations + -- incase there are multiple subsets in stable service + if stableWeight ~= 0 then + local nRoute = {} + if #stableServiceSubsets ~= 0 then + local weight = CalculateWeight(nRoute, stableWeight, #stableServiceSubsets) + for _, r in ipairs(stableServiceSubsets) do + nRoute = { + destination = { + host = stableService, + subset = r + }, + weight = weight + } + table.insert(route.route, nRoute) + end + else + nRoute = { + destination = { + host = stableService + }, + weight = stableWeight + } + table.insert(route.route, nRoute) + end + -- update every matched route + route.route[1].weight = canaryWeight + end + -- if stableService == canaryService, then do e2e release + if stableService == canaryService then + route.route[1].destination.host = stableService + route.route[1].destination.subset = "canary" + else + route.route[1].destination.host = canaryService + end + if (protocol == "http") then + table.insert(spec.http, 1, route) + elseif (protocol == "tls") then + table.insert(spec.tls, 1, route) + elseif (protocol == "tcp") then + table.insert(spec.tcp, 1, route) + end + end +end + +-- generate routes without matches, change every rule +function GenerateRoutes(spec, stableService, canaryService, stableWeight, canaryWeight, protocol) + local matchedRules = FindMatchedRules(spec, stableService, protocol) + for _, rule in ipairs(matchedRules) do + local canary + if stableService ~= canaryService then + canary = { + destination = { + host = canaryService, + }, + weight = canaryWeight, + } + else + canary = { + destination = { + host = stableService, + subset = "canary", + }, + weight = canaryWeight, + } + end + + -- incase there are multiple versions traffic already, do a for-loop + for _, route in ipairs(rule.route) do + -- update stable service weight + route.weight = CalculateWeight(route, stableWeight, #rule.route) + end + table.insert(rule.route, canary) + end +end + +if (obj.matches) then + GenerateMatchedRoutes(spec, obj.matches, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "http") + GenerateMatchedRoutes(spec, obj.matches, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tcp") + GenerateMatchedRoutes(spec, obj.matches, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tls") +else + GenerateRoutes(spec, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "http") + GenerateRoutes(spec, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tcp") + GenerateRoutes(spec, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tls") +end +return obj.data diff --git a/pkg/util/configuration/configuration.go b/pkg/util/configuration/configuration.go index 86282f5c..9bca9a79 100644 --- a/pkg/util/configuration/configuration.go +++ b/pkg/util/configuration/configuration.go @@ -31,6 +31,7 @@ const ( RolloutConfigurationName = "kruise-rollout-configuration" LuaTrafficRoutingIngressTypePrefix = "lua.traffic.routing.ingress" + LuaTrafficRoutingCustomTypePrefix = "lua.traffic.routing" ) func GetTrafficRoutingIngressLuaScript(client client.Client, iType string) (string, error) { diff --git a/pkg/util/luamanager/lua_test.go b/pkg/util/luamanager/lua_test.go index 3926ae8b..1ce2b271 100644 --- a/pkg/util/luamanager/lua_test.go +++ b/pkg/util/luamanager/lua_test.go @@ -30,6 +30,20 @@ import ( gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) +type LuaData struct { + Data Data + CanaryWeight int32 + StableWeight int32 + Matches []rolloutv1alpha1.HttpRouteMatch + CanaryService string + StableService string +} +type Data struct { + Spec interface{} `json:"spec,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` +} + func TestRunLuaScript(t *testing.T) { cases := []struct { name string diff --git a/pkg/webhook/rollout/validating/rollout_create_update_handler.go b/pkg/webhook/rollout/validating/rollout_create_update_handler.go index b9f90b54..bf3fa6a8 100644 --- a/pkg/webhook/rollout/validating/rollout_create_update_handler.go +++ b/pkg/webhook/rollout/validating/rollout_create_update_handler.go @@ -209,8 +209,8 @@ func validateRolloutSpecCanaryTraffic(traffic appsv1alpha1.TrafficRoutingRef, fl errList = append(errList, field.Invalid(fldPath.Child("Service"), traffic.Service, "TrafficRouting.Service cannot be empty")) } - if traffic.Gateway == nil && traffic.Ingress == nil { - errList = append(errList, field.Invalid(fldPath.Child("TrafficRoutings"), traffic.Ingress, "TrafficRoutings must set the gateway or ingress")) + if traffic.Gateway == nil && traffic.Ingress == nil && traffic.CustomNetworkRefs == nil { + errList = append(errList, field.Invalid(fldPath.Child("TrafficRoutings"), traffic.Ingress, "TrafficRoutings are not set")) } if traffic.Ingress != nil {