From eee6f64aa66510b0edf05d2b12c94b5583925e17 Mon Sep 17 00:00:00 2001 From: Hasit Mistry Date: Wed, 7 Sep 2022 00:14:50 -0700 Subject: [PATCH] Fixe ratelimiter decision sync (#349) Co-authored-by: Harjot Gill --- api/aperture/policy/language/v1/policy.proto | 4 +- api/gen/openapiv2/aperture.swagger.yaml | 2 +- .../aperture/policy/language/v1/policy.pb.go | 4 +- docs/gen/policies/gen.yaml | 2 +- docs/gen/policies/policy.md | 2 +- go.mod | 2 +- pkg/config/koanf-unmarshaller.go | 43 ++++++++---------- pkg/flowcontrol/common/flowcontrol.go | 2 +- .../components/actuators/rate/rate-limiter.go | 19 ++++---- .../concurrency/concurrency-limiter.go | 19 ++++---- .../dataplane/actuators/rate/rate-limiter.go | 45 +++++++++---------- pkg/policies/dataplane/engine.go | 38 +++++++++++----- pkg/policies/dataplane/iface/limiter.go | 2 +- pkg/policies/mocks/mock_limiter.go | 10 ++--- 14 files changed, 101 insertions(+), 93 deletions(-) diff --git a/api/aperture/policy/language/v1/policy.proto b/api/aperture/policy/language/v1/policy.proto index 6ad0459af3..76cbea0140 100644 --- a/api/aperture/policy/language/v1/policy.proto +++ b/api/aperture/policy/language/v1/policy.proto @@ -514,10 +514,10 @@ message RateLimiter { extensions: { key: "x-go-default" value: { - bool_value: true + bool_value: false } } - }]; // @gotags: default:"true" + }]; // @gotags: default:"false" // Number of times to lazy sync within the _limit\_reset\_interval_. uint32 num_sync = 2 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { diff --git a/api/gen/openapiv2/aperture.swagger.yaml b/api/gen/openapiv2/aperture.swagger.yaml index 0156441850..f916ffb512 100644 --- a/api/gen/openapiv2/aperture.swagger.yaml +++ b/api/gen/openapiv2/aperture.swagger.yaml @@ -190,7 +190,7 @@ definitions: type: boolean description: TODO document what happens when lazy sync is disabled title: Enables lazy sync - x-go-default: true + x-go-default: false num_sync: type: integer format: int64 diff --git a/api/gen/proto/go/aperture/policy/language/v1/policy.pb.go b/api/gen/proto/go/aperture/policy/language/v1/policy.pb.go index e69d18f505..e59315d54b 100644 --- a/api/gen/proto/go/aperture/policy/language/v1/policy.pb.go +++ b/api/gen/proto/go/aperture/policy/language/v1/policy.pb.go @@ -2347,7 +2347,7 @@ type RateLimiter_LazySync struct { // Enables lazy sync // // TODO document what happens when lazy sync is disabled - Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty" default:"true"` // @gotags: default:"true" + Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty" default:"false"` // @gotags: default:"false" // Number of times to lazy sync within the _limit\_reset\_interval_. NumSync uint32 `protobuf:"varint,2,opt,name=num_sync,json=numSync,proto3" json:"num_sync,omitempty" default:"5" validate:"gt=0"` // @gotags: default:"5" validate:"gt=0" } @@ -3616,7 +3616,7 @@ var file_aperture_policy_language_v1_policy_proto_rawDesc = []byte{ 0x0a, 0x08, 0x4c, 0x61, 0x7a, 0x79, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x32, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x42, 0x18, 0x92, 0x41, 0x15, 0x82, 0x03, 0x12, 0x0a, 0x0c, 0x78, 0x2d, 0x67, 0x6f, 0x2d, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, - 0x74, 0x12, 0x02, 0x20, 0x01, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x54, + 0x74, 0x12, 0x02, 0x20, 0x00, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x54, 0x0a, 0x08, 0x6e, 0x75, 0x6d, 0x5f, 0x73, 0x79, 0x6e, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x42, 0x39, 0x92, 0x41, 0x36, 0x82, 0x03, 0x19, 0x0a, 0x0c, 0x78, 0x2d, 0x67, 0x6f, 0x2d, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x12, 0x09, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, diff --git a/docs/gen/policies/gen.yaml b/docs/gen/policies/gen.yaml index e8a3da8ebd..1040d9bc59 100644 --- a/docs/gen/policies/gen.yaml +++ b/docs/gen/policies/gen.yaml @@ -41,7 +41,7 @@ definitions: description: TODO document what happens when lazy sync is disabled type: boolean title: Enables lazy sync - x-go-default: true + x-go-default: false x-order: 0 num_sync: description: Number of times to lazy sync within the _limit\_reset\_interval_. diff --git a/docs/gen/policies/policy.md b/docs/gen/policies/policy.md index cb0b764ef6..193cee3afd 100644 --- a/docs/gen/policies/policy.md +++ b/docs/gen/policies/policy.md @@ -123,7 +123,7 @@ eg. {any: {of: [expr1, expr2]}}.
enabled
-(bool, default: `true`) Enables lazy sync +(bool) Enables lazy sync TODO document what happens when lazy sync is disabled diff --git a/go.mod b/go.mod index 35797754e4..ddc9f5cec2 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Henry-Sarabia/sliceconv v1.0.2 github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1 github.com/buger/jsonparser v1.1.1 - github.com/buraksezer/olric v0.0.0 + github.com/buraksezer/olric v0.4.5 github.com/cenkalti/backoff v2.2.1+incompatible github.com/cenkalti/backoff/v4 v4.1.3 github.com/clarketm/json v1.17.1 diff --git a/pkg/config/koanf-unmarshaller.go b/pkg/config/koanf-unmarshaller.go index 3918a19ebf..cc7162fa94 100644 --- a/pkg/config/koanf-unmarshaller.go +++ b/pkg/config/koanf-unmarshaller.go @@ -411,42 +411,36 @@ func jsonOverrideHookFunc(replaceSlice bool) mapstructure.DecodeHookFunc { } func merge(a, b map[string]interface{}, replaceSlice bool) { + log.Trace().Bool("ReplaceSlice", replaceSlice).Interface("a", a).Interface("b", b).Msg("MERGE") for key, val := range a { // Does the key exist in the target map? // If no, add it and move on. bVal, ok := b[key] if !ok { - log.Trace().Str("key", key).Msg("override") + log.Trace().Str("key", key).Interface("Value", val).Msg("override") b[key] = val continue } if !replaceSlice { + // merge slice B into slice A values if sliceA, ok := val.([]interface{}); ok { - if sliceB, ok := val.([]interface{}); ok { - sliceMapA := []map[string]interface{}{} - sliceMapB := []map[string]interface{}{} - for _, v := range sliceA { + if sliceB, ok := bVal.([]interface{}); ok { + // iterate slices and merge their map[string]interface{} + for i, v := range sliceA { if m, ok := v.(map[string]interface{}); ok { - sliceMapA = append(sliceMapA, m) - } - } - for _, v := range sliceB { - if m, ok := v.(map[string]interface{}); ok { - sliceMapB = append(sliceMapB, m) - } - } - for i, sliceValA := range sliceMapA { - if i < len(sliceB) { - log.Trace().Str("key", key).Int("index", i).Msg("merging") - // merge - merge(sliceValA, sliceMapB[i], replaceSlice) - } else { - log.Trace().Str("key", key).Int("index", i).Msg("overriding") - // append - sliceMapB = append(sliceMapB, sliceValA) + if len(sliceB) > i { + if m2, ok := sliceB[i].(map[string]interface{}); ok { + merge(m, m2, replaceSlice) + } else { + log.Warn().Str("key", key).Interface("Value", val).Msg("unable to merge slice") + } + } else { + sliceB = append(sliceB, m) + } } } + b[key] = sliceB continue } } @@ -454,8 +448,8 @@ func merge(a, b map[string]interface{}, replaceSlice bool) { // If the incoming val is not a map, do a direct merge. if _, ok := val.(map[string]interface{}); !ok { - log.Trace().Str("key", key).Msg("override") b[key] = val + log.Trace().Str("key", key).Interface("Value", val).Interface("b[key]", b[key]).Msg("override") continue } @@ -465,10 +459,11 @@ func merge(a, b map[string]interface{}, replaceSlice bool) { log.Trace().Str("key", key).Msg("merge") merge(val.(map[string]interface{}), v, replaceSlice) default: - log.Trace().Str("key", key).Msg("override") + log.Trace().Str("key", key).Interface("Value", val).Msg("override") b[key] = val } } + log.Trace().Bool("ReplaceSlice", replaceSlice).Interface("a", a).Interface("b", b).Msg("MERGE DONE") } var json = jsoniter.Config{ diff --git a/pkg/flowcontrol/common/flowcontrol.go b/pkg/flowcontrol/common/flowcontrol.go index d4186ff92e..429364c743 100644 --- a/pkg/flowcontrol/common/flowcontrol.go +++ b/pkg/flowcontrol/common/flowcontrol.go @@ -54,7 +54,7 @@ func (h *Handler) CheckWithValues( serviceIDs []services.ServiceID, labels selectors.Labels, ) *flowcontrolv1.CheckResponse { - log.Trace().Msg("FlowControl.CheckWithValues()") + log.Trace().Interface("labels", labels.ToPlainMap()).Interface("serviceIDs", serviceIDs).Str("controlPoint", controlPoint.String()).Msg("FlowControl.CheckWithValues()") checkResponse := h.engine.ProcessRequest(controlPoint, serviceIDs, labels) h.metrics.CheckResponse(checkResponse.DecisionType, checkResponse.GetDecisionReason()) diff --git a/pkg/policies/controlplane/components/actuators/rate/rate-limiter.go b/pkg/policies/controlplane/components/actuators/rate/rate-limiter.go index 1dce419234..3f10b9faa9 100644 --- a/pkg/policies/controlplane/components/actuators/rate/rate-limiter.go +++ b/pkg/policies/controlplane/components/actuators/rate/rate-limiter.go @@ -5,6 +5,11 @@ import ( "errors" "path" + clientv3 "go.etcd.io/etcd/client/v3" + "go.uber.org/fx" + "go.uber.org/multierr" + "google.golang.org/protobuf/proto" + configv1 "github.com/fluxninja/aperture/api/gen/proto/go/aperture/common/config/v1" policydecisionsv1 "github.com/fluxninja/aperture/api/gen/proto/go/aperture/policy/decisions/v1" policylangv1 "github.com/fluxninja/aperture/api/gen/proto/go/aperture/policy/language/v1" @@ -14,10 +19,6 @@ import ( "github.com/fluxninja/aperture/pkg/paths" "github.com/fluxninja/aperture/pkg/policies/controlplane/iface" "github.com/fluxninja/aperture/pkg/policies/controlplane/runtime" - clientv3 "go.etcd.io/etcd/client/v3" - "go.uber.org/fx" - "go.uber.org/multierr" - "google.golang.org/protobuf/proto" ) type rateLimiterSync struct { @@ -133,11 +134,11 @@ func (limiterSync *rateLimiterSync) publishLimit(limitValue float64) error { limiterSync.decision.Limit = limitValue // Publish decision log.Debug().Float64("limit", limitValue).Msg("publishing rate limiter decision") - wrapper := &configv1.RateLimiterWrapper{ - RateLimiter: limiterSync.rateLimiterProto, - ComponentIndex: int64(limiterSync.componentIndex), - PolicyName: limiterSync.policyReadAPI.GetPolicyName(), - PolicyHash: limiterSync.policyReadAPI.GetPolicyHash(), + wrapper := &configv1.RateLimiterDecisionWrapper{ + RateLimiterDecision: limiterSync.decision, + ComponentIndex: int64(limiterSync.componentIndex), + PolicyName: limiterSync.policyReadAPI.GetPolicyName(), + PolicyHash: limiterSync.policyReadAPI.GetPolicyHash(), } dat, err := proto.Marshal(wrapper) if err != nil { diff --git a/pkg/policies/dataplane/actuators/concurrency/concurrency-limiter.go b/pkg/policies/dataplane/actuators/concurrency/concurrency-limiter.go index 18b9664dc2..e6e6ac946d 100644 --- a/pkg/policies/dataplane/actuators/concurrency/concurrency-limiter.go +++ b/pkg/policies/dataplane/actuators/concurrency/concurrency-limiter.go @@ -425,7 +425,7 @@ func (conLimiter *concurrencyLimiter) GetSelector() *selectorv1.Selector { } // RunLimiter . -func (conLimiter *concurrencyLimiter) RunLimiter(labels selectors.Labels) *flowcontrolv1.LimiterDecision { +func (conLimiter *concurrencyLimiter) RunLimiter(labels selectors.Labels, decision *flowcontrolv1.LimiterDecision) { var matchedWorkloadProto *policylangv1.Scheduler_Workload var matchedWorkloadIndex string // match labels against conLimiter.workloadMultiMatcher @@ -489,16 +489,13 @@ func (conLimiter *concurrencyLimiter) RunLimiter(labels selectors.Labels) *flowc if accepted { conLimiter.acceptedConcurrencyCounter.Add(float64(reqContext.Tokens)) } - - return &flowcontrolv1.LimiterDecision{ - PolicyName: conLimiter.GetPolicyName(), - PolicyHash: conLimiter.GetPolicyHash(), - ComponentIndex: conLimiter.GetComponentIndex(), - Dropped: !accepted, - Details: &flowcontrolv1.LimiterDecision_ConcurrencyLimiter_{ - ConcurrencyLimiter: &flowcontrolv1.LimiterDecision_ConcurrencyLimiter{ - WorkloadIndex: matchedWorkloadIndex, - }, + decision.PolicyName = conLimiter.GetPolicyName() + decision.PolicyHash = conLimiter.GetPolicyHash() + decision.ComponentIndex = conLimiter.GetComponentIndex() + decision.Dropped = !accepted + decision.Details = &flowcontrolv1.LimiterDecision_ConcurrencyLimiter_{ + ConcurrencyLimiter: &flowcontrolv1.LimiterDecision_ConcurrencyLimiter{ + WorkloadIndex: matchedWorkloadIndex, }, } } diff --git a/pkg/policies/dataplane/actuators/rate/rate-limiter.go b/pkg/policies/dataplane/actuators/rate/rate-limiter.go index 69dc1c8f0e..b54e67ac0b 100644 --- a/pkg/policies/dataplane/actuators/rate/rate-limiter.go +++ b/pkg/policies/dataplane/actuators/rate/rate-limiter.go @@ -85,8 +85,8 @@ func setupRateLimiterFactory( ai *agentinfo.AgentInfo, ) error { agentGroupName := ai.GetAgentGroup() - rateLimitDecisionsWatcher, err := etcdwatcher.NewWatcher(etcdClient, - path.Join(paths.RateLimiterDecisionsPath, paths.AgentGroupPrefix(agentGroupName))) + etcdPath := path.Join(paths.RateLimiterDecisionsPath) + rateLimitDecisionsWatcher, err := etcdwatcher.NewWatcher(etcdClient, etcdPath) if err != nil { return err } @@ -109,7 +109,9 @@ func setupRateLimiterFactory( } fxDriver := ¬ifiers.FxDriver{ - FxOptionsFuncs: []notifiers.FxOptionsFunc{rateLimiterFactory.newRateLimiterOptions}, + FxOptionsFuncs: []notifiers.FxOptionsFunc{ + rateLimiterFactory.newRateLimiterOptions, + }, UnmarshalPrefixNotifier: notifiers.UnmarshalPrefixNotifier{ GetUnmarshallerFunc: config.NewProtobufUnmarshaller, }, @@ -162,11 +164,11 @@ func (rateLimiterFactory *rateLimiterFactory) newRateLimiterOptions( return fx.Options(), err } - rateLimiterMessage := wrapperMessage.RateLimiter + rateLimiterProto := wrapperMessage.RateLimiter rateLimiter := &rateLimiter{ Component: wrapperMessage, - rateLimiterProto: rateLimiterMessage, + rateLimiterProto: rateLimiterProto, rateLimiterFactory: rateLimiterFactory, statusRegistry: reg, } @@ -199,10 +201,9 @@ func (rateLimiter *rateLimiter) setup(lifecycle fx.Lifecycle) error { if err != nil { return err } + decisionKey := paths.DataplaneComponentKey(rateLimiter.rateLimiterFactory.agentGroupName, rateLimiter.GetPolicyName(), rateLimiter.GetComponentIndex()) decisionNotifier := notifiers.NewUnmarshalKeyNotifier( - notifiers.Key(paths.DataplaneComponentKey(rateLimiter.rateLimiterFactory.agentGroupName, - rateLimiter.GetPolicyName(), - rateLimiter.GetComponentIndex())), + notifiers.Key(decisionKey), unmarshaller, rateLimiter.decisionUpdateCallback, ) @@ -216,7 +217,8 @@ func (rateLimiter *rateLimiter) setup(lifecycle fx.Lifecycle) error { label := rateLimiter.rateLimiterProto.GetLabelKey() + ":" + override.GetLabelValue() rateLimiter.rateLimitChecker.AddOverride(label, override.GetLimitScaleFactor()) } - rateLimiter.rateTracker, err = ratetracker.NewDistCacheRateTracker(rateLimiter.rateLimitChecker, + rateLimiter.rateTracker, err = ratetracker.NewDistCacheRateTracker( + rateLimiter.rateLimitChecker, rateLimiter.rateLimiterFactory.distCache, rateLimiter.name, rateLimiter.rateLimiterProto.GetLimitResetInterval().AsDuration()) @@ -282,7 +284,7 @@ func (rateLimiter *rateLimiter) GetSelector() *selectorv1.Selector { } // RunLimiter runs the limiter. -func (rateLimiter *rateLimiter) RunLimiter(labels selectors.Labels) *flowcontrolv1.LimiterDecision { +func (rateLimiter *rateLimiter) RunLimiter(labels selectors.Labels, decision *flowcontrolv1.LimiterDecision) { reason := flowcontrolv1.LimiterDecision_LIMITER_REASON_UNSPECIFIED label, ok, remaining, current := rateLimiter.TakeN(labels, 1) @@ -291,18 +293,16 @@ func (rateLimiter *rateLimiter) RunLimiter(labels selectors.Labels) *flowcontrol reason = flowcontrolv1.LimiterDecision_LIMITER_REASON_KEY_NOT_FOUND } - return &flowcontrolv1.LimiterDecision{ - PolicyName: rateLimiter.GetPolicyName(), - PolicyHash: rateLimiter.GetPolicyHash(), - ComponentIndex: rateLimiter.GetComponentIndex(), - Dropped: !ok, - Reason: reason, - Details: &flowcontrolv1.LimiterDecision_RateLimiter_{ - RateLimiter: &flowcontrolv1.LimiterDecision_RateLimiter{ - Label: label, - Remaining: int64(remaining), - Current: int64(current), - }, + decision.PolicyName = rateLimiter.GetPolicyName() + decision.PolicyHash = rateLimiter.GetPolicyHash() + decision.ComponentIndex = rateLimiter.GetComponentIndex() + decision.Dropped = !ok + decision.Reason = reason + decision.Details = &flowcontrolv1.LimiterDecision_RateLimiter_{ + RateLimiter: &flowcontrolv1.LimiterDecision_RateLimiter{ + Label: label, + Remaining: int64(remaining), + Current: int64(current), }, } } @@ -320,7 +320,6 @@ func (rateLimiter *rateLimiter) TakeN(labels selectors.Labels, n int) (label str label = labelKey + ":" + labelValue ok, remaining, current = rateLimiter.rateTracker.TakeN(label, n) - return } diff --git a/pkg/policies/dataplane/engine.go b/pkg/policies/dataplane/engine.go index 8f6dc788cf..908fe99c01 100644 --- a/pkg/policies/dataplane/engine.go +++ b/pkg/policies/dataplane/engine.go @@ -99,7 +99,7 @@ func (e *Engine) ProcessRequest(controlPoint selectors.ControlPoint, serviceIDs return } - // execute rate limiters first + // execute concurrency limiters concurrencyLimiters := make([]iface.Limiter, len(mmr.concurrencyLimiters)) copy(concurrencyLimiters, mmr.concurrencyLimiters) @@ -118,21 +118,39 @@ func (e *Engine) ProcessRequest(controlPoint selectors.ControlPoint, serviceIDs } func runLimiters(limiters []iface.Limiter, labels selectors.Labels) ([]*flowcontrolv1.LimiterDecision, flowcontrolv1.DecisionType) { - decisionType := flowcontrolv1.DecisionType_DECISION_TYPE_ACCEPTED var wg sync.WaitGroup - limiterDecisions := make([]*flowcontrolv1.LimiterDecision, len(limiters)) - for i, limiter := range limiters { - wg.Add(1) - panichandler.Go(func() { + var once sync.Once + + decisionType := flowcontrolv1.DecisionType_DECISION_TYPE_ACCEPTED + + setDecisionRejected := func() { + decisionType = flowcontrolv1.DecisionType_DECISION_TYPE_REJECTED + } + + execLimiter := func(limiter iface.Limiter, decision *flowcontrolv1.LimiterDecision) func() { + return func() { defer wg.Done() - decision := limiter.RunLimiter(labels) + limiter.RunLimiter(labels, decision) if decision.Dropped { - decisionType = flowcontrolv1.DecisionType_DECISION_TYPE_REJECTED + once.Do(setDecisionRejected) } - limiterDecisions[i] = decision - }) + } + } + + limiterDecisions := make([]*flowcontrolv1.LimiterDecision, len(limiters)) + // execute limiters + for i, limiter := range limiters { + wg.Add(1) + decision := &flowcontrolv1.LimiterDecision{} + limiterDecisions[i] = decision + if i == len(limiters)-1 { + execLimiter(limiter, decision)() + } else { + panichandler.Go(execLimiter(limiter, decision)) + } } wg.Wait() + return limiterDecisions, decisionType } diff --git a/pkg/policies/dataplane/iface/limiter.go b/pkg/policies/dataplane/iface/limiter.go index bae84e4cf3..a660476f98 100644 --- a/pkg/policies/dataplane/iface/limiter.go +++ b/pkg/policies/dataplane/iface/limiter.go @@ -27,6 +27,6 @@ func (limiterID LimiterID) String() string { type Limiter interface { GetPolicyName() string GetSelector() *selectorv1.Selector - RunLimiter(labels selectors.Labels) *flowcontrolv1.LimiterDecision + RunLimiter(labels selectors.Labels, decision *flowcontrolv1.LimiterDecision) GetLimiterID() LimiterID } diff --git a/pkg/policies/mocks/mock_limiter.go b/pkg/policies/mocks/mock_limiter.go index 292c3c88e2..1ccc60b50d 100644 --- a/pkg/policies/mocks/mock_limiter.go +++ b/pkg/policies/mocks/mock_limiter.go @@ -80,15 +80,13 @@ func (mr *MockLimiterMockRecorder) GetSelector() *gomock.Call { } // RunLimiter mocks base method. -func (m *MockLimiter) RunLimiter(labels selectors.Labels) *flowcontrolv1.LimiterDecision { +func (m *MockLimiter) RunLimiter(labels selectors.Labels, decision *flowcontrolv1.LimiterDecision) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RunLimiter", labels) - ret0, _ := ret[0].(*flowcontrolv1.LimiterDecision) - return ret0 + m.ctrl.Call(m, "RunLimiter", labels, decision) } // RunLimiter indicates an expected call of RunLimiter. -func (mr *MockLimiterMockRecorder) RunLimiter(labels interface{}) *gomock.Call { +func (mr *MockLimiterMockRecorder) RunLimiter(labels, decision interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunLimiter", reflect.TypeOf((*MockLimiter)(nil).RunLimiter), labels) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunLimiter", reflect.TypeOf((*MockLimiter)(nil).RunLimiter), labels, decision) }