bugfix success handling in the half-open and add flow control (#1805)

* bugfix success handling in the half-open and add flow control Signed-off-by: hlts2 <[email protected]> * deleted boolean type of isReady function Signed-off-by: hlts2 <[email protected]> * add comment and generate test code Signed-off-by: hlts2 <[email protected]> * Update internal/circuitbreaker/breaker.go * add isReady function teset Signed-off-by: hlts2 <[email protected]> * add success function test Signed-off-by: hlts2 <[email protected]> * fix count value Signed-off-by: hlts2 <[email protected]> * refactor Signed-off-by: hlts2 <[email protected]> Signed-off-by: hlts2 <[email protected]>
vdaas · Oct 14, 2022 · 667b2f4 · 667b2f4
1 parent 7ba6d4f
commit 667b2f4
Show file tree

Hide file tree

Showing 4 changed files with 300 additions and 148 deletions.
diff --git a/internal/circuitbreaker/breaker.go b/internal/circuitbreaker/breaker.go
@@ -68,8 +68,8 @@ func newBreaker(key string, opts ...BreakerOption) (*breaker, error) {
 // do executes the function given argument when the current breaker state is "Closed" or "Half-Open".
 // If the current breaker state is "Open", this function returns ErrCircuitBreakerOpenState.
 func (b *breaker) do(ctx context.Context, fn func(ctx context.Context) (val interface{}, err error)) (val interface{}, st State, err error) {
-	if !b.isReady() {
-		return nil, StateOpen, errors.ErrCircuitBreakerOpenState
+	if st, err := b.isReady(); err != nil {
+		return nil, st, err
 	}
 	val, err = fn(ctx)
 	if err != nil {
@@ -98,14 +98,33 @@ func (b *breaker) do(ctx context.Context, fn func(ctx context.Context) (val inte
 
 // isReady determines the breaker is ready or not.
 // If the current breaker state is "Closed" or "Half-Open", this function returns true.
-func (b *breaker) isReady() (ok bool) {
-	st := b.currentState()
-	return st == StateClosed || st == StateHalfOpen
+func (b *breaker) isReady() (st State, err error) {
+	st = b.currentState()
+	switch st {
+	case StateOpen:
+		return st, errors.ErrCircuitBreakerOpenState
+	case StateHalfOpen:
+
+		// For flow control in the "Half-Open" state. It is limited to 50%.
+		// If this modulo is used, 1/2 of the requests will be error. And if an error occurs, mark as failures.
+		cnt := b.count.Load().(*count)
+		if cnt.Total()%2 == 0 {
+			cnt.onFail()
+			return st, errors.ErrCircuitBreakerHalfOpenFlowLimitation
+		}
+	}
+	return st, nil
 }
 
 func (b *breaker) success() {
-	b.count.Load().(*count).onSuccess()
-	if st := b.currentState(); st == StateHalfOpen {
+	cnt := b.count.Load().(*count)
+	cnt.onSuccess()
+
+	// halfOpenErrShouldTrip.ShouldTrip returns true when the sum of the number of successes and failures is greater than the b.minSamples and when the error rate is greater than the b.halfOpenErrRate.
+	// In other words, if the error rate is less than the b.halfOpenErrRate, it can be judged that the success rate is high, so this function change to the "Close" state from "Half-Open".
+	if st := b.currentState(); st == StateHalfOpen &&
+		cnt.Total() >= b.minSamples &&
+		!b.halfOpenErrShouldTrip.ShouldTrip(cnt) {
 		log.Infof("the operation succeeded, circuit breaker state for '%s' changed,\tfrom: %s, to: %s", b.key, st.String(), StateClosed.String())
 		b.reset()
 	}