[FAB-8900] retry options in ChConfig.Query

- Retry opts added in config under channel policies
- ChConfig Query peer to retry when endorsement mismatch


Change-Id: I32f70b0081a576d86e219b49092d1741f9cfbe4f
Signed-off-by: Sudesh Shetty <[email protected]>

pkg/common/errors/retry/defaults.go

@@ -91,3 +91,8 @@ var ChannelClientRetryableCodes = map[status.Group][]status.Code{
 		status.Code(grpcCodes.Unavailable),
 	},
 }
+
+// ChannelConfigRetryableCodes error codes to be taken into account for query channel config retry
+var ChannelConfigRetryableCodes = map[status.Group][]status.Code{
+	status.EndorserClientStatus: []status.Code{status.EndorsementMismatch},
+}

pkg/common/providers/core/network.go

@@ -7,6 +7,7 @@ SPDX-License-Identifier: Apache-2.0
 package core
 
 import (
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/retry"
 	"github.com/hyperledger/fabric-sdk-go/pkg/core/config/endpoint"
 )
 
@@ -74,7 +75,15 @@ type ChannelConfig struct {
 
 //ChannelPolicies defines list of policies defined for a channel
 type ChannelPolicies struct {
-	QueryChannel map[string]int
+	//Policy for querying channel block
+	QueryChannelConfig QueryChannelConfigPolicy
+}
+
+//QueryChannelConfigPolicy defines opts for channelConfigBlock
+type QueryChannelConfigPolicy struct {
+	MinResponses int
+	MaxTargets   int
+	RetryOpts    retry.Opts
 }
 
 // PeerChannelConfig defines the peer capabilities

pkg/core/config/testdata/config_test.yaml

@@ -166,11 +166,21 @@ channels:
     # config etc.
     policies:
       #[Optional] options for retrieving channel configuration blocks
-      queryChannel:
+      queryChannelConfig:
         #[Optional] min number of success responses (from targets/peers)
         minResponses: 1
         #[Optional] channel config will be retrieved for these number of random targets
         maxTargets: 1
+        #[Optional] retry options for query config block
+        retryOpts:
+          #[Optional] number of retry attempts
+          attempts: 5
+          #[Optional] the back off interval for the first retry attempt
+          initialBackoff: 500ms
+          #[Optional] the maximum back off interval for any retry attempt
+          maxBackoff: 5s
+          #[Optional] he factor by which the initial back off period is exponentially incremented
+          backoffFactor: 2.0
 
   # multi-org test channel
   orgchannel:

pkg/core/config/testdata/config_test_embedded_pems.yaml

@@ -154,11 +154,21 @@ channels:
     # config etc.
     policies:
       #[Optional] options for retrieving channel configuration blocks
-      queryChannel:
+      queryChannelConfig:
         #[Optional] min number of success responses (from targets/peers)
         minResponses: 1
         #[Optional] channel config will be retrieved for these number of random targets
         maxTargets: 1
+        #[Optional] retry options for query config block
+        retryOpts:
+          #[Optional] number of retry attempts
+          attempts: 5
+          #[Optional] the back off interval for the first retry attempt
+          initialBackoff: 500ms
+          #[Optional] the maximum back off interval for any retry attempt
+          maxBackoff: 5s
+          #[Optional] he factor by which the initial back off period is exponentially incremented
+          backoffFactor: 2.0
 
   # multi-org test channel
   orgchannel:

pkg/core/config/testdata/config_test_pem.yaml

@@ -146,11 +146,21 @@ channels:
     # config etc.
     policies:
       #[Optional] options for retrieving channel configuration blocks
-      queryChannel:
+      queryChannelConfig:
         #[Optional] min number of success responses (from targets/peers)
         minResponses: 1
         #[Optional] channel config will be retrieved for these number of random targets
         maxTargets: 1
+        #[Optional] retry options for query config block
+        retryOpts:
+          #[Optional] number of retry attempts
+          attempts: 5
+          #[Optional] the back off interval for the first retry attempt
+          initialBackoff: 500ms
+          #[Optional] the maximum back off interval for any retry attempt
+          maxBackoff: 5s
+          #[Optional] he factor by which the initial back off period is exponentially incremented
+          backoffFactor: 2.0
 
   # multi-org test channel
   orgchannel:

pkg/core/config/testdata/template/config.yaml

@@ -147,11 +147,21 @@ channels:
     # config etc.
 #    policies:
        #[Optional] options for retrieving channel configuration blocks
-#      queryChannel:
+#      queryChannelConfig:
          #[Optional] min number of success responses (from targets/peers)
 #        minResponses: 1
          #[Optional] channel config will be retrieved for these number of random targets
 #        maxTargets: 1
+        #[Optional] retry options for query config block
+#        retryOpts:
+          #[Optional] number of retry attempts
+#          attempts: 5
+          #[Optional] the back off interval for the first retry attempt
+#          initialBackoff: 500ms
+          #[Optional] the maximum back off interval for any retry attempt
+#          maxBackoff: 5s
+          #[Optional] he factor by which the initial back off is exponentially incremented
+#          backoffFactor: 2.0
 
 #
 # list of participating organizations in this network

pkg/fab/chconfig/chconfig.go

@@ -26,15 +26,20 @@ import (
 	channelConfig "github.com/hyperledger/fabric-sdk-go/internal/github.com/hyperledger/fabric/common/channelconfig"
 
 	imsp "github.com/hyperledger/fabric-sdk-go/internal/github.com/hyperledger/fabric/msp"
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/multi"
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/retry"
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/providers/context"
 	contextImpl "github.com/hyperledger/fabric-sdk-go/pkg/context"
 )
 
 var logger = logging.NewLogger("fabsdk/fab")
 
+//overrideRetryHandler is private and used for unit-tests to test query retry behaviors
+var overrideRetryHandler retry.Handler
+
 const (
 	defaultMinResponses = 1
-	defaultMaxTargets   = 1
+	defaultMaxTargets   = 2
 )
 
 // Opts contains options for retrieving channel configuration
@@ -43,6 +48,7 @@ type Opts struct {
 	Targets      []fab.Peer  // if configured, channel config will be retrieved from peers (targets)
 	MinResponses int         // used with targets option; min number of success responses (from targets/peers)
 	MaxTargets   int         //if configured, channel config will be retrieved for these number of random targets
+	RetryOpts    retry.Opts  //opts for channel query retry handler
 }
 
 // Option func for each Opts argument
@@ -138,10 +144,12 @@ func (c *ChannelConfig) queryPeers(reqCtx reqContext.Context) (*ChannelCfg, erro
 		return nil, errors.WithMessage(err, "ledger client creation failed")
 	}
 
+	if err = c.resolveOptsFromConfig(ctx); err != nil {
+		return nil, errors.WithMessage(err, "failed to resolve opts from config")
+	}
+
 	targets := []fab.ProposalProcessor{}
-	maxTargets, minResponses := c.getLimitOpts(ctx)
 	if c.opts.Targets == nil {
-
 		// Calculate targets from config
 		chPeers, err := ctx.Config().ChannelPeers(c.channelID)
 		if err != nil {
@@ -157,18 +165,31 @@ func (c *ChannelConfig) queryPeers(reqCtx reqContext.Context) (*ChannelCfg, erro
 			targets = append(targets, newPeer)
 		}
 
-		targets = randomMaxTargets(targets, maxTargets)
+		targets = randomMaxTargets(targets, c.opts.MaxTargets)
 
 	} else {
 		targets = peersToTxnProcessors(c.opts.Targets)
 	}
 
-	block, err := l.QueryConfigBlock(reqCtx, targets, &channel.TransactionProposalResponseVerifier{MinResponses: minResponses})
+	retryHandler := retry.New(c.opts.RetryOpts)
+
+	//Unit test purpose only
+	if overrideRetryHandler != nil {
+		retryHandler = overrideRetryHandler
+	}
+
+queryConfigBlock:
+	//Perform QueryConfigBlock
+	block, err := l.QueryConfigBlock(reqCtx, targets, &channel.TransactionProposalResponseVerifier{MinResponses: c.opts.MinResponses})
+	if c.resolveRetry(retryHandler, err) {
+		goto queryConfigBlock
+	}
+
 	if err != nil {
 		return nil, errors.WithMessage(err, "QueryBlockConfig failed")
 	}
-
 	return extractConfig(c.channelID, block)
+
 }
 
 func (c *ChannelConfig) queryOrderer(reqCtx reqContext.Context) (*ChannelCfg, error) {
@@ -181,35 +202,71 @@ func (c *ChannelConfig) queryOrderer(reqCtx reqContext.Context) (*ChannelCfg, er
 	return extractConfig(c.channelID, block)
 }
 
-func (c *ChannelConfig) getLimitOpts(ctx context.Client) (int, int) {
+//resolveOptsFromConfig loads opts from config if not loaded/initialized
+func (c *ChannelConfig) resolveOptsFromConfig(ctx context.Client) error {
 
-	//Opts takes high priority, if provided in opts then it should be taken into account
-	if c.opts.MaxTargets > 0 && c.opts.MinResponses > 0 {
-		return c.opts.MaxTargets, c.opts.MinResponses
+	if c.opts.MaxTargets != 0 && c.opts.MinResponses != 0 && c.opts.RetryOpts.RetryableCodes != nil {
+		//already loaded
+		return nil
 	}
 
 	//If missing from opts, check config and update opts from config
 	chSdkCfg, err := ctx.Config().ChannelConfig(c.channelID)
 	if err != nil {
 		//ver rare, but return default in case of error
-		return defaultMaxTargets, defaultMinResponses
+		return err
 	}
 
 	if c.opts.MaxTargets == 0 {
-		c.opts.MaxTargets = chSdkCfg.Policies.QueryChannel["maxTargets"]
+		c.opts.MaxTargets = chSdkCfg.Policies.QueryChannelConfig.MaxTargets
 		if c.opts.MaxTargets == 0 {
 			c.opts.MaxTargets = defaultMaxTargets
 		}
 	}
 
 	if c.opts.MinResponses == 0 {
-		c.opts.MinResponses = chSdkCfg.Policies.QueryChannel["minResponses"]
+		c.opts.MinResponses = chSdkCfg.Policies.QueryChannelConfig.MinResponses
 		if c.opts.MinResponses == 0 {
 			c.opts.MinResponses = defaultMinResponses
 		}
 	}
 
-	return c.opts.MaxTargets, c.opts.MinResponses
+	if c.opts.RetryOpts.RetryableCodes == nil {
+		c.opts.RetryOpts = chSdkCfg.Policies.QueryChannelConfig.RetryOpts
+
+		if c.opts.RetryOpts.Attempts == 0 {
+			c.opts.RetryOpts.Attempts = retry.DefaultAttempts
+		}
+
+		if c.opts.RetryOpts.InitialBackoff == 0 {
+			c.opts.RetryOpts.InitialBackoff = retry.DefaultInitialBackoff
+		}
+
+		if c.opts.RetryOpts.BackoffFactor == 0 {
+			c.opts.RetryOpts.BackoffFactor = retry.DefaultBackoffFactor
+		}
+
+		if c.opts.RetryOpts.MaxBackoff == 0 {
+			c.opts.RetryOpts.MaxBackoff = retry.DefaultMaxBackoff
+		}
+
+		c.opts.RetryOpts.RetryableCodes = retry.ChannelConfigRetryableCodes
+	}
+
+	return nil
+}
+
+func (c *ChannelConfig) resolveRetry(handler retry.Handler, err error) bool {
+	errs, ok := err.(multi.Errors)
+	if !ok {
+		errs = append(errs, err)
+	}
+	for _, e := range errs {
+		if handler.Required(e) {
+			return true
+		}
+	}
+	return false
 }
 
 // WithPeers encapsulates peers to Option
@@ -244,6 +301,14 @@ func WithMaxTargets(maxTargets int) Option {
 	}
 }
 
+// WithRetryOpts encapsulates retry opts to Option
+func WithRetryOpts(retryOpts retry.Opts) Option {
+	return func(opts *Opts) error {
+		opts.RetryOpts = retryOpts
+		return nil
+	}
+}
+
 // prepareQueryConfigOpts Reads channel config options from Option array
 func prepareOpts(options ...Option) (Opts, error) {
 	opts := Opts{}