[FAB-10944] Use Invoker for int test retries

This change removes explicit Sleeps from the integration tests. Retry polling is replaced with the Invoker. Change-Id: I0283fc4c5525ebd1e7d5c38bbbbe5a8ff6a5c1e7 Signed-off-by: Troy Ronda <[email protected]>
hyperledger · Jun 29, 2018 · 632e832 · 632e832
1 parent 5f82d04
commit 632e832
Show file tree

Hide file tree

Showing 10 changed files with 199 additions and 159 deletions.
diff --git a/pkg/common/errors/retry/defaults.go b/pkg/common/errors/retry/defaults.go
@@ -164,3 +164,30 @@ var ChannelClientRetryableCodes = map[status.Group][]status.Code{
 var ChannelConfigRetryableCodes = map[status.Group][]status.Code{
 	status.EndorserClientStatus: {status.EndorsementMismatch},
 }
+
+// TestRetryableCodes are used by tests to determine error situations that can be retried.
+var TestRetryableCodes = map[status.Group][]status.Code{
+	status.TestStatus: {
+		status.GenericTransient,
+	},
+}
+
+const (
+	// TestAttempts number of retry attempts made by default
+	TestAttempts = 10
+	// TestInitialBackoff default initial backoff
+	TestInitialBackoff = 200 * time.Millisecond
+	// TestMaxBackoff default maximum backoff
+	TestMaxBackoff = 5 * time.Second
+	// TestBackoffFactor default backoff factor
+	TestBackoffFactor = 1.75
+)
+
+// TestRetryOpts are used by tests to determine retry parameters.
+var TestRetryOpts = Opts{
+	Attempts:       TestAttempts,
+	InitialBackoff: TestInitialBackoff,
+	MaxBackoff:     TestMaxBackoff,
+	BackoffFactor:  TestBackoffFactor,
+	RetryableCodes: TestRetryableCodes,
+}
diff --git a/pkg/common/errors/status/codes.go b/pkg/common/errors/status/codes.go
@@ -45,7 +45,7 @@ const (
 	// SignatureVerificationFailed is when signature fails verification
 	SignatureVerificationFailed Code = 8
 
-	// MissingEndorsement is if an endoresement is missing
+	// MissingEndorsement is if an endorsement is missing
 	MissingEndorsement Code = 9
 
 	// QueryEndorsers error indicates that no endorser group was found that would
@@ -58,6 +58,9 @@ const (
 
 	// ChaincodeAlreadyLaunching indicates that an attempt for multiple simultaneous invokes was made to launch chaincode
 	ChaincodeAlreadyLaunching Code = 22
+
+	// GenericTransient is generally used by tests to indicate that a retry is possible
+	GenericTransient Code = 12
 )
 
 // CodeName maps the codes in this packages to human-readable strings
@@ -74,6 +77,7 @@ var CodeName = map[int32]string{
 	9:  "MISSING_ENDORSEMENT",
 	10: "CHAINCODE_ERROR",
 	11: "QUERY_ENDORSERS",
+	12: "GENERIC_TRANSIENT",
 	21: "NO_MATCHING_CERTIFICATE_AUTHORITY_ENTITY",
 	22: "NO_MATCHING_PEER_ENTITY",
 	23: "NO_MATCHING_ORDERER_ENTITY",

diff --git a/pkg/common/errors/status/status.go b/pkg/common/errors/status/status.go
@@ -81,6 +81,9 @@ const (
 
 	// DiscoveryServerStatus status returned by the Discovery Server
 	DiscoveryServerStatus
+
+	// TestStatus is used by tests to create retry codes.
+	TestStatus
 )
 
 // GroupName maps the groups in this packages to human-readable strings
@@ -97,6 +100,7 @@ var GroupName = map[int32]string{
 	9:  "Client Status",
 	10: "Chaincode status",
 	11: "Discovery status",
+	12: "Test status",
 }
 
 func (g Group) String() string {

diff --git a/test/integration/base_test_setup.go b/test/integration/base_test_setup.go
@@ -7,14 +7,15 @@ SPDX-License-Identifier: Apache-2.0
 package integration
 
 import (
+	"fmt"
 	"os"
 	"path"
 	"testing"
-	"time"
 
 	mspclient "github.com/hyperledger/fabric-sdk-go/pkg/client/msp"
 	"github.com/hyperledger/fabric-sdk-go/pkg/client/resmgmt"
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/retry"
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/status"
 	contextAPI "github.com/hyperledger/fabric-sdk-go/pkg/common/providers/context"
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/providers/core"
 	fabAPI "github.com/hyperledger/fabric-sdk-go/pkg/common/providers/fab"
@@ -30,6 +31,7 @@ import (
 	"github.com/hyperledger/fabric-sdk-go/third_party/github.com/hyperledger/fabric/common/cauthdsl"
 	cb "github.com/hyperledger/fabric-sdk-go/third_party/github.com/hyperledger/fabric/protos/common"
 	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
 )
 
 // BaseSetupImpl implementation of BaseTestSetup
@@ -309,22 +311,23 @@ func DiscoverLocalPeers(ctxProvider contextAPI.ClientProvider, expectedPeers int
 		return nil, errors.Wrap(err, "error creating local context")
 	}
 
-	var peers []fabAPI.Peer
-	for i := 0; i < 10; i++ {
-		peers, err = ctx.LocalDiscoveryService().GetPeers()
-		if err != nil {
-			return nil, errors.Wrapf(err, "error getting peers for MSP [%s]", ctx.Identifier().MSPID)
-		}
-		if len(peers) >= expectedPeers {
-			break
-		}
-		// wait some time to allow the gossip to propagate the peers discovery
-		time.Sleep(3 * time.Second)
-	}
-	if expectedPeers != len(peers) {
-		return nil, errors.Errorf("Expecting %d peers but got %d", expectedPeers, len(peers))
+	discoveredPeers, err := retry.NewInvoker(retry.New(retry.TestRetryOpts)).Invoke(
+		func() (interface{}, error) {
+			peers, err := ctx.LocalDiscoveryService().GetPeers()
+			if err != nil {
+				return nil, errors.Wrapf(err, "error getting peers for MSP [%s]", ctx.Identifier().MSPID)
+			}
+			if len(peers) < expectedPeers {
+				return nil, status.New(status.TestStatus, status.GenericTransient.ToInt32(), fmt.Sprintf("Expecting %d peers but got %d", expectedPeers, len(peers)), nil)
+			}
+			return peers, nil
+		},
+	)
+	if err != nil {
+		return nil, err
 	}
-	return peers, nil
+
+	return discoveredPeers.([]fabAPI.Peer), nil
 }
 
 // EnsureChannelCreatedAndPeersJoined creates a channel, joins all peers in the given orgs to the channel and updates the anchor peers of each org.
@@ -350,22 +353,22 @@ func EnsureChannelCreatedAndPeersJoined(t *testing.T, sdk *fabsdk.FabricSDK, cha
 // In Fabric 1.0 there is a bug that panics the orderer if more than one config update is added to the same block.
 // This function may be invoked after each config update as a workaround.
 func WaitForOrdererConfigUpdate(t *testing.T, client *resmgmt.Client, channelID string, genesis bool, lastConfigBlock uint64) uint64 {
-	for i := 0; i < 10; i++ {
-		chConfig, err := client.QueryConfigFromOrderer(channelID, resmgmt.WithOrdererEndpoint("orderer.example.com"))
-		if err != nil {
-			t.Logf("orderer returned err [%d, %d, %s]", i, lastConfigBlock, err)
-			time.Sleep(2 * time.Second)
-			continue
-		}
 
-		currentBlock := chConfig.BlockNumber()
-		t.Logf("WaitForOrdererConfigUpdate [%d, %d, %d]", i, currentBlock, lastConfigBlock)
-		if currentBlock > lastConfigBlock || genesis {
-			return currentBlock
-		}
-		time.Sleep(2 * time.Second)
-	}
+	blockNum, err := retry.NewInvoker(retry.New(retry.TestRetryOpts)).Invoke(
+		func() (interface{}, error) {
+			chConfig, err := client.QueryConfigFromOrderer(channelID, resmgmt.WithOrdererEndpoint("orderer.example.com"))
+			if err != nil {
+				return nil, status.New(status.TestStatus, status.GenericTransient.ToInt32(), err.Error(), nil)
+			}
+
+			currentBlock := chConfig.BlockNumber()
+			if currentBlock <= lastConfigBlock && !genesis {
+				return nil, status.New(status.TestStatus, status.GenericTransient.ToInt32(), fmt.Sprintf("Block number was not incremented [%d, %d]", currentBlock, lastConfigBlock), nil)
+			}
+			return &currentBlock, nil
+		},
+	)
 
-	t.Fatal("orderer did not update channel config")
-	return 0
+	require.NoError(t, err)
+	return *blockNum.(*uint64)
 }
diff --git a/test/integration/expiredorderer/expired_certificate_test.go b/test/integration/expiredorderer/expired_certificate_test.go
@@ -21,8 +21,6 @@ import (
 	"github.com/hyperledger/fabric-sdk-go/test/integration"
 	"github.com/hyperledger/fabric-sdk-go/test/metadata"
 
-	"time"
-
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/logging"
 	"github.com/hyperledger/fabric-sdk-go/pkg/core/config"
 	"github.com/hyperledger/fabric-sdk-go/pkg/core/config/lookup"
@@ -57,7 +55,6 @@ func TestExpiredCert(t *testing.T) {
 		t.Fatalf("Failed to create new SDK: %s", err)
 	}
 	defer sdk.Close()
-	time.Sleep(100 * time.Millisecond)
 
 	// Delete all private keys from the crypto suite store
 	// and users from the user store at the end
@@ -102,7 +99,6 @@ func TestExpiredCert(t *testing.T) {
 	if err == nil {
 		t.Fatal("Expected error: calling orderer 'orderer.example.com:7050' failed: Orderer Client Status Code: (2) CONNECTION_FAILED....")
 	}
-	time.Sleep(100 * time.Millisecond)
 
 }
 

diff --git a/test/integration/fab/channel_ledger_test.go b/test/integration/fab/channel_ledger_test.go
@@ -7,12 +7,13 @@ SPDX-License-Identifier: Apache-2.0
 package fab
 
 import (
+	"fmt"
 	"path"
 	"strconv"
 	"testing"
-	"time"
 
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/retry"
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/status"
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/providers/fab"
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/providers/msp"
 	"github.com/hyperledger/fabric-sdk-go/test/integration"
@@ -174,34 +175,26 @@ func verifyTargetsChangedBlockState(t *testing.T, client *channel.Client, chainc
 }
 
 func verifyTargetChangedBlockState(t *testing.T, client *channel.Client, chaincodeID string, target string, expectedValue int) {
-
-	const (
-		maxRetries = 10
-		retrySleep = 500 * time.Millisecond
-	)
-
-	for r := 0; r < 10; r++ {
-		req := channel.Request{
-			ChaincodeID: chaincodeID,
-			Fcn:         "invoke",
-			Args:        integration.ExampleCCQueryArgs(),
-		}
-
-		resp, err := client.Query(req, channel.WithTargetEndpoints(target), channel.WithRetry(retry.DefaultChannelOpts))
-		require.NoError(t, err, "query funds failed")
-		valueAfterInvoke := resp.Payload
-
-		// Verify that transaction changed block state
-		valueAfterInvokeInt, _ := strconv.Atoi(string(valueAfterInvoke))
-		if expectedValue == valueAfterInvokeInt {
-			return
-		}
-
-		t.Logf("On Attempt [%d / %d]: Response didn't match expected value [%d, %d]", r, maxRetries, valueAfterInvokeInt, expectedValue)
-		time.Sleep(retrySleep)
+	req := channel.Request{
+		ChaincodeID: chaincodeID,
+		Fcn:         "invoke",
+		Args:        integration.ExampleCCQueryArgs(),
 	}
 
-	t.Fatal("Exceeded max retries")
+	_, err := retry.NewInvoker(retry.New(retry.TestRetryOpts)).Invoke(
+		func() (interface{}, error) {
+			resp, err := client.Query(req, channel.WithTargetEndpoints(target), channel.WithRetry(retry.DefaultChannelOpts))
+			require.NoError(t, err, "query funds failed")
+
+			// Verify that transaction changed block state
+			actualValue, _ := strconv.Atoi(string(resp.Payload))
+			if expectedValue != actualValue {
+				return nil, status.New(status.TestStatus, status.GenericTransient.ToInt32(), fmt.Sprintf("ledger value didn't match expectation [%d, %d]", expectedValue, actualValue), nil)
+			}
+			return &actualValue, nil
+		},
+	)
+	require.NoError(t, err)
 }
 
 func testQueryTransaction(t *testing.T, ledgerClient *ledger.Client, txID fab.TransactionID, targets []string) {

diff --git a/test/integration/fab/discoveryclient_test.go b/test/integration/fab/discoveryclient_test.go
@@ -16,12 +16,15 @@ import (
 
 	mspclient "github.com/hyperledger/fabric-sdk-go/pkg/client/msp"
 	"github.com/hyperledger/fabric-sdk-go/pkg/client/resmgmt"
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/retry"
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/status"
 	contextAPI "github.com/hyperledger/fabric-sdk-go/pkg/common/providers/context"
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/providers/fab"
 	packager "github.com/hyperledger/fabric-sdk-go/pkg/fab/ccpackager/gopackager"
 	"github.com/hyperledger/fabric-sdk-go/pkg/fab/comm"
 	"github.com/hyperledger/fabric-sdk-go/pkg/fab/discovery"
 	"github.com/hyperledger/fabric-sdk-go/test/integration"
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -240,23 +243,23 @@ func testEndorsers(t *testing.T, sdk *fabsdk.FabricSDK, interest *fabdiscovery.C
 	peerCfg1, err := comm.NetworkPeerConfig(ctx.EndpointConfig(), peer0Org1)
 	require.NoErrorf(t, err, "error getting peer config for [%s]", peer0Org1)
 
-	var chanResp discclient.ChannelResponse
-	var lastErr error
-	for r := 0; r < 10; r++ {
-		chanResp, lastErr = sendEndorserQuery(t, ctx, client, interest, peerCfg1.PeerConfig)
-		if lastErr == nil {
-			break
-		}
-		if strings.Contains(lastErr.Error(), "failed constructing descriptor for chaincodes") {
-			// This error is a result of Gossip not being up-to-date with the instantiated chaincodes of all peers.
-			// A retry should resolve the error.
-			t.Logf("Got transient error from discovery: %s. Retrying in 3 seconds", lastErr)
-			time.Sleep(3 * time.Second)
-		} else {
-			t.Fatalf("Got error from discovery: %s", lastErr)
-		}
-	}
-	require.NoError(t, lastErr)
+	chResponse, err := retry.NewInvoker(retry.New(retry.TestRetryOpts)).Invoke(
+		func() (interface{}, error) {
+			chanResp, err := sendEndorserQuery(t, ctx, client, interest, peerCfg1.PeerConfig)
+			if err != nil && strings.Contains(err.Error(), "failed constructing descriptor for chaincodes") {
+				// This error is a result of Gossip not being up-to-date with the instantiated chaincodes of all peers.
+				// A retry should resolve the error.
+				return nil, status.New(status.TestStatus, status.GenericTransient.ToInt32(), err.Error(), nil)
+			} else if err != nil {
+				return nil, errors.WithMessage(err, "Got error from discovery")
+			}
+
+			return chanResp, nil
+		},
+	)
+
+	require.NoError(t, err)
+	chanResp := chResponse.(discclient.ChannelResponse)
 
 	// Get endorsers a few times, since each time a different set may be returned
 	for i := 0; i < 3; i++ {

diff --git a/test/integration/orgs/multiple_orgs_minconfig_test.go b/test/integration/orgs/multiple_orgs_minconfig_test.go
@@ -11,9 +11,10 @@ package orgs
 
 import (
 	"testing"
-	"time"
 
 	"github.com/hyperledger/fabric-sdk-go/pkg/client/common/discovery/dynamicdiscovery"
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/retry"
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/status"
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/providers/context"
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/providers/fab"
 	"github.com/hyperledger/fabric-sdk-go/pkg/fabsdk"
@@ -108,23 +109,26 @@ func discoverPeers(t *testing.T, sdk *fabsdk.FabricSDK) []fab.Peer {
 	discovery, err := chCtx.ChannelService().Discovery()
 	require.NoErrorf(t, err, "Error getting discovery service for channel [%s]", channelID)
 
-	var peers []fab.Peer
-	for i := 0; i < 10; i++ {
-		peers, err = discovery.GetPeers()
-		require.NoErrorf(t, err, "Error getting peers for channel [%s]", channelID)
-
-		t.Logf("Peers of channel [%s]:", channelID)
-		for i, p := range peers {
-			t.Logf("%d- [%s] - MSP [%s]", i, p.URL(), p.MSPID())
-		}
-		if len(peers) >= 3 {
-			break
-		}
+	const expectedPeers = 3
+
+	discoveredPeers, err := retry.NewInvoker(retry.New(retry.TestRetryOpts)).Invoke(
+		func() (interface{}, error) {
+			peers, err := discovery.GetPeers()
+			require.NoErrorf(t, err, "Error getting peers for channel [%s]", channelID)
+
+			t.Logf("Peers of channel [%s]:", channelID)
+			for i, p := range peers {
+				t.Logf("%d- [%s] - MSP [%s]", i, p.URL(), p.MSPID())
+			}
+			if len(peers) < expectedPeers {
+				return nil, status.New(status.TestStatus, status.GenericTransient.ToInt32(), err.Error(), nil)
+			}
+			return peers, nil
+		},
+	)
 
-		// wait some time to allow the gossip to propagate the peers discovery
-		time.Sleep(3 * time.Second)
-	}
-	return peers
+	require.NoError(t, err)
+	return discoveredPeers.([]fab.Peer)
 }
 
 // DynamicDiscoveryProviderFactory is configured with dynamic (endorser) selection provider