[FAB-12464] add explicit retries in integration tests

Some integration tests fail itermittently mainly in the z build (randomly on x86 as well). This patch adds an explicit retry invocations with a custom retriable code to force a retry on these specific random errors. Change-Id: Ifb452a5cbe9c0c0aae4fc78383f78e096e5f23b5 Signed-off-by: Baha Shaaban <[email protected]>
hyperledger · Nov 12, 2018 · 303f01c · 303f01c
1 parent d8a85e8
commit 303f01c
Show file tree

Hide file tree

Showing 7 changed files with 110 additions and 33 deletions.
diff --git a/pkg/common/errors/retry/defaults.go b/pkg/common/errors/retry/defaults.go
@@ -133,6 +133,7 @@ var ChannelClientRetryableCodes = map[status.Group][]status.Code{
 	status.EndorserClientStatus: {
 		status.ConnectionFailed, status.EndorsementMismatch,
 		status.PrematureChaincodeExecution,
+		status.Code(pb.TxValidationCode_MVCC_READ_CONFLICT),
 		status.ChaincodeAlreadyLaunching,
 		status.ChaincodeNameNotFound,
 	},

diff --git a/test/integration/e2e/orgs/multiple_orgs_test.go b/test/integration/e2e/orgs/multiple_orgs_test.go
@@ -273,13 +273,27 @@ func checkLedgerInfo(ledgerClient *ledger.Client, t *testing.T, ledgerInfoBefore
 		t.Fatal("Block size did not increase after transaction")
 	}
 	// Test Query Block by Hash - retrieve current block by number
-	block, err := ledgerClient.QueryBlock(ledgerInfoAfter.BCI.Height-1, ledger.WithTargets(orgTestPeer0.(fab.Peer), orgTestPeer1.(fab.Peer)), ledger.WithMinTargets(2))
+	//block, err := ledgerClient.QueryBlock(ledgerInfoAfter.BCI.Height-1, ledger.WithTargets(orgTestPeer0.(fab.Peer), orgTestPeer1.(fab.Peer)), ledger.WithMinTargets(2))
+	// invoke QueryBlock in retryable mode to ensure all peers have responded
+	block, err := retry.NewInvoker(retry.New(retry.TestRetryOpts)).Invoke(
+		func() (interface{}, error) {
+			b, e := ledgerClient.QueryBlock(ledgerInfoAfter.BCI.Height-1, ledger.WithTargets(orgTestPeer0.(fab.Peer), orgTestPeer1.(fab.Peer)), ledger.WithMinTargets(2))
+			if e != nil {
+				// return a retryable code if # of responses is less than the # of targets sent (in this case 2 responses needed)
+				if strings.Contains(e.Error(), "is less than MinTargets") {
+					return nil, status.New(status.TestStatus, status.GenericTransient.ToInt32(), fmt.Sprintf("QueryBlock returned error: %v", e), nil)
+				}
+			}
+			return b, e
+		},
+	)
 	if err != nil {
 		t.Fatalf("QueryBlock return error: %s", err)
 	}
 	if block == nil {
 		t.Fatal("Block info not available")
 	}
+
 	// Get transaction info
 	transactionInfo, err := ledgerClient.QueryTransaction(transactionID, ledger.WithTargets(orgTestPeer0.(fab.Peer), orgTestPeer1.(fab.Peer)), ledger.WithMinTargets(2))
 	if err != nil {

diff --git a/test/integration/pkg/client/channel/channel_client_pvt_test.go b/test/integration/pkg/client/channel/channel_client_pvt_test.go
@@ -9,12 +9,14 @@ SPDX-License-Identifier: Apache-2.0
 package channel
 
 import (
+	"fmt"
 	"strconv"
 	"strings"
 	"sync"
 	"testing"
 
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/retry"
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/status"
 
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/multi"
 	"github.com/stretchr/testify/assert"
@@ -309,18 +311,34 @@ func TestChannelClientRollsBackPvtDataIfMvccReadConflict(t *testing.T) {
 	require.Truef(t, len(errs) > 0 && strings.Contains(errs[0].Error(), "MVCC_READ_CONFLICT"), "could not reproduce MVCC_READ_CONFLICT")
 
 	// read current value of private data collection
-	resp, err := chClient.Query(
-		channel.Request{
-			ChaincodeID: ccID,
-			Fcn:         "getprivate",
-			Args:        [][]byte{[]byte(coll), []byte(key)},
+	//resp, err := chClient.Query(
+	//	channel.Request{
+	//		ChaincodeID: ccID,
+	//		Fcn:         "getprivate",
+	//		Args:        [][]byte{[]byte(coll), []byte(key)},
+	//	},
+	//	channel.WithRetry(retry.TestRetryOpts),
+	//)
+	resp, err := retry.NewInvoker(retry.New(retry.TestRetryOpts)).Invoke(
+		func() (interface{}, error) {
+			b, e := chClient.Query(
+				channel.Request{
+					ChaincodeID: ccID,
+					Fcn:         "getprivate",
+					Args:        [][]byte{[]byte(coll), []byte(key)},
+				},
+				channel.WithRetry(retry.TestRetryOpts),
+			)
+			if e != nil || strings.TrimSpace(string(b.Payload)) == "" {
+				return nil, status.New(status.TestStatus, status.GenericTransient.ToInt32(), fmt.Sprintf("getprivate data returned error: %v", e), nil)
+			}
+			return b, e
 		},
-		channel.WithRetry(retry.TestRetryOpts),
 	)
 	require.NoErrorf(t, err, "error attempting to read private data")
-	require.NotEmptyf(t, resp.Payload, "reading private data returned empty response")
+	require.NotEmptyf(t, strings.TrimSpace(string(resp.(channel.Response).Payload)), "reading private data returned empty response")
 
-	actual, err := strconv.Atoi(string(resp.Payload))
+	actual, err := strconv.Atoi(string(resp.(channel.Response).Payload))
 	require.NoError(t, err)
 
 	assert.Truef(t, actual == expected, "Private data not rolled back during MVCC_READ_CONFLICT")

diff --git a/test/integration/pkg/client/channel/channel_client_test.go b/test/integration/pkg/client/channel/channel_client_test.go
@@ -622,13 +622,13 @@ func TestNoEndpoints(t *testing.T) {
 	expected1_1Err := "targets were not provided"                   // When running with 1.1 DynamicSelection
 	expected1_2Err := "no endorsement combination can be satisfied" // When running with 1.2 FabricSelection
 	if !strings.Contains(err.Error(), expected1_1Err) && !strings.Contains(err.Error(), expected1_2Err) {
-		t.Fatal("Should have failed due to no chaincode query peers")
+		t.Fatal("Query chaincode should have failed due to no chaincode query peers, error (if any): ", err)
 	}
 
 	// Test execute transaction: since peer has been disabled for endorsement this transaction should fail
 	_, err = chClient.Execute(channel.Request{ChaincodeID: mainChaincodeID, Fcn: "invoke", Args: integration.ExampleCCTxRandomSetArgs()},
 		channel.WithRetry(retry.DefaultChannelOpts))
 	if !strings.Contains(err.Error(), expected1_1Err) && !strings.Contains(err.Error(), expected1_2Err) {
-		t.Fatal("Should have failed due to no chaincode query peers")
+		t.Fatal("Execute chaincode should have failed due to no chaincode query peers, error (if any): ", err)
 	}
 }
diff --git a/test/integration/pkg/client/ledger/channel_ledger_test.go b/test/integration/pkg/client/ledger/channel_ledger_test.go
@@ -9,6 +9,7 @@ package ledger
 import (
 	"fmt"
 	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/retry"
@@ -149,11 +150,21 @@ func changeBlockState(t *testing.T, client *channel.Client, queryArg [][]byte, m
 		Fcn:         "invoke",
 		Args:        queryArg,
 	}
-	resp, err := client.Query(req, channel.WithRetry(retry.DefaultChannelOpts))
+	//resp, err := client.Query(req, channel.WithRetry(retry.DefaultChannelOpts))
+	resp, err := retry.NewInvoker(retry.New(retry.TestRetryOpts)).Invoke(
+		func() (interface{}, error) {
+			response, err := client.Query(req, channel.WithRetry(retry.DefaultChannelOpts))
+
+			if err != nil && strings.Contains(err.Error(), "Nil amount") {
+				return nil, status.New(status.TestStatus, status.GenericTransient.ToInt32(), "query funds failed", nil)
+			}
+			return response, nil
+		},
+	)
 	if err != nil {
 		return "", 0, errors.WithMessage(err, "query funds failed")
 	}
-	value := resp.Payload
+	value := resp.(channel.Response).Payload
 
 	// Start transaction that will change block state
 	txID, err := moveFundsAndGetTxID(t, client, moveArg, chaincodeID)

diff --git a/test/integration/pkg/client/ledger/ledger_queries_test.go b/test/integration/pkg/client/ledger/ledger_queries_test.go
@@ -13,6 +13,8 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"github.com/hyperledger/fabric-sdk-go/pkg/client/ledger"
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/retry"
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/status"
 	providersFab "github.com/hyperledger/fabric-sdk-go/pkg/common/providers/fab"
 	"github.com/hyperledger/fabric-sdk-go/pkg/core/config"
 	"github.com/hyperledger/fabric-sdk-go/pkg/fab"
@@ -41,6 +43,39 @@ func TestLedgerClientQueries(t *testing.T) {
 		t.Fatalf("QueryInfo return error: %s", err)
 	}
 
+	testPeerConfig(ledgerInfo, t)
+
+	// Same query with target
+	target := testSetup.Targets[0]
+	//ledgerInfoFromTarget, err := client.QueryInfo(ledger.WithTargetEndpoints(target))
+	ledgerInfoFromTarget, err := retry.NewInvoker(retry.New(retry.TestRetryOpts)).Invoke(
+		func() (interface{}, error) {
+			response, e := client.QueryInfo(ledger.WithTargetEndpoints(target))
+
+			if err != nil && (strings.Contains(e.Error(), "QueryInfo failed") || strings.Contains(e.Error(), "Number of responses")) {
+				return nil, status.New(status.TestStatus, status.GenericTransient.ToInt32(), "query funds failed", nil)
+			}
+			if !proto.Equal(response.BCI, ledgerInfo.BCI) {
+				return nil, status.New(status.TestStatus, status.GenericTransient.ToInt32(), "results mismatch between default and target peers", nil)
+			}
+			return response, nil
+		},
+	)
+	if err != nil {
+		t.Fatalf("QueryInfo return error: %s", err)
+	}
+
+	if !proto.Equal(ledgerInfoFromTarget.(*providersFab.BlockchainInfoResponse).BCI, ledgerInfo.BCI) {
+		t.Fatal("Expecting same result from default peer and target peer")
+	}
+
+	testQueryBlockNumberByHash(client, ledgerInfo, t)
+
+	testQueryBlockNumber(client, t, 0)
+
+}
+func testPeerConfig(ledgerInfo *providersFab.BlockchainInfoResponse, t *testing.T) {
+	sdk := mainSDK
 	configBackend, err := sdk.Config()
 	if err != nil {
 		t.Fatalf("failed to get config backend, error: %s", err)
@@ -59,22 +94,6 @@ func TestLedgerClientQueries(t *testing.T) {
 	if !strings.Contains(ledgerInfo.Endorser, expectedPeerConfig1.URL) && !strings.Contains(ledgerInfo.Endorser, expectedPeerConfig2.URL) {
 		t.Fatalf("Expecting %s or %s, got %s", expectedPeerConfig1.URL, expectedPeerConfig2.URL, ledgerInfo.Endorser)
 	}
-
-	// Same query with target
-	target := testSetup.Targets[0]
-	ledgerInfoFromTarget, err := client.QueryInfo(ledger.WithTargetEndpoints(target))
-	if err != nil {
-		t.Fatalf("QueryInfo return error: %s", err)
-	}
-
-	if !proto.Equal(ledgerInfoFromTarget.BCI, ledgerInfo.BCI) {
-		t.Fatal("Expecting same result from default peer and target peer")
-	}
-
-	testQueryBlockNumberByHash(client, ledgerInfo, t)
-
-	testQueryBlockNumber(client, t, 0)
-
 }
 
 func testQueryBlockNumberByHash(client *ledger.Client, ledgerInfo *providersFab.BlockchainInfoResponse, t *testing.T) {

diff --git a/test/integration/pkg/fabsdk/provider/sdk_provider_test.go b/test/integration/pkg/fabsdk/provider/sdk_provider_test.go
@@ -9,6 +9,7 @@ package provider
 import (
 	"fmt"
 	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/status"
@@ -61,13 +62,26 @@ func TestDynamicSelection(t *testing.T) {
 	chClient, err := channel.New(org1ChannelClientContext)
 	require.NoError(t, err, "Failed to create new channel client")
 
-	response, err := chClient.Query(channel.Request{ChaincodeID: chaincodeID, Fcn: "invoke", Args: queryArg},
-		channel.WithRetry(retry.TestRetryOpts))
+	//response, err := chClient.Query(channel.Request{ChaincodeID: chaincodeID, Fcn: "invoke", Args: queryArg},
+	//	channel.WithRetry(retry.TestRetryOpts))
+	response, err := retry.NewInvoker(retry.New(retry.TestRetryOpts)).Invoke(
+		func() (interface{}, error) {
+			b, e := chClient.Query(channel.Request{ChaincodeID: chaincodeID, Fcn: "invoke", Args: queryArg},
+				channel.WithRetry(retry.TestRetryOpts))
+			if e != nil {
+				// return a retryable code if key/value query is nil (ie not propagated to all peers yet)
+				if strings.Contains(e.Error(), "Nil amount for") {
+					return nil, status.New(status.TestStatus, status.GenericTransient.ToInt32(), fmt.Sprintf("QueryBlock returned error: %v", e), nil)
+				}
+			}
+			return b, e
+		},
+	)
 	require.NoError(t, err, "Failed to query funds, ccID: %s, queryArgs: [%+v]", chaincodeID, queryArg)
-	value := response.Payload
+	value := response.(channel.Response).Payload
 
 	// Move funds
-	response, err = chClient.Execute(channel.Request{ChaincodeID: chaincodeID, Fcn: "invoke", Args: moveTxArg},
+	_, err = chClient.Execute(channel.Request{ChaincodeID: chaincodeID, Fcn: "invoke", Args: moveTxArg},
 		channel.WithRetry(retry.DefaultChannelOpts))
 	require.NoError(t, err, "Failed to move funds, ccID: %s, queryArgs:[%+v]", chaincodeID, moveTxArg)