[FAB-9717] support for multiple config backends

- multiple backends can be supplied to create new SDK.
- lookup logic will go through first backend to find
value for given key. If key-value not found then it falls
back to the next one in backend list.
- refactored endpointConfig and identityConfig to fix
few bugs when entity matchers are provided.
- Fixed integration-tests-local issues and updated read me

Change-Id: Id4a05f55749ec5a88e2086d7df4751bd9a7f1e05
Signed-off-by: Sudesh Shetty <[email protected]>

README.md

@@ -158,9 +158,8 @@ cd $GOPATH/src/github.com/hyperledger/fabric-sdk-go
 
 # Use script to setup parameters for integration tests and execute them
 # Previously we use to have hostnames like Fabric CA server, orderer and peer pointed to localhost
-# Now since we removed this now, We will be using a different config file config_test_local.yaml
-# which has the Fabric CA server, orderer and peers pointed to localhost
-make -args configFile=config_test_local.yaml integration-tests-local
+# Now since we removed this now, We will be using a different configuration
+make integration-tests-local
 
 # Or more generally, run integration tests at a different code level (prev, stable, prerelease, devstable)
 # and fixture target version
@@ -174,10 +173,10 @@ make -args configFile=config_test_local.yaml integration-tests-local
 # which has the Fabric CA server, orderer and peers pointed to localhost
 # It is also possible to run integration tests using go test directly. For example:
 #cd $GOPATH/src/github.com/hyperledger/fabric-sdk-go/test/integration/
-#go -args configFile=config_test_local.yaml test
+#go -args testLocal=true test
 
 #cd $GOPATH/src/github.com/hyperledger/fabric-sdk-go/test/integration/orgs
-#go -args configFile=config_test_local.yaml test
+#go -args testLocal=true test
 
 # You should review test/scripts/integration.sh for options and details.
 # Note: you should generally prefer the scripted version to setup parameters for you.

pkg/client/common/discovery/discoveryfilter_test.go

@@ -34,7 +34,7 @@ func TestDiscoveryFilter(t *testing.T) {
 		t.Fatalf(err.Error())
 	}
 
-	config1, err := fabImpl.ConfigFromBackend(configBackend)
+	config1, err := fabImpl.ConfigFromBackend(configBackend...)
 	if err != nil {
 		t.Fatalf(err.Error())
 	}

pkg/client/common/discovery/staticdiscovery/staticdiscovery_test.go

@@ -23,7 +23,7 @@ func TestStaticDiscovery(t *testing.T) {
 		t.Fatalf(err.Error())
 	}
 
-	config1, err := fabImpl.ConfigFromBackend(configBackend)
+	config1, err := fabImpl.ConfigFromBackend(configBackend...)
 	if err != nil {
 		t.Fatalf(err.Error())
 	}
@@ -58,7 +58,7 @@ func TestStaticDiscoveryWhenChannelIsEmpty(t *testing.T) {
 		t.Fatalf(err.Error())
 	}
 
-	config1, err := fabImpl.ConfigFromBackend(configBackend)
+	config1, err := fabImpl.ConfigFromBackend(configBackend...)
 	if err != nil {
 		t.Fatalf(err.Error())
 	}
@@ -74,7 +74,7 @@ func TestStaticLocalDiscovery(t *testing.T) {
 	configBackend, err := config.FromFile("../../../../../test/fixtures/config/config_test.yaml")()
 	assert.NoError(t, err)
 
-	config1, err := fabImpl.ConfigFromBackend(configBackend)
+	config1, err := fabImpl.ConfigFromBackend(configBackend...)
 	assert.NoError(t, err)
 
 	discoveryProvider, err := New(config1)

pkg/client/common/selection/staticselection/staticselection_test.go

@@ -30,7 +30,7 @@ func TestStaticSelection(t *testing.T) {
 		t.Fatalf(err.Error())
 	}
 
-	config, err := fabImpl.ConfigFromBackend(configBackend)
+	config, err := fabImpl.ConfigFromBackend(configBackend...)
 	if err != nil {
 		t.Fatalf(err.Error())
 	}

pkg/client/msp/client_test.go

@@ -175,10 +175,10 @@ func (f *textFixture) setup() *fabsdk.FabricSDK {
 	}
 
 	//Override ca matchers for this test
-	customBackend := getCustomBackend(backend)
+	customBackend := getCustomBackend(backend...)
 
-	configProvider := func() (core.ConfigBackend, error) {
-		return customBackend, nil
+	configProvider := func() ([]core.ConfigBackend, error) {
+		return []core.ConfigBackend{customBackend}, nil
 	}
 
 	// Instantiate the SDK
@@ -233,12 +233,12 @@ func randomUsername() string {
 	return "user" + strconv.Itoa(rand.Intn(500000))
 }
 
-func getCustomBackend(backend core.ConfigBackend) *mocks.MockConfigBackend {
+func getCustomBackend(backend ...core.ConfigBackend) *mocks.MockConfigBackend {
 	backendMap := make(map[string]interface{})
 
 	//Custom URLs for ca configs
 	networkConfig := fab.NetworkConfig{}
-	configLookup := lookup.New(backend)
+	configLookup := lookup.New(backend...)
 	configLookup.UnmarshalKey("certificateAuthorities", &networkConfig.CertificateAuthorities)
 
 	ca1Config := networkConfig.CertificateAuthorities["ca.org1.example.com"]

pkg/client/resmgmt/resmgmt_test.go

@@ -306,7 +306,7 @@ func TestOrdererConfigFail(t *testing.T) {
 	assert.Nil(t, err)
 
 	// remove channel orderer and global orderers from config backend
-	configBackend := getNoOrdererBackend(backend)
+	configBackend := getNoOrdererBackend(backend...)
 
 	noOrdererConfig, err := fabImpl.ConfigFromBackend(configBackend)
 	assert.Nil(t, err)
@@ -328,7 +328,7 @@ func TestJoinChannelNoOrdererConfig(t *testing.T) {
 	assert.Nil(t, err)
 
 	// remove channel orderer and global orderers from config backend
-	configBackend := getNoOrdererBackend(backend)
+	configBackend := getNoOrdererBackend(backend...)
 	noOrdererConfig, err := fabImpl.ConfigFromBackend(configBackend)
 	if err != nil {
 		t.Fatal(err)
@@ -343,7 +343,7 @@ func TestJoinChannelNoOrdererConfig(t *testing.T) {
 	assert.Contains(t, err.Error(), "channel config not found")
 
 	// Misconfigured channel orderer
-	configBackend = getInvalidChannelOrdererBackend(backend)
+	configBackend = getInvalidChannelOrdererBackend(backend...)
 	invalidChOrdererConfig, err := fabImpl.ConfigFromBackend(configBackend)
 
 	if err != nil {
@@ -359,7 +359,7 @@ func TestJoinChannelNoOrdererConfig(t *testing.T) {
 	}
 
 	// Misconfigured global orderer (cert cannot be loaded)
-	configBackend = getInvalidOrdererBackend(backend)
+	configBackend = getInvalidOrdererBackend(backend...)
 	invalidOrdererConfig, err := fabImpl.ConfigFromBackend(configBackend)
 	if err != nil {
 		t.Fatal(err)
@@ -977,7 +977,7 @@ func TestCCProposal(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	cfg, err := fabImpl.ConfigFromBackend(configBackend)
+	cfg, err := fabImpl.ConfigFromBackend(configBackend...)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1019,12 +1019,12 @@ func TestCCProposal(t *testing.T) {
 	}
 
 	// Test no event source in config
-	backend, err := configImpl.FromFile(configPath)()
+	backends, err := configImpl.FromFile(configPath)()
 	if err != nil {
 		t.Fatal(err)
 	}
-	configBackend = getNoEventSourceBackend(backend)
-	cfg, err = fabImpl.ConfigFromBackend(configBackend)
+	backend := getNoEventSourceBackend(backends...)
+	cfg, err = fabImpl.ConfigFromBackend(backend)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1109,7 +1109,7 @@ func getNetworkConfig(t *testing.T) fab.EndpointConfig {
 		t.Fatal(err)
 	}
 
-	config, err := fabImpl.ConfigFromBackend(configBackend)
+	config, err := fabImpl.ConfigFromBackend(configBackend...)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1348,30 +1348,30 @@ func createClientContextWithError(fabCtx context.Client) context.ClientProvider
 	}
 }
 
-func getCustomBackend(backend core.ConfigBackend) *mocks.MockConfigBackend {
+func getCustomBackend(backend ...core.ConfigBackend) *mocks.MockConfigBackend {
 
 	backendMap := make(map[string]interface{})
-	backendMap["client"], _ = backend.Lookup("client")
-	backendMap["certificateAuthorities"], _ = backend.Lookup("certificateAuthorities")
-	backendMap["entityMatchers"], _ = backend.Lookup("entityMatchers")
-	backendMap["peers"], _ = backend.Lookup("peers")
-	backendMap["organizations"], _ = backend.Lookup("organizations")
-	backendMap["orderers"], _ = backend.Lookup("orderers")
-	backendMap["channels"], _ = backend.Lookup("channels")
+	backendMap["client"], _ = backend[0].Lookup("client")
+	backendMap["certificateAuthorities"], _ = backend[0].Lookup("certificateAuthorities")
+	backendMap["entityMatchers"], _ = backend[0].Lookup("entityMatchers")
+	backendMap["peers"], _ = backend[0].Lookup("peers")
+	backendMap["organizations"], _ = backend[0].Lookup("organizations")
+	backendMap["orderers"], _ = backend[0].Lookup("orderers")
+	backendMap["channels"], _ = backend[0].Lookup("channels")
 
 	return &mocks.MockConfigBackend{KeyValueMap: backendMap}
 }
 
-func getNoOrdererBackend(backend core.ConfigBackend) *mocks.MockConfigBackend {
+func getNoOrdererBackend(backend ...core.ConfigBackend) *mocks.MockConfigBackend {
 
-	mockConfigBackend := getCustomBackend(backend)
+	mockConfigBackend := getCustomBackend(backend...)
 	mockConfigBackend.KeyValueMap["channels"] = nil
 	mockConfigBackend.KeyValueMap["orderers"] = nil
 
 	return mockConfigBackend
 }
 
-func getInvalidChannelOrdererBackend(backend core.ConfigBackend) *mocks.MockConfigBackend {
+func getInvalidChannelOrdererBackend(backend ...core.ConfigBackend) *mocks.MockConfigBackend {
 
 	//Create an invalid channel
 	channels := make(map[string]fab.ChannelNetworkConfig)
@@ -1380,17 +1380,17 @@ func getInvalidChannelOrdererBackend(backend core.ConfigBackend) *mocks.MockConf
 	}
 	channels["mychannel"] = mychannel
 
-	mockConfigBackend := getCustomBackend(backend)
+	mockConfigBackend := getCustomBackend(backend...)
 	mockConfigBackend.KeyValueMap["channels"] = channels
 
 	return mockConfigBackend
 }
 
-func getInvalidOrdererBackend(backend core.ConfigBackend) *mocks.MockConfigBackend {
+func getInvalidOrdererBackend(backend ...core.ConfigBackend) *mocks.MockConfigBackend {
 
 	//Create invalid orderer
 	networkConfig := fab.NetworkConfig{}
-	err := lookup.New(backend).UnmarshalKey("orderers", &networkConfig.Orderers)
+	err := lookup.New(backend...).UnmarshalKey("orderers", &networkConfig.Orderers)
 	if err != nil {
 		panic(err)
 	}
@@ -1399,17 +1399,17 @@ func getInvalidOrdererBackend(backend core.ConfigBackend) *mocks.MockConfigBacke
 	exampleOrderer.TLSCACerts.Pem = validRootCA
 	networkConfig.Orderers["orderer.example.com"] = exampleOrderer
 
-	mockConfigBackend := getCustomBackend(backend)
+	mockConfigBackend := getCustomBackend(backend...)
 	mockConfigBackend.KeyValueMap["orderers"] = networkConfig.Orderers
 
 	return mockConfigBackend
 }
 
-func getNoEventSourceBackend(backend core.ConfigBackend) *mocks.MockConfigBackend {
+func getNoEventSourceBackend(backend ...core.ConfigBackend) *mocks.MockConfigBackend {
 
 	//Create no event source channels
 	networkConfig := fab.NetworkConfig{}
-	err := lookup.New(backend).UnmarshalKey("channels", &networkConfig.Channels)
+	err := lookup.New(backend...).UnmarshalKey("channels", &networkConfig.Channels)
 	if err != nil {
 		panic(err)
 	}
@@ -1419,7 +1419,7 @@ func getNoEventSourceBackend(backend core.ConfigBackend) *mocks.MockConfigBacken
 	mychannel.Peers["peer0.org1.example.com"] = chPeer
 	networkConfig.Channels["mychannel"] = mychannel
 
-	mockConfigBackend := getCustomBackend(backend)
+	mockConfigBackend := getCustomBackend(backend...)
 	mockConfigBackend.KeyValueMap["channels"] = networkConfig.Channels
 
 	return mockConfigBackend

pkg/common/providers/core/provider.go

@@ -26,7 +26,7 @@ type Providers interface {
 }
 
 //ConfigProvider provides config backend for SDK
-type ConfigProvider func() (ConfigBackend, error)
+type ConfigProvider func() ([]ConfigBackend, error)
 
 //ConfigBackend backend for all config types in SDK
 type ConfigBackend interface {

pkg/core/config/config.go

@@ -37,14 +37,14 @@ type Option func(opts *options) error
 // FromReader loads configuration from in.
 // configType can be "json" or "yaml".
 func FromReader(in io.Reader, configType string, opts ...Option) core.ConfigProvider {
-	return func() (core.ConfigBackend, error) {
+	return func() ([]core.ConfigBackend, error) {
 		return initFromReader(in, configType, opts...)
 	}
 }
 
 // FromFile reads from named config file
 func FromFile(name string, opts ...Option) core.ConfigProvider {
-	return func() (core.ConfigBackend, error) {
+	return func() ([]core.ConfigBackend, error) {
 		backend, err := newBackend(opts...)
 		if err != nil {
 			return nil, err
@@ -65,19 +65,19 @@ func FromFile(name string, opts ...Option) core.ConfigProvider {
 
 		setLogLevel(backend)
 
-		return backend, nil
+		return []core.ConfigBackend{backend}, nil
 	}
 }
 
 // FromRaw will initialize the configs from a byte array
 func FromRaw(configBytes []byte, configType string, opts ...Option) core.ConfigProvider {
-	return func() (core.ConfigBackend, error) {
+	return func() ([]core.ConfigBackend, error) {
 		buf := bytes.NewBuffer(configBytes)
 		return initFromReader(buf, configType, opts...)
 	}
 }
 
-func initFromReader(in io.Reader, configType string, opts ...Option) (core.ConfigBackend, error) {
+func initFromReader(in io.Reader, configType string, opts ...Option) ([]core.ConfigBackend, error) {
 	backend, err := newBackend(opts...)
 	if err != nil {
 		return nil, err
@@ -96,7 +96,7 @@ func initFromReader(in io.Reader, configType string, opts ...Option) (core.Confi
 	}
 	setLogLevel(backend)
 
-	return backend, nil
+	return []core.ConfigBackend{backend}, nil
 }
 
 // WithEnvPrefix defines the prefix for environment variable overrides.

pkg/core/config/config_test.go

@@ -105,9 +105,13 @@ func TestInitConfigWithCmdRoot(t *testing.T) {
 		t.Fatalf("Failed to initialize config backend with cmd root. Error: %s", err)
 	}
 
-	config := cryptosuite.ConfigFromBackend(configBackend1)
+	if len(configBackend1) == 0 {
+		t.Fatal("invalid backend")
+	}
+
+	config := cryptosuite.ConfigFromBackend(configBackend1...)
 
-	secAlg, ok := configBackend1.Lookup("client.BCCSP.security.hashAlgorithm")
+	secAlg, ok := configBackend1[0].Lookup("client.BCCSP.security.hashAlgorithm")
 	if !ok {
 		t.Fatal("supposed to get valid value")
 	}
@@ -162,13 +166,17 @@ func TestMultipleVipers(t *testing.T) {
 		t.Log(err.Error())
 	}
 
+	if len(configBackend1) == 0 {
+		t.Fatal("invalid backend")
+	}
+
 	// Make sure initial value is unaffected
 	testValue2 := viper.GetString("test.testkey")
 	if testValue2 != "testvalue" {
 		t.Fatalf("Expected testvalue after config initialization")
 	}
 	// Make sure Go SDK config is unaffected
-	testValue3, ok := configBackend1.Lookup("client.BCCSP.security.softVerify")
+	testValue3, ok := configBackend1[0].Lookup("client.BCCSP.security.softVerify")
 	if !ok {
 		t.Fatalf("Expected valid value")
 	}
@@ -214,7 +222,11 @@ func TestEnvironmentVariablesSpecificCmdRoot(t *testing.T) {
 		t.Log(err.Error())
 	}
 
-	value, ok := configBackend1.Lookup("env.test")
+	if len(configBackend1) == 0 {
+		t.Fatal("invalid backend")
+	}
+
+	value, ok := configBackend1[0].Lookup("env.test")
 	if value != "456" || !ok {
 		t.Fatalf("Expected environment variable value but got: %s", testValue)
 	}
@@ -234,7 +246,10 @@ func setUp(m *testing.M) {
 	if err != nil {
 		fmt.Println(err.Error())
 	}
-	configBackend = cfgBackend
+	if len(cfgBackend) != 1 {
+		panic("invalid backend found")
+	}
+	configBackend = cfgBackend[0]
 }
 
 func teardown() {
@@ -329,11 +344,6 @@ func badOpt() Option {
 }
 
 func TestConfigBackend_Lookup(t *testing.T) {
-	var err error
-	configBackend, err = FromFile(configTestFilePath)()
-	if err != nil {
-		t.Fatalf("Unexpected error reading config backend: %v", err)
-	}
 
 	checkConfigStringKey(t, "name", "global-trade-network", true)