FABG-897 Convert protobuf to/from JSON

At the moment, the SDK works on protobuf messages in a
namespace that is different from the namespace used by Fabric,
so the native Fabric JSON marshalling/unmarshalling cannot be
applied. This change exposes JSON marshalling/unmarashalling
of protobuf messages in the SDK namespace.

Change-Id: I160234ad176b42345620495e2cda017d1d0837ec
Signed-off-by: Aleksandar Likic <[email protected]>

go.mod

@@ -45,3 +45,6 @@ require (
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/yaml.v2 v2.2.1
 )
+
+// TEMPORARY - so that we can create a commit for the third_party fabric module.
+replace github.com/hyperledger/fabric-sdk-go/third_party/github.com/hyperledger/fabric => ./third_party/github.com/hyperledger/fabric

pkg/fab/mocks/mockdata.go

@@ -64,6 +64,7 @@ type MockConfigGroupBuilder struct {
 	ChannelCapabilities     []string
 	ApplicationCapabilities []string
 	OrdererCapabilities     []string
+	PolicyRefs              []string
 }
 
 // MockConfigBlockBuilder is used to build a mock Chain configuration block
@@ -217,14 +218,12 @@ func (b *MockConfigGroupBuilder) buildOrdererGroup() *common.ConfigGroup {
 			"Admins":          b.buildBasicConfigPolicy(),
 		},
 		Values: map[string]*common.ConfigValue{
-			channelConfig.BatchSizeKey:                 b.buildBatchSizeConfigValue(),
-			channelConfig.AnchorPeersKey:               b.buildAnchorPeerConfigValue(),
-			channelConfig.ConsensusTypeKey:             b.buildConsensusTypeConfigValue(),
-			channelConfig.BatchTimeoutKey:              b.buildBatchTimeoutConfigValue(),
-			channelConfig.ChannelRestrictionsKey:       b.buildChannelRestrictionsConfigValue(),
-			channelConfig.HashingAlgorithmKey:          b.buildHashingAlgorithmConfigValue(),
-			channelConfig.BlockDataHashingStructureKey: b.buildBlockDataHashingStructureConfigValue(),
-			channelConfig.CapabilitiesKey:              b.buildCapabilitiesConfigValue(b.OrdererCapabilities),
+			channelConfig.ConsensusTypeKey:       b.buildConsensusTypeConfigValue(),
+			channelConfig.BatchSizeKey:           b.buildBatchSizeConfigValue(),
+			channelConfig.BatchTimeoutKey:        b.buildBatchTimeoutConfigValue(),
+			channelConfig.ChannelRestrictionsKey: b.buildChannelRestrictionsConfigValue(),
+			channelConfig.CapabilitiesKey:        b.buildCapabilitiesConfigValue(b.OrdererCapabilities),
+			channelConfig.KafkaBrokersKey:        b.buildKafkaBrokersConfigValue(),
 		},
 		Version:   b.Version,
 		ModPolicy: b.ModPolicy,
@@ -262,13 +261,6 @@ func (b *MockConfigGroupBuilder) buildBatchSizeConfigValue() *common.ConfigValue
 		Value:     marshalOrPanic(b.buildBatchSize())}
 }
 
-func (b *MockConfigGroupBuilder) buildAnchorPeerConfigValue() *common.ConfigValue {
-	return &common.ConfigValue{
-		Version:   b.Version,
-		ModPolicy: b.ModPolicy,
-		Value:     marshalOrPanic(b.buildAnchorPeer())}
-}
-
 func (b *MockConfigGroupBuilder) buildConsensusTypeConfigValue() *common.ConfigValue {
 	return &common.ConfigValue{
 		Version:   b.Version,
@@ -290,25 +282,25 @@ func (b *MockConfigGroupBuilder) buildChannelRestrictionsConfigValue() *common.C
 		Value:     marshalOrPanic(b.buildChannelRestrictions())}
 }
 
-func (b *MockConfigGroupBuilder) buildHashingAlgorithmConfigValue() *common.ConfigValue {
+func (b *MockConfigGroupBuilder) buildCapabilitiesConfigValue(capabilityNames []string) *common.ConfigValue {
 	return &common.ConfigValue{
 		Version:   b.Version,
 		ModPolicy: b.ModPolicy,
-		Value:     marshalOrPanic(b.buildHashingAlgorithm())}
+		Value:     marshalOrPanic(b.buildCapabilities(capabilityNames))}
 }
 
-func (b *MockConfigGroupBuilder) buildBlockDataHashingStructureConfigValue() *common.ConfigValue {
+func (b *MockConfigGroupBuilder) buildKafkaBrokersConfigValue() *common.ConfigValue {
 	return &common.ConfigValue{
 		Version:   b.Version,
 		ModPolicy: b.ModPolicy,
-		Value:     marshalOrPanic(b.buildBlockDataHashingStructure())}
+		Value:     marshalOrPanic(b.buildKafkaBrokers())}
 }
 
-func (b *MockConfigGroupBuilder) buildCapabilitiesConfigValue(capabilityNames []string) *common.ConfigValue {
+func (b *MockConfigGroupBuilder) buildACLsConfigValue(policyRefs []string) *common.ConfigValue {
 	return &common.ConfigValue{
 		Version:   b.Version,
 		ModPolicy: b.ModPolicy,
-		Value:     marshalOrPanic(b.buildCapabilities(capabilityNames))}
+		Value:     marshalOrPanic(b.buildACLs(policyRefs))}
 }
 
 func (b *MockConfigGroupBuilder) buildBatchSize() *ab.BatchSize {
@@ -319,16 +311,10 @@ func (b *MockConfigGroupBuilder) buildBatchSize() *ab.BatchSize {
 	}
 }
 
-func (b *MockConfigGroupBuilder) buildAnchorPeer() *pp.AnchorPeers {
-	ap := pp.AnchorPeer{Host: "sample-host", Port: 22}
-	return &pp.AnchorPeers{
-		AnchorPeers: []*pp.AnchorPeer{&ap},
-	}
-}
-
 func (b *MockConfigGroupBuilder) buildConsensusType() *ab.ConsensusType {
 	return &ab.ConsensusType{
-		Type: "sample-Consensus-Type",
+		Type:  "kafka",
+		State: ab.ConsensusType_STATE_NORMAL,
 	}
 }
 
@@ -344,18 +330,6 @@ func (b *MockConfigGroupBuilder) buildChannelRestrictions() *ab.ChannelRestricti
 	}
 }
 
-func (b *MockConfigGroupBuilder) buildHashingAlgorithm() *common.HashingAlgorithm {
-	return &common.HashingAlgorithm{
-		Name: "SHA2",
-	}
-}
-
-func (b *MockConfigGroupBuilder) buildBlockDataHashingStructure() *common.BlockDataHashingStructure {
-	return &common.BlockDataHashingStructure{
-		Width: 64,
-	}
-}
-
 func (b *MockConfigGroupBuilder) buildCapabilities(capabilityNames []string) *common.Capabilities {
 	capabilities := make(map[string]*common.Capability)
 	for _, capability := range capabilityNames {
@@ -366,6 +340,23 @@ func (b *MockConfigGroupBuilder) buildCapabilities(capabilityNames []string) *co
 	}
 }
 
+func (b *MockConfigGroupBuilder) buildKafkaBrokers() *ab.KafkaBrokers {
+	brokers := []string{"kafkabroker"}
+	return &ab.KafkaBrokers{
+		Brokers: brokers,
+	}
+}
+
+func (b *MockConfigGroupBuilder) buildACLs(policyRefs []string) *pp.ACLs {
+	acls := make(map[string]*pp.APIResource)
+	for _, policyRef := range policyRefs {
+		acls[policyRef] = &pp.APIResource{PolicyRef: policyRef}
+	}
+	return &pp.ACLs{
+		Acls: acls,
+	}
+}
+
 func (b *MockConfigGroupBuilder) buildMSPConfig(name string) *mb.MSPConfig {
 	return &mb.MSPConfig{
 		Type:   0,
@@ -430,9 +421,8 @@ func (b *MockConfigGroupBuilder) buildApplicationGroup() *common.ConfigGroup {
 			"Readers": b.buildSignatureConfigPolicy(),
 		},
 		Values: map[string]*common.ConfigValue{
-			channelConfig.BatchSizeKey:    b.buildBatchSizeConfigValue(),
 			channelConfig.CapabilitiesKey: b.buildCapabilitiesConfigValue(b.ApplicationCapabilities),
-			// TODO: More
+			channelConfig.ACLsKey:         b.buildACLsConfigValue(b.PolicyRefs),
 		},
 		Version:   b.Version,
 		ModPolicy: b.ModPolicy,

pkg/util/protolator/json.go

@@ -0,0 +1,26 @@
+// Copyright SecureKey Technologies Inc. All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package protolator
+
+import (
+	"io"
+
+	"github.com/golang/protobuf/proto"
+	plator "github.com/hyperledger/fabric-sdk-go/internal/github.com/hyperledger/fabric/common/tools/protolator"
+)
+
+// DeepMarshalJSON marshals msg to w as JSON, but instead of marshaling bytes fields which contain nested
+// marshaled messages as base64 (like the standard proto encoding), these nested messages are remarshaled
+// as the JSON representation of those messages.  This is done so that the JSON representation is as non-binary
+// and human readable as possible.
+func DeepMarshalJSON(w io.Writer, msg proto.Message) error {
+	return plator.DeepMarshalJSON(w, msg)
+}
+
+// DeepUnmarshalJSON takes JSON output as generated by DeepMarshalJSON and decodes it into msg
+// This includes re-marshaling the expanded nested elements to binary form
+func DeepUnmarshalJSON(r io.Reader, msg proto.Message) error {
+	return plator.DeepUnmarshalJSON(r, msg)
+}

pkg/util/protolator/json_test.go

@@ -0,0 +1,66 @@
+// Copyright SecureKey Technologies Inc. All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package protolator
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/providers/fab"
+	"github.com/hyperledger/fabric-sdk-go/pkg/fab/mocks"
+	"github.com/hyperledger/fabric-sdk-go/third_party/github.com/hyperledger/fabric/protos/common"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var cert = `-----BEGIN CERTIFICATE-----
+MIICYjCCAgmgAwIBAgIUB3CTDOU47sUC5K4kn/Caqnh114YwCgYIKoZIzj0EAwIw
+fzELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDVNh
+biBGcmFuY2lzY28xHzAdBgNVBAoTFkludGVybmV0IFdpZGdldHMsIEluYy4xDDAK
+BgNVBAsTA1dXVzEUMBIGA1UEAxMLZXhhbXBsZS5jb20wHhcNMTYxMDEyMTkzMTAw
+WhcNMjExMDExMTkzMTAwWjB/MQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZv
+cm5pYTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEfMB0GA1UEChMWSW50ZXJuZXQg
+V2lkZ2V0cywgSW5jLjEMMAoGA1UECxMDV1dXMRQwEgYDVQQDEwtleGFtcGxlLmNv
+bTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABKIH5b2JaSmqiQXHyqC+cmknICcF
+i5AddVjsQizDV6uZ4v6s+PWiJyzfA/rTtMvYAPq/yeEHpBUB1j053mxnpMujYzBh
+MA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQXZ0I9
+qp6CP8TFHZ9bw5nRtZxIEDAfBgNVHSMEGDAWgBQXZ0I9qp6CP8TFHZ9bw5nRtZxI
+EDAKBggqhkjOPQQDAgNHADBEAiAHp5Rbp9Em1G/UmKn8WsCbqDfWecVbZPQj3RK4
+oG5kQQIgQAe4OOKYhJdh3f7URaKfGTf492/nmRmtK+ySKjpHSrU=
+-----END CERTIFICATE-----
+`
+
+func TestMarshalBlock(t *testing.T) {
+
+	capability1 := "V1_1_PVTDATA_EXPERIMENTAL"
+	v2_0Capability := "V2_0"
+
+	builder := &mocks.MockConfigBlockBuilder{
+		MockConfigGroupBuilder: mocks.MockConfigGroupBuilder{
+			ModPolicy: "Admins",
+			MSPNames: []string{
+				"Org1MSP",
+				"Org2MSP",
+			},
+			OrdererAddress:          "localhost:9999",
+			RootCA:                  cert,
+			ChannelCapabilities:     []string{fab.V1_1Capability},
+			OrdererCapabilities:     []string{fab.V1_1Capability, v2_0Capability},
+			ApplicationCapabilities: []string{fab.V1_2Capability, capability1},
+		},
+		Index:           0,
+		LastConfigIndex: 0,
+	}
+
+	block := builder.Build()
+	var buf bytes.Buffer
+	err := DeepMarshalJSON(&buf, block)
+	assert.Nil(t, err, "Error marshalling block")
+
+	newBlock := &common.Block{}
+	newBlock.Reset()
+	err = DeepUnmarshalJSON(bytes.NewReader(buf.Bytes()), newBlock)
+	assert.Nil(t, err, "Error unmarshalling block")
+}

scripts/_go/src/pkcs11helper/go.sum

@@ -1,2 +1 @@
-github.com/miekg/pkcs11 v0.0.0-20190329070431-55f3fac3af27 h1:XA/VH+SzpYyukhgh7v2mTp8rZoKKITXR/x3FIizVEXs=
 github.com/miekg/pkcs11 v0.0.0-20190329070431-55f3fac3af27/go.mod h1:WCBAbTOdfhHhz7YXujeZMF7owC4tPb1naKFsgfUISjo=

scripts/third_party_pins/fabric/apply_fabric_protos_common.sh

@@ -128,6 +128,7 @@ do
     sed -i'' -e "/proto.RegisterType/s/InputId\"/${NAMESPACE_PREFIX}InputId\"/g" "${TMP_PROJECT_PATH}/${i}"
     sed -i'' -e "/proto.RegisterType/s/PlainDelegatedOutput\"/${NAMESPACE_PREFIX}PlainDelegatedOutput\"/g" "${TMP_PROJECT_PATH}/${i}"
   fi
+  sed -i'' -e "s/protobuf:\(.*\)enum=\(.*\)/protobuf:\1enum=${NAMESPACE_PREFIX}\2/g" "${TMP_PROJECT_PATH}/${i}"
 done
 
 # Copy patched project into internal paths

scripts/third_party_pins/fabric/apply_fabric_protos_network.sh

@@ -84,6 +84,7 @@ do
     sed -i'' -e "/proto.RegisterType/s/gossip/${NAMESPACE_PREFIX}gossip/g" "${TMP_PROJECT_PATH}/${i}"
     sed -i'' -e "/proto.RegisterEnum/s/gossip/${NAMESPACE_PREFIX}gossip/g" "${TMP_PROJECT_PATH}/${i}"
   fi
+  sed -i'' -e "s/protobuf:\(.*\)enum=\(.*\)/protobuf:\1enum=${NAMESPACE_PREFIX}\2/g" "${TMP_PROJECT_PATH}/${i}"
 done
 
 # Copy patched project into internal paths