diff --git a/pkg/client/channel/benchmark/README.md b/pkg/client/channel/benchmark/README.md deleted file mode 100644 index 6f4a791d70..0000000000 --- a/pkg/client/channel/benchmark/README.md +++ /dev/null @@ -1,89 +0,0 @@ -#Benchmark testing of Channel Client - This benchmark test has 1 valid call of Channel Client's Execute() function - - Under the directory where this file resides, the test commands are run as shown under the below comments: - ( - * on a Macbook Pro, warning messages are stripped out below for conciseness - * Benchmark is using Go's test command with -bench=ExecuteTx - * the -run=notest flag means execute a non-existant 'notest' in the current folder - This will avoid running normal unit tests along with the benchmarks - * by default, the benchmark tool decides when it collected enough information and stops - * the use of -benchtime=XXs forces the benchmark to keep executing until this time has elapsed - This allows the tool to run for longer times and collect more accurate information for larger execution loads - * the benchmark output format is as follows: - benchmarkname [logs from benchamark tests-They have removed from the example commands below] NbOfOperationExecutions TimeInNanoSeconds/OperationExecuted MemoryAllocated/OperationExecuted NbOfAllocations/OperationExecuted - Example from below commands: - BenchmarkExecuteTx-8 [logs removed] 100000 164854 ns/op 5743056 B/op 50449 allocs/op - - * the command output also shows the environment and the package used for the benchmark exection: - goos: darwin - goarch: amd64 - pkg: github.com/hyperledger/fabric-sdk-go/pkg/fabsdk/test/benchmark - ) - -TODO Need a more controlled benchmark about channel client (perhaps analyze perf profiling data to investigate more fine grained memory/performance issues) - -$ go test -run=notest -bench=ExecuteTx -goos: darwin -goarch: amd64 -pkg: github.com/hyperledger/fabric-sdk-go/pkg/client/channel/benchmark -BenchmarkExecuteTx-8 1000 1293015 ns/op 219451 B/op 3002 allocs/op -PASS -ok github.com/hyperledger/fabric-sdk-go/pkg/client/channel/benchmark 2.294s -$ go test -run=notest -bench=ExecuteTx -benchtime=10s -goos: darwin -goarch: amd64 -pkg: github.com/hyperledger/fabric-sdk-go/pkg/client/channel/benchmark -BenchmarkExecuteTx-8 10000 1308766 ns/op 219258 B/op 3001 allocs/op -PASS -ok github.com/hyperledger/fabric-sdk-go/pkg/client/channel/benchmark 14.031s -$ go test -run=notest -bench=ExecuteTx -benchtime=30s -goos: darwin -goarch: amd64 -pkg: github.com/hyperledger/fabric-sdk-go/pkg/client/channel/benchmark -BenchmarkExecuteTx-8 30000 1285455 ns/op 219377 B/op 3001 allocs/op -PASS -ok github.com/hyperledger/fabric-sdk-go/pkg/client/channel/benchmark 52.563s -$ go test -run=notest -bench=ExecuteTx -benchtime=60s -goos: darwin -goarch: amd64 -pkg: github.com/hyperledger/fabric-sdk-go/pkg/client/channel/benchmark -BenchmarkExecuteTx-8 100000 1373466 ns/op 219192 B/op 3001 allocs/op -PASS -ok github.com/hyperledger/fabric-sdk-go/pkg/client/channel/benchmark 151.617s -$ go test -run=notest -bench=ExecuteTx -benchtime=120s -goos: darwin -goarch: amd64 -pkg: github.com/hyperledger/fabric-sdk-go/pkg/client/channel/benchmark -BenchmarkExecuteTx-8 200000 1347777 ns/op 219278 B/op 3001 allocs/op -PASS -ok github.com/hyperledger/fabric-sdk-go/pkg/client/channel/benchmark 284.201s - - -#Benchmark CPU & Memory performance analysis - In order to generate profiling data for the chClient benchmark, the go test command can be extended to generate these. - Note: If the below command complains about cpu.out or mem.out files are missing, create these files with empty content - prior to running the command: - -go test -v -run=notest -bench=ExecuteTx -benchtime=1s -outputdir ./bench1s -cpuprofile cpu.out -memprofilerate 1 -memprofile mem.out - - once ./bench1s has a valid cpu.out and mem.out content, then we can use go pprof command to examine the perf data. - - The above command will also generate the go binary file from which the profiling data is generated (benchmark.test). - - - * For CPU perf data: -go tool pprof benchmark.test ./bench1s/cpu.out - - * For Memory - allocation data (count by number of allocation): -go tool pprof --inuse_objects benchmark.test ./bench1s/mem.out - - * For Memory - total allocation data: -go tool pprof --alloc_space benchmark.test ./bench1s/mem.out - - - to generate the PDF report from the go tool pprof cli, simply execute -pdf - - or in svg format simply run: -svg \ No newline at end of file diff --git a/pkg/fab/events/mocks/mockdelserver.go b/pkg/fab/events/mocks/mockdelserver.go index 0eb255c593..c965117677 100755 --- a/pkg/fab/events/mocks/mockdelserver.go +++ b/pkg/fab/events/mocks/mockdelserver.go @@ -25,7 +25,7 @@ type MockDeliverServer struct { // Note: the mock broadcast server should setup either deliveries or fileteredDeliveries, not both at once. // the same for mock endorser server, it should either call NewMockDeliverServerWithDeliveries or NewMockDeliverServerWithFilteredDeliveries - // to get a new instance of mockDeliveryServer + // to get a new instance of MockDeliverServer //for mocking communication with a mockBroadCastServer, this channel will receive common blocks sent by that mockBroadcastServer deliveries <-chan *cb.Block diff --git a/pkg/fab/mocks/mockbroadcastserver.go b/pkg/fab/mocks/mockbroadcastserver.go index 30df195e98..75c8ab3e10 100644 --- a/pkg/fab/mocks/mockbroadcastserver.go +++ b/pkg/fab/mocks/mockbroadcastserver.go @@ -50,6 +50,9 @@ type MockBroadcastServer struct { Deliveries chan *common.Block FilteredDeliveries chan *pb.FilteredBlock blkNum uint64 + // mutexes to ensure parallel channel deliveries are sent in sequence + delMtx sync.Mutex + filteredDelMtx sync.Mutex } // Broadcast mock broadcast @@ -99,21 +102,29 @@ func (m *MockBroadcastServer) mockBlockDelivery(payload []byte) error { return err } if m.Deliveries != nil { - block := mocks.NewBlock(chdr.ChannelId, - mocks.NewTransaction(chdr.TxId, pb.TxValidationCode_VALID, common.HeaderType_MESSAGE), - ) - // m.blkNum is used by FilteredBlock only - - m.Deliveries <- block + go func() { + m.delMtx.Lock() + defer m.delMtx.Unlock() + block := mocks.NewBlock(chdr.ChannelId, + mocks.NewTransaction(chdr.TxId, pb.TxValidationCode_VALID, common.HeaderType_MESSAGE), + ) + // m.blkNum is used by FilteredBlock only + + m.Deliveries <- block + }() } else if m.FilteredDeliveries != nil { - filteredBlock := mocks.NewFilteredBlock(chdr.ChannelId, - mocks.NewFilteredTx(chdr.TxId, pb.TxValidationCode_VALID), - ) - // increase m.blkNum to mock adding of filtered blocks to the ledger - m.blkNum++ - filteredBlock.Number = m.blkNum - - m.FilteredDeliveries <- filteredBlock + go func() { + m.filteredDelMtx.Lock() + defer m.filteredDelMtx.Unlock() + filteredBlock := mocks.NewFilteredBlock(chdr.ChannelId, + mocks.NewFilteredTx(chdr.TxId, pb.TxValidationCode_VALID), + ) + // increase m.blkNum to mock adding of filtered blocks to the ledger + m.blkNum++ + filteredBlock.Number = m.blkNum + + m.FilteredDeliveries <- filteredBlock + }() } return nil diff --git a/test/performance/pkg/client/channel/README.md b/test/performance/pkg/client/channel/README.md new file mode 100644 index 0000000000..5842666549 --- /dev/null +++ b/test/performance/pkg/client/channel/README.md @@ -0,0 +1,101 @@ +#Benchmark testing of Channel Client + This benchmark test has 1 valid call of Channel Client's Execute() function + + Under the directory where this file resides, the test commands are run as shown under the below comments: + ( + * on a Macbook Pro, warning messages are stripped out below for conciseness + * Benchmark is using Go's test command with -bench=ExecuteTx + * the -run=notest flag means execute a non-existant 'notest' in the current folder + This will avoid running normal unit tests along with the benchmarks + * by default, the benchmark tool decides when it collected enough information and stops + * the use of -benchtime=XXs forces the benchmark to keep executing until this time has elapsed + This allows the tool to run for longer times and collect more accurate information for larger execution loads + * the benchmark output format is as follows: + benchmarkname [logs from benchamark tests-They have removed from the example commands below] NbOfOperationExecutions TimeInNanoSeconds/OperationExecuted MemoryAllocated/OperationExecuted NbOfAllocations/OperationExecuted + Example from below commands: + BenchmarkExecuteTx-8 [logs removed] 100000 164854 ns/op 5743056 B/op 50449 allocs/op + + * the command output also shows the environment and the package used for the benchmark exection: + goos: darwin + goarch: amd64 + pkg: github.com/hyperledger/fabric-sdk-go/test/performance/pkg/client/channel + + UPDATE: there are now 2 benchmarks in this file, the first being the original one which runs with sequential executions and the second makes use of + parallel client executions to simulate simultanous calls from the client (command line outputs below are updated to reflect these two calls). + - To run the parallel test use -bench=ExecuteTxParallel + - to run all benchmarks use -bench=* or -bench=ExecuteTx* + + NOTE: Since the peers/orderers are mocked and running in the same process as the benchmark's, the perf data for this benchmark includes info for both + the benchmark and the mocked servers which decreases the overall performance results. To get exact performance data for this channel client of the Go SDK, + one needs to run benchmarks against a real Fabric network with peers and orderers running in Docker containers. + ) + +$ go test -run=notest -bench=ExecuteTx* +goos: darwin +goarch: amd64 +pkg: github.com/hyperledger/fabric-sdk-go/test/performance/pkg/client/channel +BenchmarkExecuteTx-8 1000 1318277 ns/op 219871 B/op 3023 allocs/op +BenchmarkExecuteTxParallel-8 3000 527525 ns/op 218158 B/op 3008 allocs/op +PASS +ok github.com/hyperledger/fabric-sdk-go/test/performance/pkg/client/channel 4.027s +$ go test -run=notest -bench=ExecuteTx* -benchtime=10s +goos: darwin +goarch: amd64 +pkg: github.com/hyperledger/fabric-sdk-go/test/performance/pkg/client/channel +BenchmarkExecuteTx-8 10000 1310724 ns/op 219738 B/op 3023 allocs/op +BenchmarkExecuteTxParallel-8 30000 475787 ns/op 218055 B/op 3008 allocs/op +PASS +ok github.com/hyperledger/fabric-sdk-go/test/performance/pkg/client/channel 37.898s +$ go test -run=notest -bench=ExecuteTx* -benchtime=30s +goos: darwin +goarch: amd64 +pkg: github.com/hyperledger/fabric-sdk-go/test/performance/pkg/client/channel +BenchmarkExecuteTx-8 30000 1308714 ns/op 219803 B/op 3023 allocs/op +BenchmarkExecuteTxParallel-8 100000 510626 ns/op 218082 B/op 3008 allocs/op +PASS +ok github.com/hyperledger/fabric-sdk-go/test/performance/pkg/client/channel 114.451s +$ go test -run=notest -bench=ExecuteTx* -benchtime=60s +goos: darwin +goarch: amd64 +pkg: github.com/hyperledger/fabric-sdk-go/test/performance/pkg/client/channel +BenchmarkExecuteTx-8 100000 1308884 ns/op 219786 B/op 3023 allocs/op +BenchmarkExecuteTxParallel-8 200000 499403 ns/op 218031 B/op 3008 allocs/op +PASS +ok github.com/hyperledger/fabric-sdk-go/test/performance/pkg/client/channel 249.928s +$ go test -run=notest -bench=ExecuteTx* -benchtime=120s +goos: darwin +goarch: amd64 +pkg: github.com/hyperledger/fabric-sdk-go/test/performance/pkg/client/channel +BenchmarkExecuteTx-8 200000 1298885 ns/op 219818 B/op 3023 allocs/op +BenchmarkExecuteTxParallel-8 500000 501436 ns/op 218011 B/op 3008 allocs/op +PASS +ok github.com/hyperledger/fabric-sdk-go/test/performance/pkg/client/channel 529.397s + + +#Benchmark CPU & Memory performance analysis + In order to generate profiling data for the chClient benchmark, the go test command can be extended to generate these. + Note: If the below command complains about cpu.out or mem.out files are missing, create these files with empty content + prior to running the command: + +go test -v -run=notest -bench=ExecuteTx -benchtime=1s -outputdir ./bench1s -cpuprofile cpu.out -memprofilerate 1 -memprofile mem.out + + once ./bench1s has a valid cpu.out and mem.out content, then we can use go pprof command to examine the perf data. + + The above command will also generate the go binary file from which the profiling data is generated (benchmark.test). + + + * For CPU perf data analysis: +go tool pprof benchmark.test ./bench1s/cpu.out + + * For Memory - allocation data analysis (count by number of allocation): +go tool pprof --inuse_objects benchmark.test ./bench1s/mem.out + + * For Memory - total allocation data analysis: +go tool pprof --alloc_space benchmark.test ./bench1s/mem.out + + + to generate the PDF report from the go tool pprof cli, simply execute: +pdf + + or in svg format simply run: +svg \ No newline at end of file diff --git a/pkg/client/channel/benchmark/chclient_fixture_test.go b/test/performance/pkg/client/channel/chclient_fixture_test.go similarity index 89% rename from pkg/client/channel/benchmark/chclient_fixture_test.go rename to test/performance/pkg/client/channel/chclient_fixture_test.go index 9c8570cd38..6e5f292637 100644 --- a/pkg/client/channel/benchmark/chclient_fixture_test.go +++ b/test/performance/pkg/client/channel/chclient_fixture_test.go @@ -4,7 +4,7 @@ Copyright SecureKey Technologies Inc. All Rights Reserved. SPDX-License-Identifier: Apache-2.0 */ -package benchmark +package channel import ( "fmt" @@ -19,10 +19,12 @@ import ( fabImpl "github.com/hyperledger/fabric-sdk-go/pkg/fab" "github.com/hyperledger/fabric-sdk-go/pkg/fabsdk" mspImpl "github.com/hyperledger/fabric-sdk-go/pkg/msp" + "github.com/hyperledger/fabric-sdk-go/pkg/util/pathvar" + "google.golang.org/grpc/testdata" ) const ( - configPath = "../../../core/config/testdata/config_test.yaml" + configPath = "${GOPATH}/src/github.com/hyperledger/fabric-sdk-go/pkg/core/config/testdata/config_test.yaml" ) type testFixture struct { @@ -34,7 +36,7 @@ type testFixture struct { func (f *testFixture) setup() (*fabsdk.FabricSDK, context.Client) { var err error - backend, err := config.FromFile(configPath)() + backend, err := config.FromFile(testdata.Path(pathvar.Subst(configPath)))() if err != nil { panic(err) } diff --git a/pkg/client/channel/benchmark/chclient_test.go b/test/performance/pkg/client/channel/chclient_test.go similarity index 86% rename from pkg/client/channel/benchmark/chclient_test.go rename to test/performance/pkg/client/channel/chclient_test.go index 559b2235bd..1d64316ad9 100644 --- a/pkg/client/channel/benchmark/chclient_test.go +++ b/test/performance/pkg/client/channel/chclient_test.go @@ -4,7 +4,7 @@ Copyright SecureKey Technologies Inc. All Rights Reserved. SPDX-License-Identifier: Apache-2.0 */ -package benchmark +package channel import ( "fmt" @@ -48,6 +48,9 @@ var chClient *channel.Client var fixture *testFixture var ordererMockSrv *fcmocks.MockBroadcastServer var mockEndorserServer *MockEndorserServer +var chRq = channel.Request{ChaincodeID: "testCC", Fcn: "invoke", Args: [][]byte{[]byte("move"), []byte("b")}} +var endorserURL = fmt.Sprintf("%s:%d", testhost, testport) +var ordererURL = fmt.Sprintf("%s:%d", testBrodcasthost, testBroadcastport) func BenchmarkExecuteTx(b *testing.B) { // report memory allocations for this benchmark @@ -56,13 +59,30 @@ func BenchmarkExecuteTx(b *testing.B) { // using channel Client, let's start the benchmark b.ResetTimer() for n := 0; n < b.N; n++ { - chClient.Execute(channel.Request{ChaincodeID: "testCC", Fcn: "invoke", Args: [][]byte{[]byte("move"), []byte("b")}}) + chClient.Execute(chRq) //require.NoError(b, err, "expected no error for valid channel client invoke") //b.Logf("Execute Responses: %s", resp.Responses) } } +func BenchmarkExecuteTxParallel(b *testing.B) { + // report memory allocations for this benchmark + b.ReportAllocs() + + // using channel Client, let's start the benchmark + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + chClient.Execute(chRq) + //require.NoError(b, err, "expected no error for valid channel client parallel invoke") + + //b.Logf("Execute Responses: %s", resp.Responses) + } + }) +} + func TestMain(m *testing.M) { setUp(m) r := m.Run() @@ -87,15 +107,15 @@ func setUp(m *testing.M) { // setup mocked peer mockEndorserServer = &MockEndorserServer{Creds: creds} - mockEndorserServer.SetMockPeer(&MockPeer{MockName: "Peer1", MockURL: fmt.Sprintf("%s:%d", testhost, testport), MockRoles: []string{}, MockCert: nil, MockMSP: "Org1MSP", Status: 200, + mockEndorserServer.SetMockPeer(&MockPeer{MockName: "Peer1", MockURL: endorserURL, MockRoles: []string{}, MockCert: nil, MockMSP: "Org1MSP", Status: 200, Payload: payloadMap}) // create a delivery channel for the orderer and the deliveryservice d := make(chan *pb.FilteredBlock, 10) - fmt.Println("***************** Mocked Peer Started: ", mockEndorserServer.Start(fmt.Sprintf("%s:%d", testhost, testport), d), " ******************************") + fmt.Println("***************** Mocked Peer Started: ", mockEndorserServer.Start(endorserURL, d), " ******************************") - // setup mocked CA + // setup real client sdk and context (no mocks for the client side) fixture = &testFixture{} var ctx context.Client sdkClient, ctx = fixture.setup() @@ -109,15 +129,15 @@ func setUp(m *testing.M) { panic(fmt.Sprintf("Failed to create new orderer tls creds from file: %s", err)) } - // create a mock orderer broadCast server with a filteredDelivereies channel that communicates with a delivery server + // create a mock ordererBroadcastServer with a filteredDeliveries channel that communicates with a delivery server ordererMockSrv = &fcmocks.MockBroadcastServer{Creds: creds, FilteredDeliveries: d} - fmt.Println("***************** Mocked Orderer Started: ", ordererMockSrv.Start(fmt.Sprintf("%s:%d", testBrodcasthost, testBroadcastport)), " ******************************") + fmt.Println("***************** Mocked Orderer Started: ", ordererMockSrv.Start(ordererURL), " ******************************") chClient = setupChannelClient(fixture.endpointConfig, ctx) } func teardown() { - // do any teadown activities here .. + // do any teardown activities here .. sdkClient.Close() mockEndorserServer.Stop() fixture.close() diff --git a/pkg/client/channel/benchmark/mockendorserserver.go b/test/performance/pkg/client/channel/mockendorserserver.go similarity index 99% rename from pkg/client/channel/benchmark/mockendorserserver.go rename to test/performance/pkg/client/channel/mockendorserserver.go index 208aca4ce8..b39a4eb65a 100644 --- a/pkg/client/channel/benchmark/mockendorserserver.go +++ b/test/performance/pkg/client/channel/mockendorserserver.go @@ -4,7 +4,7 @@ Copyright SecureKey Technologies Inc. All Rights Reserved. SPDX-License-Identifier: Apache-2.0 */ -package benchmark +package channel import ( "crypto/ecdsa"