From d7046a18d15acf488a7309b3cf13adcea5108750 Mon Sep 17 00:00:00 2001 From: Fedor Partanskiy Date: Sat, 24 Aug 2024 02:14:02 +0300 Subject: [PATCH] replacing old functions that are not in the new protobuf package Signed-off-by: Fedor Partanskiy --- .../ledger/blkstorage/block_serialization.go | 64 +++++++------------ .../blkstorage/block_serialization_test.go | 7 +- common/ledger/blkstorage/block_stream.go | 6 +- common/ledger/blkstorage/block_stream_test.go | 4 +- .../blkstorage/blockfile_helper_test.go | 4 +- common/ledger/blkstorage/blockfile_mgr.go | 64 ++++++++++--------- .../ledger/blkstorage/blockfile_mgr_test.go | 8 +-- common/ledger/blkstorage/blockindex.go | 52 +++++++-------- common/ledger/blkstorage/protobuf_util.go | 31 ++++----- .../ledger/blkstorage/protobuf_util_test.go | 22 +++---- common/ledger/blkstorage/reset_test.go | 4 +- common/ledger/util/util.go | 9 ++- common/ledger/util/util_test.go | 8 +-- core/ledger/kvledger/kv_ledger.go | 3 +- .../privacyenabledstate/snapshot_test.go | 43 ++++++++----- core/ledger/pvtdatastorage/kv_encoding.go | 21 ++++-- core/ledger/pvtdatastorage/store.go | 32 ++++++---- core/peer/configtx_test.go | 8 +-- .../rest/protolator_handlers_test.go | 9 +-- internal/pkg/gateway/registry.go | 5 +- orderer/consensus/etcdraft/node.go | 7 +- 21 files changed, 217 insertions(+), 194 deletions(-) diff --git a/common/ledger/blkstorage/block_serialization.go b/common/ledger/blkstorage/block_serialization.go index 5f3f94fed83..15b9ffe7087 100644 --- a/common/ledger/blkstorage/block_serialization.go +++ b/common/ledger/blkstorage/block_serialization.go @@ -7,10 +7,10 @@ SPDX-License-Identifier: Apache-2.0 package blkstorage import ( - "github.com/golang/protobuf/proto" "github.com/hyperledger/fabric-protos-go/common" "github.com/hyperledger/fabric/protoutil" "github.com/pkg/errors" + "google.golang.org/protobuf/encoding/protowire" ) type serializedBlockInfo struct { @@ -26,21 +26,14 @@ type txindexInfo struct { } func serializeBlock(block *common.Block) ([]byte, *serializedBlockInfo, error) { - buf := proto.NewBuffer(nil) - var err error + var buf []byte info := &serializedBlockInfo{} info.blockHeader = block.Header info.metadata = block.Metadata - if err = addHeaderBytes(block.Header, buf); err != nil { - return nil, nil, err - } - if info.txOffsets, err = addDataBytesAndConstructTxIndexInfo(block.Data, buf); err != nil { - return nil, nil, err - } - if err = addMetadataBytes(block.Metadata, buf); err != nil { - return nil, nil, err - } - return buf.Bytes(), info, nil + buf = addHeaderBytes(block.Header, buf) + info.txOffsets, buf = addDataBytesAndConstructTxIndexInfo(block.Data, buf) + buf = addMetadataBytes(block.Metadata, buf) + return buf, info, nil } func deserializeBlock(serializedBlockBytes []byte) (*common.Block, error) { @@ -79,55 +72,44 @@ func extractSerializedBlockInfo(serializedBlockBytes []byte) (*serializedBlockIn return info, nil } -func addHeaderBytes(blockHeader *common.BlockHeader, buf *proto.Buffer) error { - if err := buf.EncodeVarint(blockHeader.Number); err != nil { - return errors.Wrapf(err, "error encoding the block number [%d]", blockHeader.Number) - } - if err := buf.EncodeRawBytes(blockHeader.DataHash); err != nil { - return errors.Wrapf(err, "error encoding the data hash [%v]", blockHeader.DataHash) - } - if err := buf.EncodeRawBytes(blockHeader.PreviousHash); err != nil { - return errors.Wrapf(err, "error encoding the previous hash [%v]", blockHeader.PreviousHash) - } - return nil +func addHeaderBytes(blockHeader *common.BlockHeader, buf []byte) []byte { + buf = protowire.AppendVarint(buf, blockHeader.Number) + buf = protowire.AppendBytes(buf, blockHeader.DataHash) + buf = protowire.AppendBytes(buf, blockHeader.PreviousHash) + return buf } -func addDataBytesAndConstructTxIndexInfo(blockData *common.BlockData, buf *proto.Buffer) ([]*txindexInfo, error) { +func addDataBytesAndConstructTxIndexInfo(blockData *common.BlockData, buf []byte) ([]*txindexInfo, []byte) { var txOffsets []*txindexInfo - if err := buf.EncodeVarint(uint64(len(blockData.Data))); err != nil { - return nil, errors.Wrap(err, "error encoding the length of block data") - } + buf = protowire.AppendVarint(buf, uint64(len(blockData.Data))) for _, txEnvelopeBytes := range blockData.Data { - offset := len(buf.Bytes()) + offset := len(buf) txid, err := protoutil.GetOrComputeTxIDFromEnvelope(txEnvelopeBytes) if err != nil { logger.Warningf("error while extracting txid from tx envelope bytes during serialization of block. Ignoring this error as this is caused by a malformed transaction. Error:%s", err) } - if err := buf.EncodeRawBytes(txEnvelopeBytes); err != nil { - return nil, errors.Wrap(err, "error encoding the transaction envelope") - } - idxInfo := &txindexInfo{txID: txid, loc: &locPointer{offset, len(buf.Bytes()) - offset}} + buf = protowire.AppendBytes(buf, txEnvelopeBytes) + idxInfo := &txindexInfo{txID: txid, loc: &locPointer{offset, len(buf) - offset}} txOffsets = append(txOffsets, idxInfo) } - return txOffsets, nil + return txOffsets, buf } -func addMetadataBytes(blockMetadata *common.BlockMetadata, buf *proto.Buffer) error { +func addMetadataBytes(blockMetadata *common.BlockMetadata, buf []byte) []byte { numItems := uint64(0) if blockMetadata != nil { numItems = uint64(len(blockMetadata.Metadata)) } - if err := buf.EncodeVarint(numItems); err != nil { - return errors.Wrap(err, "error encoding the length of metadata") + buf = protowire.AppendVarint(buf, numItems) + if blockMetadata == nil { + return buf } for _, b := range blockMetadata.Metadata { - if err := buf.EncodeRawBytes(b); err != nil { - return errors.Wrap(err, "error encoding the block metadata") - } + buf = protowire.AppendBytes(buf, b) } - return nil + return buf } func extractHeader(buf *buffer) (*common.BlockHeader, error) { diff --git a/common/ledger/blkstorage/block_serialization_test.go b/common/ledger/blkstorage/block_serialization_test.go index 38e4f1b0cf6..a861133e7f4 100644 --- a/common/ledger/blkstorage/block_serialization_test.go +++ b/common/ledger/blkstorage/block_serialization_test.go @@ -9,11 +9,11 @@ package blkstorage import ( "testing" - "github.com/golang/protobuf/proto" "github.com/hyperledger/fabric-protos-go/common" "github.com/hyperledger/fabric/common/ledger/testutil" "github.com/hyperledger/fabric/protoutil" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" ) func TestBlockSerialization(t *testing.T) { @@ -101,7 +101,10 @@ func testSerializedBlockInfo(t *testing.T, block *common.Block, c *testutilTxIDC require.Equal(t, indexTxID, txid) b := bb[indexOffset.offset:] - length, num := proto.DecodeVarint(b) + length, num := protowire.ConsumeVarint(b) + if num < 0 { + length, num = 0, 0 + } txEnvBytesFromBB := b[num : num+int(length)] require.Equal(t, txEnvBytes, txEnvBytesFromBB) } diff --git a/common/ledger/blkstorage/block_stream.go b/common/ledger/blkstorage/block_stream.go index f38851012bf..10ed5ddbf66 100644 --- a/common/ledger/blkstorage/block_stream.go +++ b/common/ledger/blkstorage/block_stream.go @@ -12,8 +12,8 @@ import ( "io" "os" - "github.com/golang/protobuf/proto" "github.com/pkg/errors" + "google.golang.org/protobuf/encoding/protowire" ) // ErrUnexpectedEndOfBlockfile error used to indicate an unexpected end of a file segment @@ -105,8 +105,8 @@ func (s *blockfileStream) nextBlockBytesAndPlacementInfo() ([]byte, *blockPlacem if lenBytes, err = s.reader.Peek(peekBytes); err != nil { return nil, nil, errors.Wrapf(err, "error peeking [%d] bytes from block file", peekBytes) } - length, n := proto.DecodeVarint(lenBytes) - if n == 0 { + length, n := protowire.ConsumeVarint(lenBytes) + if n <= 0 { // proto.DecodeVarint did not consume any byte at all which means that the bytes // representing the size of the block are partial bytes if !moreContentAvailable { diff --git a/common/ledger/blkstorage/block_stream_test.go b/common/ledger/blkstorage/block_stream_test.go index 83a83f682ad..cb83826b600 100644 --- a/common/ledger/blkstorage/block_stream_test.go +++ b/common/ledger/blkstorage/block_stream_test.go @@ -9,10 +9,10 @@ package blkstorage import ( "testing" - "github.com/golang/protobuf/proto" "github.com/hyperledger/fabric-protos-go/common" "github.com/hyperledger/fabric/common/ledger/testutil" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" ) func TestBlockfileStream(t *testing.T) { @@ -53,7 +53,7 @@ func testBlockfileStream(t *testing.T, numBlocks int) { func TestBlockFileStreamUnexpectedEOF(t *testing.T) { partialBlockBytes := []byte{} dummyBlockBytes := testutil.ConstructRandomBytes(t, 100) - lenBytes := proto.EncodeVarint(uint64(len(dummyBlockBytes))) + lenBytes := protowire.AppendVarint(nil, uint64(len(dummyBlockBytes))) partialBlockBytes = append(partialBlockBytes, lenBytes...) partialBlockBytes = append(partialBlockBytes, dummyBlockBytes...) testBlockFileStreamUnexpectedEOF(t, 10, partialBlockBytes[:1]) diff --git a/common/ledger/blkstorage/blockfile_helper_test.go b/common/ledger/blkstorage/blockfile_helper_test.go index 39b1c48d357..7d8860b8c7d 100644 --- a/common/ledger/blkstorage/blockfile_helper_test.go +++ b/common/ledger/blkstorage/blockfile_helper_test.go @@ -12,9 +12,9 @@ import ( "path/filepath" "testing" - "github.com/golang/protobuf/proto" "github.com/hyperledger/fabric/common/ledger/testutil" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" ) func TestConstructBlockfilesInfo(t *testing.T) { @@ -64,7 +64,7 @@ func TestConstructBlockfilesInfo(t *testing.T) { lastTestBlk := bg.NextTestBlocks(1)[0] blockBytes, _, err := serializeBlock(lastTestBlk) require.NoError(t, err) - partialByte := append(proto.EncodeVarint(uint64(len(blockBytes))), blockBytes[len(blockBytes)/2:]...) + partialByte := append(protowire.AppendVarint(nil, uint64(len(blockBytes))), blockBytes[len(blockBytes)/2:]...) blockfileMgr.currentFileWriter.append(partialByte, true) checkBlockfilesInfoFromFS(t, blkStoreDir, blockfileMgr.blockfilesInfo) diff --git a/common/ledger/blkstorage/blockfile_mgr.go b/common/ledger/blkstorage/blockfile_mgr.go index fbcd6b2054c..257146be117 100644 --- a/common/ledger/blkstorage/blockfile_mgr.go +++ b/common/ledger/blkstorage/blockfile_mgr.go @@ -21,6 +21,7 @@ import ( "github.com/hyperledger/fabric/internal/fileutil" "github.com/hyperledger/fabric/protoutil" "github.com/pkg/errors" + "google.golang.org/protobuf/encoding/protowire" ) const ( @@ -195,7 +196,7 @@ func bootstrapFromSnapshottedTxIDs( return err } - if err := fileutil.CreateAndSyncFileAtomically( + if err = fileutil.CreateAndSyncFileAtomically( rootDir, bootstrappingSnapshotInfoTempFile, bootstrappingSnapshotInfoFile, @@ -307,7 +308,7 @@ func (mgr *blockfileMgr) addBlock(block *common.Block) error { currentOffset := mgr.blockfilesInfo.latestFileSize blockBytesLen := len(blockBytes) - blockBytesEncodedLen := proto.EncodeVarint(uint64(blockBytesLen)) + blockBytesEncodedLen := protowire.AppendVarint(nil, uint64(blockBytesLen)) totalBytesToAppend := blockBytesLen + len(blockBytesEncodedLen) // Determine if we need to start a new file since the size of this block @@ -649,7 +650,10 @@ func (mgr *blockfileMgr) fetchTransactionEnvelope(lp *fileLocPointer) (*common.E if txEnvelopeBytes, err = mgr.fetchRawBytes(lp); err != nil { return nil, err } - _, n := proto.DecodeVarint(txEnvelopeBytes) + _, n := protowire.ConsumeVarint(txEnvelopeBytes) + if n < 0 { + n = 0 + } return protoutil.GetEnvelopeFromBlock(txEnvelopeBytes[n:]) } @@ -757,49 +761,49 @@ type blockfilesInfo struct { } func (i *blockfilesInfo) marshal() ([]byte, error) { - buffer := proto.NewBuffer([]byte{}) - var err error - if err = buffer.EncodeVarint(uint64(i.latestFileNumber)); err != nil { - return nil, errors.Wrapf(err, "error encoding the latestFileNumber [%d]", i.latestFileNumber) - } - if err = buffer.EncodeVarint(uint64(i.latestFileSize)); err != nil { - return nil, errors.Wrapf(err, "error encoding the latestFileSize [%d]", i.latestFileSize) - } - if err = buffer.EncodeVarint(i.lastPersistedBlock); err != nil { - return nil, errors.Wrapf(err, "error encoding the lastPersistedBlock [%d]", i.lastPersistedBlock) - } + var buf []byte + buf = protowire.AppendVarint(buf, uint64(i.latestFileNumber)) + buf = protowire.AppendVarint(buf, uint64(i.latestFileSize)) + buf = protowire.AppendVarint(buf, i.lastPersistedBlock) var noBlockFilesMarker uint64 if i.noBlockFiles { noBlockFilesMarker = 1 } - if err = buffer.EncodeVarint(noBlockFilesMarker); err != nil { - return nil, errors.Wrapf(err, "error encoding noBlockFiles [%d]", noBlockFilesMarker) - } - return buffer.Bytes(), nil + buf = protowire.AppendVarint(buf, noBlockFilesMarker) + + return buf, nil } func (i *blockfilesInfo) unmarshal(b []byte) error { - buffer := proto.NewBuffer(b) - var val uint64 - var noBlockFilesMarker uint64 - var err error + var ( + val uint64 + noBlockFilesMarker uint64 + position int + ) - if val, err = buffer.DecodeVarint(); err != nil { - return err + val, n := protowire.ConsumeVarint(b[position:]) + if n < 0 { + return protowire.ParseError(n) } + position += n i.latestFileNumber = int(val) - if val, err = buffer.DecodeVarint(); err != nil { - return err + val, n = protowire.ConsumeVarint(b[position:]) + if n < 0 { + return protowire.ParseError(n) } + position += n i.latestFileSize = int(val) - if val, err = buffer.DecodeVarint(); err != nil { - return err + val, n = protowire.ConsumeVarint(b[position:]) + if n < 0 { + return protowire.ParseError(n) } + position += n i.lastPersistedBlock = val - if noBlockFilesMarker, err = buffer.DecodeVarint(); err != nil { - return err + noBlockFilesMarker, n = protowire.ConsumeVarint(b[position:]) + if n < 0 { + return protowire.ParseError(n) } i.noBlockFiles = noBlockFilesMarker == 1 return nil diff --git a/common/ledger/blkstorage/blockfile_mgr_test.go b/common/ledger/blkstorage/blockfile_mgr_test.go index 4d16f9dcaf7..cf092dedaa9 100644 --- a/common/ledger/blkstorage/blockfile_mgr_test.go +++ b/common/ledger/blkstorage/blockfile_mgr_test.go @@ -11,13 +11,13 @@ import ( "os" "testing" - "github.com/golang/protobuf/proto" "github.com/hyperledger/fabric-protos-go/common" "github.com/hyperledger/fabric-protos-go/peer" "github.com/hyperledger/fabric/common/ledger/testutil" "github.com/hyperledger/fabric/internal/pkg/txflags" "github.com/hyperledger/fabric/protoutil" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" ) func TestBlockfileMgrBlockReadWrite(t *testing.T) { @@ -101,7 +101,7 @@ func testBlockfileMgrCrashDuringWriting(t *testing.T, numBlksBeforeSavingBlkfile // simulate a crash scenario lastBlockBytes := []byte{} - encodedLen := proto.EncodeVarint(uint64(numLastBlockBytes)) + encodedLen := protowire.AppendVarint(nil, uint64(numLastBlockBytes)) randomBytes := testutil.ConstructRandomBytes(t, numLastBlockBytes) lastBlockBytes = append(lastBlockBytes, encodedLen...) lastBlockBytes = append(lastBlockBytes, randomBytes...) @@ -397,7 +397,7 @@ func TestBlockfileMgrFileRolling(t *testing.T) { by, _, err := serializeBlock(block) require.NoError(t, err, "Error while serializing block") blockBytesSize := len(by) - encodedLen := proto.EncodeVarint(uint64(blockBytesSize)) + encodedLen := protowire.AppendVarint(nil, uint64(blockBytesSize)) size += blockBytesSize + len(encodedLen) } @@ -466,7 +466,7 @@ func testBlockfileMgrSimulateCrashAtFirstBlockInFile(t *testing.T, deleteBlkfile // move to next file and simulate crash scenario while writing the first block blockfileMgr.moveToNextFile() partialBytesForNextBlock := append( - proto.EncodeVarint(uint64(10000)), + protowire.AppendVarint(nil, uint64(10000)), []byte("partialBytesForNextBlock depicting a crash during first block in file")..., ) blockfileMgr.currentFileWriter.append(partialBytesForNextBlock, true) diff --git a/common/ledger/blkstorage/blockindex.go b/common/ledger/blkstorage/blockindex.go index 8e4c5a6743d..be392b130ab 100644 --- a/common/ledger/blkstorage/blockindex.go +++ b/common/ledger/blkstorage/blockindex.go @@ -20,6 +20,7 @@ import ( "github.com/hyperledger/fabric/common/ledger/util/leveldbhelper" "github.com/hyperledger/fabric/internal/pkg/txflags" "github.com/pkg/errors" + "google.golang.org/protobuf/encoding/protowire" ) const ( @@ -478,11 +479,14 @@ func constructBlockNumTranNumKey(blockNum uint64, txNum uint64) []byte { } func encodeBlockNum(blockNum uint64) []byte { - return proto.EncodeVarint(blockNum) + return protowire.AppendVarint(nil, blockNum) } func decodeBlockNum(blockNumBytes []byte) uint64 { - blockNum, _ := proto.DecodeVarint(blockNumBytes) + blockNum, num := protowire.ConsumeVarint(blockNumBytes) + if num < 0 { + return 0 + } return blockNum } @@ -510,38 +514,34 @@ func newFileLocationPointer(fileSuffixNum int, beginningOffset int, relativeLP * } func (flp *fileLocPointer) marshal() ([]byte, error) { - buffer := proto.NewBuffer([]byte{}) - e := buffer.EncodeVarint(uint64(flp.fileSuffixNum)) - if e != nil { - return nil, errors.Wrapf(e, "unexpected error while marshaling fileLocPointer [%s]", flp) - } - e = buffer.EncodeVarint(uint64(flp.offset)) - if e != nil { - return nil, errors.Wrapf(e, "unexpected error while marshaling fileLocPointer [%s]", flp) - } - e = buffer.EncodeVarint(uint64(flp.bytesLength)) - if e != nil { - return nil, errors.Wrapf(e, "unexpected error while marshaling fileLocPointer [%s]", flp) - } - return buffer.Bytes(), nil + var buf []byte + buf = protowire.AppendVarint(buf, uint64(flp.fileSuffixNum)) + buf = protowire.AppendVarint(buf, uint64(flp.offset)) + buf = protowire.AppendVarint(buf, uint64(flp.bytesLength)) + + return buf, nil } func (flp *fileLocPointer) unmarshal(b []byte) error { - buffer := proto.NewBuffer(b) - i, e := buffer.DecodeVarint() - if e != nil { - return errors.Wrapf(e, "unexpected error while unmarshalling bytes [%#v] into fileLocPointer", b) + var position int + + i, n := protowire.ConsumeVarint(b[position:]) + if n < 0 { + return errors.Wrapf(protowire.ParseError(n), "unexpected error while unmarshalling bytes [%#v] into fileLocPointer", b) } + position += n flp.fileSuffixNum = int(i) - i, e = buffer.DecodeVarint() - if e != nil { - return errors.Wrapf(e, "unexpected error while unmarshalling bytes [%#v] into fileLocPointer", b) + i, n = protowire.ConsumeVarint(b[position:]) + if n < 0 { + return errors.Wrapf(protowire.ParseError(n), "unexpected error while unmarshalling bytes [%#v] into fileLocPointer", b) } + position += n flp.offset = int(i) - i, e = buffer.DecodeVarint() - if e != nil { - return errors.Wrapf(e, "unexpected error while unmarshalling bytes [%#v] into fileLocPointer", b) + + i, n = protowire.ConsumeVarint(b[position:]) + if n < 0 { + return errors.Wrapf(protowire.ParseError(n), "unexpected error while unmarshalling bytes [%#v] into fileLocPointer", b) } flp.bytesLength = int(i) return nil diff --git a/common/ledger/blkstorage/protobuf_util.go b/common/ledger/blkstorage/protobuf_util.go index 3ebddc33806..38dfd7c0614 100644 --- a/common/ledger/blkstorage/protobuf_util.go +++ b/common/ledger/blkstorage/protobuf_util.go @@ -7,42 +7,43 @@ SPDX-License-Identifier: Apache-2.0 package blkstorage import ( - "github.com/golang/protobuf/proto" "github.com/pkg/errors" + "google.golang.org/protobuf/encoding/protowire" ) // buffer provides a wrapper on top of proto.Buffer. // The purpose of this wrapper is to get to know the current position in the []byte type buffer struct { - buf *proto.Buffer + buf []byte position int } // newBuffer constructs a new instance of Buffer func newBuffer(b []byte) *buffer { - return &buffer{proto.NewBuffer(b), 0} + return &buffer{b, 0} } // DecodeVarint wraps the actual method and updates the position func (b *buffer) DecodeVarint() (uint64, error) { - val, err := b.buf.DecodeVarint() - if err == nil { - b.position += proto.SizeVarint(val) - } else { - err = errors.Wrap(err, "error decoding varint with proto.Buffer") + v, n := protowire.ConsumeVarint(b.buf[b.position:]) + if n < 0 { + return 0, errors.Wrap(protowire.ParseError(n), "error decoding varint with proto.Buffer") } - return val, err + b.position += n + return v, nil } // DecodeRawBytes wraps the actual method and updates the position func (b *buffer) DecodeRawBytes(alloc bool) ([]byte, error) { - val, err := b.buf.DecodeRawBytes(alloc) - if err == nil { - b.position += proto.SizeVarint(uint64(len(val))) + len(val) - } else { - err = errors.Wrap(err, "error decoding raw bytes with proto.Buffer") + v, n := protowire.ConsumeBytes(b.buf[b.position:]) + if n < 0 { + return nil, errors.Wrap(protowire.ParseError(n), "error decoding raw bytes with proto.Buffer") } - return val, err + b.position += n + if alloc { + v = append([]byte(nil), v...) + } + return v, nil } // GetBytesConsumed returns the offset of the current position in the underlying []byte diff --git a/common/ledger/blkstorage/protobuf_util_test.go b/common/ledger/blkstorage/protobuf_util_test.go index 90ad742458c..b9ff46f9050 100644 --- a/common/ledger/blkstorage/protobuf_util_test.go +++ b/common/ledger/blkstorage/protobuf_util_test.go @@ -9,22 +9,22 @@ package blkstorage import ( "testing" - "github.com/golang/protobuf/proto" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" ) func TestBuffer(t *testing.T) { - pb := proto.NewBuffer(nil) - pb.EncodeVarint(10) - pos1 := len(pb.Bytes()) - pb.EncodeRawBytes([]byte("JunkText")) - pos2 := len(pb.Bytes()) - pb.EncodeRawBytes([]byte("YetAnotherJunkText")) - pos3 := len(pb.Bytes()) - pb.EncodeVarint(1000000) - pos4 := len(pb.Bytes()) + var pb []byte + pb = protowire.AppendVarint(pb, 10) + pos1 := len(pb) + pb = protowire.AppendBytes(pb, []byte("JunkText")) + pos2 := len(pb) + pb = protowire.AppendBytes(pb, []byte("YetAnotherJunkText")) + pos3 := len(pb) + pb = protowire.AppendVarint(pb, 1000000) + pos4 := len(pb) - b := newBuffer(pb.Bytes()) + b := newBuffer(pb) b.DecodeVarint() require.Equal(t, pos1, b.GetBytesConsumed()) b.DecodeRawBytes(false) diff --git a/common/ledger/blkstorage/reset_test.go b/common/ledger/blkstorage/reset_test.go index 33a6e0bb47c..35b03ca546b 100644 --- a/common/ledger/blkstorage/reset_test.go +++ b/common/ledger/blkstorage/reset_test.go @@ -12,11 +12,11 @@ import ( "testing" "github.com/davecgh/go-spew/spew" - "github.com/golang/protobuf/proto" "github.com/hyperledger/fabric-protos-go/common" "github.com/hyperledger/fabric/common/ledger/testutil" "github.com/hyperledger/fabric/protoutil" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" ) func TestResetToGenesisBlkSingleBlkFile(t *testing.T) { @@ -259,7 +259,7 @@ func testutilEstimateTotalSizeOnDisk(t *testing.T, blocks []*common.Block) int { by, _, err := serializeBlock(block) require.NoError(t, err) blockBytesSize := len(by) - encodedLen := proto.EncodeVarint(uint64(blockBytesSize)) + encodedLen := protowire.AppendVarint(nil, uint64(blockBytesSize)) size += blockBytesSize + len(encodedLen) } return size diff --git a/common/ledger/util/util.go b/common/ledger/util/util.go index fea296a244a..d368e7e647e 100644 --- a/common/ledger/util/util.go +++ b/common/ledger/util/util.go @@ -20,8 +20,8 @@ import ( "encoding/binary" "fmt" - "github.com/golang/protobuf/proto" "github.com/pkg/errors" + "google.golang.org/protobuf/encoding/protowire" ) // EncodeOrderPreservingVarUint64 returns a byte-representation for a uint64 number such that @@ -41,7 +41,7 @@ func EncodeOrderPreservingVarUint64(number uint64) []byte { break } } - sizeBytes := proto.EncodeVarint(uint64(size)) + sizeBytes := protowire.AppendVarint(nil, uint64(size)) if len(sizeBytes) > 1 { panic(fmt.Errorf("[]sizeBytes should not be more than one byte because the max number it needs to hold is 8. size=%d", size)) } @@ -54,7 +54,10 @@ func EncodeOrderPreservingVarUint64(number uint64) []byte { // DecodeOrderPreservingVarUint64 decodes the number from the bytes obtained from method 'EncodeOrderPreservingVarUint64'. // It returns the decoded number, the number of bytes that are consumed in the process, and an error if the input bytes are invalid. func DecodeOrderPreservingVarUint64(bytes []byte) (uint64, int, error) { - s, numBytes := proto.DecodeVarint(bytes) + s, numBytes := protowire.ConsumeVarint(bytes) + if numBytes < 0 { + s, numBytes = 0, 0 + } switch { case numBytes != 1: diff --git a/common/ledger/util/util_test.go b/common/ledger/util/util_test.go index 8047321d1ce..506178461b8 100644 --- a/common/ledger/util/util_test.go +++ b/common/ledger/util/util_test.go @@ -21,8 +21,8 @@ import ( "fmt" "testing" - "github.com/golang/protobuf/proto" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" ) func TestBasicEncodingDecoding(t *testing.T) { @@ -62,7 +62,7 @@ func TestDecodingAppendedValues(t *testing.T) { func TestDecodingBadInputBytes(t *testing.T) { // error case when num consumed bytes > 1 - sizeBytes := proto.EncodeVarint(uint64(1000)) + sizeBytes := protowire.AppendVarint(nil, uint64(1000)) _, _, err := DecodeOrderPreservingVarUint64(sizeBytes) require.Equal(t, fmt.Sprintf("number of consumed bytes from DecodeVarint is invalid, expected 1, but got %d", len(sizeBytes)), err.Error()) @@ -72,12 +72,12 @@ func TestDecodingBadInputBytes(t *testing.T) { require.Equal(t, "number of consumed bytes from DecodeVarint is invalid, expected 1, but got 0", err.Error()) // error case when size is more than available bytes - inputBytes := proto.EncodeVarint(uint64(8)) + inputBytes := protowire.AppendVarint(nil, uint64(8)) _, _, err = DecodeOrderPreservingVarUint64(inputBytes) require.Equal(t, "decoded size (8) from DecodeVarint is more than available bytes (0)", err.Error()) // error case when size is greater than 8 - bigSizeBytes := proto.EncodeVarint(uint64(12)) + bigSizeBytes := protowire.AppendVarint(nil, uint64(12)) _, _, err = DecodeOrderPreservingVarUint64(bigSizeBytes) require.Equal(t, "decoded size from DecodeVarint is invalid, expected <=8, but got 12", err.Error()) } diff --git a/core/ledger/kvledger/kv_ledger.go b/core/ledger/kvledger/kv_ledger.go index ce87ec3c169..a73828687d5 100644 --- a/core/ledger/kvledger/kv_ledger.go +++ b/core/ledger/kvledger/kv_ledger.go @@ -36,6 +36,7 @@ import ( "github.com/hyperledger/fabric/internal/pkg/txflags" "github.com/hyperledger/fabric/protoutil" "github.com/pkg/errors" + "google.golang.org/protobuf/encoding/protowire" ) var logger = flogging.MustGetLogger("kvledger") @@ -811,7 +812,7 @@ func (l *kvLedger) addBlockCommitHash(block *common.Block, updateBatchBytes []by var valueBytes []byte txValidationCode := block.Metadata.Metadata[common.BlockMetadataIndex_TRANSACTIONS_FILTER] - valueBytes = append(valueBytes, proto.EncodeVarint(uint64(len(txValidationCode)))...) + valueBytes = append(valueBytes, protowire.AppendVarint(nil, uint64(len(txValidationCode)))...) valueBytes = append(valueBytes, txValidationCode...) valueBytes = append(valueBytes, updateBatchBytes...) valueBytes = append(valueBytes, l.commitHash...) diff --git a/core/ledger/kvledger/txmgmt/privacyenabledstate/snapshot_test.go b/core/ledger/kvledger/txmgmt/privacyenabledstate/snapshot_test.go index 8cbc64b9806..c913e818a21 100644 --- a/core/ledger/kvledger/txmgmt/privacyenabledstate/snapshot_test.go +++ b/core/ledger/kvledger/txmgmt/privacyenabledstate/snapshot_test.go @@ -22,6 +22,9 @@ import ( "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/privacyenabledstate/mock" "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" + protoV2 "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/protoadapt" ) var testNewHashFunc = func() (hash.Hash, error) { @@ -558,18 +561,24 @@ func TestSnapshotImportErrorPropagation(t *testing.T) { require.NoError(t, os.Remove(dataFile)) fileContent := []byte{snapshotFileFormat} - buf := proto.NewBuffer(nil) - require.NoError(t, - buf.EncodeMessage( - &SnapshotRecord{ - Version: []byte("bad-version-bytes"), - }, - ), - ) - fileContent = append(fileContent, buf.Bytes()...) + sr := &SnapshotRecord{ + Version: []byte("bad-version-bytes"), + } + srTmp := protoadapt.MessageV2Of(sr) + var buf []byte + buf = protowire.AppendVarint(buf, uint64(proto.Size(sr))) + nbuf, err := protoV2.MarshalOptions{ + Deterministic: false, + AllowPartial: true, + }.MarshalAppend(buf, srTmp) + require.NoError(t, err) + if len(buf) == len(nbuf) { + require.True(t, srTmp.ProtoReflect().IsValid()) + } + fileContent = append(fileContent, nbuf...) require.NoError(t, os.WriteFile(dataFile, fileContent, 0o600)) - err := dbEnv.GetProvider().ImportFromSnapshot( + err = dbEnv.GetProvider().ImportFromSnapshot( generateLedgerID(t), version.NewHeight(10, 10), snapshotDir) require.Contains(t, err.Error(), "error while decoding version") @@ -612,9 +621,9 @@ func TestSnapshotImportErrorPropagation(t *testing.T) { require.NoError(t, os.Remove(metadataFile)) fileContentWithMissingCCName := []byte{snapshotFileFormat} - buf := proto.NewBuffer(nil) - require.NoError(t, buf.EncodeVarint(5)) - fileContentWithMissingCCName = append(fileContentWithMissingCCName, buf.Bytes()...) + var buf []byte + buf = protowire.AppendVarint(buf, 5) + fileContentWithMissingCCName = append(fileContentWithMissingCCName, buf...) require.NoError(t, os.WriteFile(metadataFile, fileContentWithMissingCCName, 0o600)) err := dbEnv.GetProvider().ImportFromSnapshot( @@ -630,10 +639,10 @@ func TestSnapshotImportErrorPropagation(t *testing.T) { require.NoError(t, os.Remove(metadataFile)) fileContentWithMissingCCName := []byte{snapshotFileFormat} - buf := proto.NewBuffer(nil) - require.NoError(t, buf.EncodeVarint(1)) - require.NoError(t, buf.EncodeRawBytes([]byte("my-chaincode"))) - fileContentWithMissingCCName = append(fileContentWithMissingCCName, buf.Bytes()...) + var buf []byte + buf = protowire.AppendVarint(buf, 1) + buf = protowire.AppendBytes(buf, []byte("my-chaincode")) + fileContentWithMissingCCName = append(fileContentWithMissingCCName, buf...) require.NoError(t, os.WriteFile(metadataFile, fileContentWithMissingCCName, 0o600)) err := dbEnv.GetProvider().ImportFromSnapshot( diff --git a/core/ledger/pvtdatastorage/kv_encoding.go b/core/ledger/pvtdatastorage/kv_encoding.go index 4f1b0d45e43..d84382f83d2 100644 --- a/core/ledger/pvtdatastorage/kv_encoding.go +++ b/core/ledger/pvtdatastorage/kv_encoding.go @@ -16,6 +16,7 @@ import ( "github.com/hyperledger/fabric-protos-go/ledger/rwset" "github.com/hyperledger/fabric/core/ledger/internal/version" "github.com/pkg/errors" + "google.golang.org/protobuf/encoding/protowire" ) var ( @@ -52,11 +53,14 @@ func getExpiryKeysForRangeScan(minBlkNum, maxBlkNum uint64) ([]byte, []byte) { } func encodeLastCommittedBlockVal(blockNum uint64) []byte { - return proto.EncodeVarint(blockNum) + return protowire.AppendVarint(nil, blockNum) } func decodeLastCommittedBlockVal(blockNumBytes []byte) uint64 { - s, _ := proto.DecodeVarint(blockNumBytes) + s, num := protowire.ConsumeVarint(blockNumBytes) + if num < 0 { + return 0 + } return s } @@ -220,12 +224,12 @@ func decodeBootKVHashesVal(b []byte) (*BootKVHashes, error) { } func encodeLastBlockInBootSnapshotVal(blockNum uint64) []byte { - return proto.EncodeVarint(blockNum) + return protowire.AppendVarint(nil, blockNum) } func decodeLastBlockInBootSnapshotVal(blockNumBytes []byte) (uint64, error) { - s, n := proto.DecodeVarint(blockNumBytes) - if n == 0 { + s, n := protowire.ConsumeVarint(blockNumBytes) + if n <= 0 { return 0, errors.New("unexpected bytes for interpreting as varint") } return s, nil @@ -408,7 +412,7 @@ func encodeReverseOrderVarUint64(number uint64) []byte { } size := 8 - numFFBytes encodedBytes := make([]byte, size+1) - encodedBytes[0] = proto.EncodeVarint(uint64(numFFBytes))[0] + encodedBytes[0] = protowire.AppendVarint(nil, uint64(numFFBytes))[0] copy(encodedBytes[1:], bytes[numFFBytes:]) return encodedBytes } @@ -416,7 +420,10 @@ func encodeReverseOrderVarUint64(number uint64) []byte { // decodeReverseOrderVarUint64 decodes the number from the bytes obtained from function 'EncodeReverseOrderVarUint64'. // Also, returns the number of bytes that are consumed in the process func decodeReverseOrderVarUint64(bytes []byte) (uint64, int) { - s, _ := proto.DecodeVarint(bytes) + s, num := protowire.ConsumeVarint(bytes) + if num < 0 { + s = 0 + } numFFBytes := int(s) decodedBytes := make([]byte, 8) realBytesNum := 8 - numFFBytes diff --git a/core/ledger/pvtdatastorage/store.go b/core/ledger/pvtdatastorage/store.go index b2210d804cb..98e73bf5aa3 100644 --- a/core/ledger/pvtdatastorage/store.go +++ b/core/ledger/pvtdatastorage/store.go @@ -24,6 +24,7 @@ import ( "github.com/hyperledger/fabric/core/ledger/pvtdatapolicy" "github.com/hyperledger/fabric/core/ledger/util" "github.com/pkg/errors" + "google.golang.org/protobuf/encoding/protowire" ) var logger = flogging.MustGetLogger("pvtdatastorage") @@ -173,8 +174,8 @@ type storeEntries struct { // and is stored as the value of lastUpdatedOldBlocksKey (defined in kv_encoding.go) type lastUpdatedOldBlocksList []uint64 -//////// Provider functions ///////////// -////////////////////////////////////////// +// ////// Provider functions ///////////// +// //////////////////////////////////////// // NewProvider instantiates a StoreProvider func NewProvider(conf *PrivateDataConfig) (*Provider, error) { @@ -255,8 +256,8 @@ func (p *Provider) Drop(ledgerid string) error { return p.dbProvider.Drop(ledgerid) } -//////// store functions //////////////// -////////////////////////////////////////// +// ////// store functions //////////////// +// //////////////////////////////////////// func (s *Store) initState() error { var err error @@ -420,8 +421,11 @@ func (s *Store) GetLastUpdatedOldBlocksPvtData() (map[uint64][]*ledger.TxPvtData } func (s *Store) getLastUpdatedOldBlocksList() ([]uint64, error) { - var v []byte - var err error + var ( + v []byte + err error + position int + ) if v, err = s.db.Get(lastUpdatedOldBlocksKey); err != nil { return nil, err } @@ -430,16 +434,18 @@ func (s *Store) getLastUpdatedOldBlocksList() ([]uint64, error) { } var updatedBlksList []uint64 - buf := proto.NewBuffer(v) - numBlks, err := buf.DecodeVarint() - if err != nil { - return nil, err + numBlks, n := protowire.ConsumeVarint(v[position:]) + if n < 0 { + return nil, protowire.ParseError(n) } + position += n + for i := 0; i < int(numBlks); i++ { - blkNum, err := buf.DecodeVarint() - if err != nil { - return nil, err + blkNum, n := protowire.ConsumeVarint(v[position:]) + if n < 0 { + return nil, protowire.ParseError(n) } + position += n updatedBlksList = append(updatedBlksList, blkNum) } return updatedBlksList, nil diff --git a/core/peer/configtx_test.go b/core/peer/configtx_test.go index 800893e350d..c6edde48995 100644 --- a/core/peer/configtx_test.go +++ b/core/peer/configtx_test.go @@ -46,7 +46,7 @@ func TestConfigTxCreateLedger(t *testing.T) { retrievedchanConf, err := RetrievePersistedChannelConfig(ledger) require.NoError(t, err) - require.Equal(t, proto.CompactTextString(chanConf), proto.CompactTextString(retrievedchanConf)) + require.True(t, proto.Equal(chanConf, retrievedchanConf)) } func TestConfigTxErrorScenarios(t *testing.T) { @@ -87,18 +87,18 @@ func TestConfigTxUpdateChanConfig(t *testing.T) { retrievedchanConf, err := RetrievePersistedChannelConfig(lgr) require.NoError(t, err) - require.Equal(t, proto.CompactTextString(chanConf), proto.CompactTextString(retrievedchanConf)) + require.True(t, proto.Equal(chanConf, retrievedchanConf)) helper.mockCreateChain(t, channelID, lgr) defer helper.clearMockChains() bs := helper.peer.channels[channelID].bundleSource inMemoryChanConf := bs.ConfigtxValidator().ConfigProto() - require.Equal(t, proto.CompactTextString(chanConf), proto.CompactTextString(inMemoryChanConf)) + require.True(t, proto.Equal(chanConf, inMemoryChanConf)) retrievedchanConf, err = RetrievePersistedChannelConfig(lgr) require.NoError(t, err) - require.Equal(t, proto.CompactTextString(bs.ConfigtxValidator().ConfigProto()), proto.CompactTextString(retrievedchanConf)) + require.True(t, proto.Equal(bs.ConfigtxValidator().ConfigProto(), retrievedchanConf)) lgr.Close() helper.clearMockChains() diff --git a/internal/configtxlator/rest/protolator_handlers_test.go b/internal/configtxlator/rest/protolator_handlers_test.go index 9093b390472..93310bae19e 100644 --- a/internal/configtxlator/rest/protolator_handlers_test.go +++ b/internal/configtxlator/rest/protolator_handlers_test.go @@ -18,6 +18,7 @@ import ( cb "github.com/hyperledger/fabric-protos-go/common" "github.com/hyperledger/fabric/protoutil" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/protoadapt" ) var ( @@ -47,7 +48,7 @@ var ( func TestProtolatorDecode(t *testing.T) { data, err := proto.Marshal(testProto) require.NoError(t, err) - url := fmt.Sprintf("/protolator/decode/%s", proto.MessageReflect(testProto).Descriptor().FullName()) + url := fmt.Sprintf("/protolator/decode/%s", protoadapt.MessageV2Of(testProto).ProtoReflect().Descriptor().FullName()) req, _ := http.NewRequest("POST", url, bytes.NewReader(data)) rec := httptest.NewRecorder() @@ -63,7 +64,7 @@ func TestProtolatorDecode(t *testing.T) { } func TestProtolatorEncode(t *testing.T) { - url := fmt.Sprintf("/protolator/encode/%s", proto.MessageReflect(testProto).Descriptor().FullName()) + url := fmt.Sprintf("/protolator/encode/%s", protoadapt.MessageV2Of(testProto).ProtoReflect().Descriptor().FullName()) req, _ := http.NewRequest("POST", url, bytes.NewReader([]byte(testOutput))) rec := httptest.NewRecorder() @@ -98,7 +99,7 @@ func TestProtolatorEncodeNonExistantProto(t *testing.T) { } func TestProtolatorDecodeBadData(t *testing.T) { - url := fmt.Sprintf("/protolator/decode/%s", proto.MessageReflect(testProto).Descriptor().FullName()) + url := fmt.Sprintf("/protolator/decode/%s", protoadapt.MessageV2Of(testProto).ProtoReflect().Descriptor().FullName()) req, _ := http.NewRequest("POST", url, bytes.NewReader([]byte("Garbage"))) @@ -110,7 +111,7 @@ func TestProtolatorDecodeBadData(t *testing.T) { } func TestProtolatorEncodeBadData(t *testing.T) { - url := fmt.Sprintf("/protolator/encode/%s", proto.MessageReflect(testProto).Descriptor().FullName()) + url := fmt.Sprintf("/protolator/encode/%s", protoadapt.MessageV2Of(testProto).ProtoReflect().Descriptor().FullName()) req, _ := http.NewRequest("POST", url, bytes.NewReader([]byte("Garbage"))) diff --git a/internal/pkg/gateway/registry.go b/internal/pkg/gateway/registry.go index f51ec09a019..6ef5865ec3c 100644 --- a/internal/pkg/gateway/registry.go +++ b/internal/pkg/gateway/registry.go @@ -25,6 +25,8 @@ import ( gossipdiscovery "github.com/hyperledger/fabric/gossip/discovery" "github.com/hyperledger/fabric/internal/pkg/gateway/ledger" "github.com/pkg/errors" + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/protoadapt" ) type Discovery interface { @@ -58,7 +60,8 @@ type endorserState struct { func (reg *registry) endorsementPlan(channel string, interest *peer.ChaincodeInterest, preferredEndorser *endorser) (*plan, error) { descriptor, err := reg.discovery.PeersForEndorsement(gossipcommon.ChannelID(channel), interest) if err != nil { - logger.Errorw("PeersForEndorsement failed.", "error", err, "channel", channel, "ChaincodeInterest", proto.MarshalTextString(interest)) + b, _ := prototext.Marshal(protoadapt.MessageV2Of(interest)) + logger.Errorw("PeersForEndorsement failed.", "error", err, "channel", channel, "ChaincodeInterest", b) return nil, errors.Wrap(err, "no combination of peers can be derived which satisfy the endorsement policy") } diff --git a/orderer/consensus/etcdraft/node.go b/orderer/consensus/etcdraft/node.go index 7ffeb3760ab..54089b4a81b 100644 --- a/orderer/consensus/etcdraft/node.go +++ b/orderer/consensus/etcdraft/node.go @@ -14,7 +14,6 @@ import ( "time" "code.cloudfoundry.org/clock" - "github.com/golang/protobuf/proto" "github.com/hyperledger/fabric-lib-go/common/flogging" "github.com/hyperledger/fabric-protos-go/orderer" "github.com/hyperledger/fabric-protos-go/orderer/etcdraft" @@ -22,6 +21,7 @@ import ( "github.com/pkg/errors" "go.etcd.io/etcd/raft/v3" "go.etcd.io/etcd/raft/v3/raftpb" + "google.golang.org/protobuf/encoding/protowire" ) var ( @@ -74,7 +74,10 @@ func (n *node) start(fresh, join bool) { // determine the node to start campaign by selecting the node with ID equals to: // hash(channelID) % cluster_size + 1 sha := sha256.Sum256([]byte(n.chainID)) - number, _ := proto.DecodeVarint(sha[24:]) + number, s := protowire.ConsumeVarint(sha[24:]) + if s < 0 { + number = 0 + } if n.config.ID == number%uint64(len(raftPeers))+1 { campaign = true }