Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

avoid memcpy when using RocksDB slice #2183

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+163 −117
Open

avoid memcpy when using RocksDB slice #2183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions curvefs/src/metaserver/dentry_storage.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -379,8 +379,7 @@ MetaStatusCode DentryStorage::List(const Dentry& dentry,
time.start();
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
seekTimes++;
std::string skey = iterator->Key();
std::string svalue = iterator->Value();
absl::string_view skey = iterator->Key();
if (!StringStartWith(skey, sprefix)) {
break;
} else if (!iterator->ParseFromValue(&current)) {
Expand Down
21 changes: 11 additions & 10 deletions curvefs/src/metaserver/inode_storage.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ bool InodeStorage::GetAllInodeId(std::list<uint64_t>* ids) {

Key4Inode key;
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
if (!conv_.ParseFromString(iterator->Key(), &key)) {
if (!conv_.ParseFromStringView(iterator->Key(), &key)) {
return false;
}
ids->push_back(key.inodeId);
Expand Down Expand Up @@ -372,12 +372,12 @@ MetaStatusCode InodeStorage::DelS3ChunkInfoList(
}

Key4S3ChunkInfoList key;
std::vector<std::string> key2del;
std::vector<absl::string_view> key2del;
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
std::string skey = iterator->Key();
absl::string_view skey = iterator->Key();
if (!StringStartWith(skey, sprefix)) {
break;
} else if (!conv_.ParseFromString(skey, &key)) {
} else if (!conv_.ParseFromStringView(skey, &key)) {
return MetaStatusCode::PARSE_FROM_STRING_FAILED;
}

Expand All @@ -397,8 +397,9 @@ MetaStatusCode InodeStorage::DelS3ChunkInfoList(
}
}

for (const auto& skey : key2del) {
if (!txn->SDel(table4S3ChunkInfo_, skey).ok()) {
// TODO: how to avoid copy here?
for (auto skey : key2del) {
if (!txn->SDel(std::string(table4S3ChunkInfo_), std::string(skey)).ok()) {
LOG(ERROR) << "Delete key failed, skey=" << skey;
return MetaStatusCode::STORAGE_INTERNAL_ERROR;
}
Expand Down Expand Up @@ -475,9 +476,8 @@ MetaStatusCode InodeStorage::PaddingInodeS3ChunkInfo(int32_t fsId,
Key4S3ChunkInfoList key;
S3ChunkInfoList list;
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
std::string skey = iterator->Key();
std::string svalue = iterator->Value();
if (!conv_.ParseFromString(skey, &key)) {
absl::string_view skey = iterator->Key();
if (!conv_.ParseFromStringView(skey, &key)) {
return MetaStatusCode::PARSE_FROM_STRING_FAILED;
} else if (!iterator->ParseFromValue(&list)) {
return MetaStatusCode::PARSE_FROM_STRING_FAILED;
Expand Down Expand Up @@ -541,7 +541,8 @@ MetaStatusCode InodeStorage::GetAllVolumeExtent(uint32_t fsId,
continue;
}

if (!slice->ParseFromString(iter->Value())) {
absl::string_view val = iter->Value();
if (!slice->ParseFromArray(val.data(), val.size())) {
LOG(ERROR) << "Parse ExtentSlice failed, fsId: " << fsId
<< ", inodeId: " << inodeId;
extents->Clear();
Expand Down
13 changes: 10 additions & 3 deletions curvefs/src/metaserver/metastore.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -738,11 +738,18 @@ MetaStatusCode MetaStoreImpl::GetOrModifyS3ChunkInfo(

void MetaStoreImpl::PrepareStreamBuffer(butil::IOBuf* buffer,
uint64_t chunkIndex,
const std::string& value) {
absl::string_view value) {
buffer->clear();
/*
buffer->append(std::to_string(chunkIndex));
buffer->append(":");
buffer->append(value);
*/
butil::IOBufAppender append_buffer;
append_buffer.append(std::to_string(chunkIndex));
append_buffer.push_back(':');
append_buffer.append(value.data(), value.size());
append_buffer.move_to(*buffer);
}

MetaStatusCode MetaStoreImpl::SendS3ChunkInfoByStream(
Expand All @@ -752,8 +759,8 @@ MetaStatusCode MetaStoreImpl::SendS3ChunkInfoByStream(
Key4S3ChunkInfoList key;
Converter conv;
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
std::string skey = iterator->Key();
if (!conv.ParseFromString(skey, &key)) {
absl::string_view skey = iterator->Key();
if (!conv.ParseFromStringView(skey, &key)) {
return MetaStatusCode::PARSE_FROM_STRING_FAILED;
}

Expand Down
3 changes: 2 additions & 1 deletion curvefs/src/metaserver/metastore.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include <memory>
#include <string>

#include "absl/strings/string_view.h"
#include "curvefs/proto/metaserver.pb.h"
#include "curvefs/src/common/rpc_stream.h"
#include "curvefs/src/metaserver/copyset/snapshot_closure.h"
Expand Down Expand Up @@ -275,7 +276,7 @@ class MetaStoreImpl : public MetaStore {

void PrepareStreamBuffer(butil::IOBuf* buffer,
uint64_t chunkIndex,
const std::string& value);
absl::string_view value);

void SaveBackground(const std::string& path,
DumpFileClosure* child,
Expand Down
46 changes: 23 additions & 23 deletions curvefs/src/metaserver/metastore_fstream.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,10 @@ std::shared_ptr<Partition> MetaStoreFStream::GetPartition(
}

bool MetaStoreFStream::LoadPartition(uint32_t partitionId,
const std::string& key,
const std::string& value) {
absl::string_view key,
absl::string_view value) {
PartitionInfo partitionInfo;
if (!conv_->ParseFromString(value, &partitionInfo)) {
if (!conv_->ParseFromArray(value, &partitionInfo)) {
LOG(ERROR) << "Decode PartitionInfo failed";
return false;
}
Expand All @@ -98,16 +98,16 @@ bool MetaStoreFStream::LoadPartition(uint32_t partitionId,
}

bool MetaStoreFStream::LoadInode(uint32_t partitionId,
const std::string& key,
const std::string& value) {
absl::string_view key,
absl::string_view value) {
auto partition = GetPartition(partitionId);
if (nullptr == partition) {
LOG(ERROR) << "Partition not found, partitionId = " << partitionId;
return false;
}

Inode inode;
if (!conv_->ParseFromString(value, &inode)) {
if (!conv_->ParseFromArray(value, &inode)) {
LOG(ERROR) << "Decode inode failed";
return false;
}
Expand All @@ -123,8 +123,8 @@ bool MetaStoreFStream::LoadInode(uint32_t partitionId,

bool MetaStoreFStream::LoadDentry(uint8_t version,
uint32_t partitionId,
const std::string& key,
const std::string& value) {
absl::string_view key,
absl::string_view value) {
auto partition = GetPartition(partitionId);
if (nullptr == partition) {
LOG(ERROR) << "Partition not found, partitionId = " << partitionId;
Expand All @@ -134,12 +134,12 @@ bool MetaStoreFStream::LoadDentry(uint8_t version,
DentryVec vec;
if (version == 1) {
Dentry dentry;
if (!conv_->ParseFromString(value, &dentry)) {
if (!conv_->ParseFromArray(value, &dentry)) {
LOG(ERROR) << "Decode dentry failed";
return false;
}
*vec.add_dentrys() = dentry;
} else if (!conv_->ParseFromString(value, &vec)) {
} else if (!conv_->ParseFromArray(value, &vec)) {
LOG(ERROR) << "Decode dentry vector failed";
return false;
}
Expand All @@ -154,16 +154,16 @@ bool MetaStoreFStream::LoadDentry(uint8_t version,
}

bool MetaStoreFStream::LoadPendingTx(uint32_t partitionId,
const std::string& key,
const std::string& value) {
absl::string_view key,
absl::string_view value) {
auto partition = GetPartition(partitionId);
if (nullptr == partition) {
LOG(ERROR) << "Partition not found, partitionId = " << partitionId;
return false;
}

PrepareRenameTxRequest pendingTx;
if (!conv_->ParseFromString(value, &pendingTx)) {
if (!conv_->ParseFromArray(value, &pendingTx)) {
LOG(ERROR) << "Decode pending tx failed";
return false;
}
Expand All @@ -176,8 +176,8 @@ bool MetaStoreFStream::LoadPendingTx(uint32_t partitionId,
}

bool MetaStoreFStream::LoadInodeS3ChunkInfoList(uint32_t partitionId,
const std::string& key,
const std::string& value) {
absl::string_view key,
absl::string_view value) {
auto partition = GetPartition(partitionId);
if (nullptr == partition) {
LOG(ERROR) << "Partition not found, partitionId = " << partitionId;
Expand All @@ -186,10 +186,10 @@ bool MetaStoreFStream::LoadInodeS3ChunkInfoList(uint32_t partitionId,

S3ChunkInfoList list;
Key4S3ChunkInfoList key4list;
if (!conv_->ParseFromString(key, &key4list)) {
if (!conv_->ParseFromStringView(key, &key4list)) {
LOG(ERROR) << "Decode Key4S3ChunkInfoList failed";
return false;
} else if (!conv_->ParseFromString(value, &list)) {
} else if (!conv_->ParseFromArray(value, &list)) {
LOG(ERROR) << "Decode S3ChunkInfoList failed";
return false;
}
Expand All @@ -209,8 +209,8 @@ bool MetaStoreFStream::LoadInodeS3ChunkInfoList(uint32_t partitionId,
}

bool MetaStoreFStream::LoadVolumeExtentList(uint32_t partitionId,
const std::string& key,
const std::string& value) {
absl::string_view key,
absl::string_view value) {
auto partition = GetPartition(partitionId);
if (!partition) {
LOG(ERROR) << "Partition not found, partitionId: " << partitionId;
Expand All @@ -220,13 +220,13 @@ bool MetaStoreFStream::LoadVolumeExtentList(uint32_t partitionId,
Key4VolumeExtentSlice sliceKey;
VolumeExtentSlice slice;

if (!sliceKey.ParseFromString(key)) {
if (!sliceKey.ParseFromStringView(key)) {
LOG(ERROR) << "Fail to decode Key4VolumeExtentSlice, key: `" << key
<< "`";
return false;
}

if (!conv_->ParseFromString(value, &slice)) {
if (!conv_->ParseFromArray(value, &slice)) {
LOG(ERROR) << "Decode VolumeExtentSlice failed";
return false;
}
Expand Down Expand Up @@ -337,8 +337,8 @@ bool MetaStoreFStream::Load(const std::string& pathname, uint8_t* version) {
auto callback = [&](uint8_t version,
ENTRY_TYPE entryType,
uint32_t partitionId,
const std::string& key,
const std::string& value) -> bool {
absl::string_view key,
absl::string_view value) -> bool {
switch (entryType) {
case ENTRY_TYPE::PARTITION:
++totalPartition;
Expand Down
24 changes: 12 additions & 12 deletions curvefs/src/metaserver/metastore_fstream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,29 +58,29 @@ class MetaStoreFStream {

private:
bool LoadPartition(uint32_t partitionId,
const std::string& key,
const std::string& value);
absl::string_view key,
absl::string_view value);

bool LoadInode(uint32_t partitionId,
const std::string& key,
const std::string& value);
absl::string_view key,
absl::string_view value);

bool LoadDentry(uint8_t version,
uint32_t partitionId,
const std::string& key,
const std::string& value);
absl::string_view key,
absl::string_view value);

bool LoadPendingTx(uint32_t partitionId,
const std::string& key,
const std::string& value);
absl::string_view key,
absl::string_view value);

bool LoadInodeS3ChunkInfoList(uint32_t partitionId,
const std::string& key,
const std::string& value);
absl::string_view key,
absl::string_view value);

bool LoadVolumeExtentList(uint32_t partitionId,
const std::string& key,
const std::string& value);
absl::string_view key,
absl::string_view value);

std::shared_ptr<Iterator> NewPartitionIterator();

Expand Down
21 changes: 21 additions & 0 deletions curvefs/src/metaserver/storage/converter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include <string>
#include <type_traits>

#include "absl/strings/string_view.h"
#include "curvefs/src/metaserver/storage/common.h"

namespace curvefs {
Expand Down Expand Up @@ -81,6 +82,10 @@ class StorageKey {

virtual std::string SerializeToString() const = 0;
virtual bool ParseFromString(const std::string& value) = 0;

virtual bool ParseFromStringView(absl::string_view value) {
return ParseFromString(std::string(value));
}
};

/* rules for key serialization:
Expand Down Expand Up @@ -344,6 +349,22 @@ class Converter {
bool ParseFromString(const std::string& value, Entry* entry) {
return entry->ParseFromString(value);
}

template <typename Entry,
typename = typename std::enable_if<
std::is_base_of<google::protobuf::Message, Entry>::value ||
std::is_base_of<StorageKey, Entry>::value>::type>
bool ParseFromStringView(absl::string_view value, Entry* entry) {
return entry->ParseFromStringView(value);
}

template <typename Entry,
typename = typename std::enable_if<
std::is_base_of<google::protobuf::Message, Entry>::value ||
std::is_base_of<StorageKey, Entry>::value>::type>
bool ParseFromArray(absl::string_view value, Entry* entry) {
return entry->ParseFromArray(value.data(), value.size());
}
};

} // namespace storage
Expand Down
12 changes: 6 additions & 6 deletions curvefs/src/metaserver/storage/dumpfile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,20 +216,20 @@ DUMPFILE_ERROR DumpFile::SaveInt(Int num, off_t* offset, uint32_t* checkSum) {
return retCode;
}

DUMPFILE_ERROR DumpFile::SaveString(const std::string& str,
DUMPFILE_ERROR DumpFile::SaveString(absl::string_view str,
off_t* offset,
uint32_t* checkSum) {
size_t length = str.size();
auto retCode = Write(str.c_str(), *offset, length);
auto retCode = Write(str.data(), *offset, length);
if (retCode == DUMPFILE_ERROR::OK) {
*offset = (*offset) + length;
*checkSum = CRC32(*checkSum, str.c_str(), length);
*checkSum = CRC32(*checkSum, str.data(), length);
}

return retCode;
}

DUMPFILE_ERROR DumpFile::SaveEntry(const std::string& entry,
DUMPFILE_ERROR DumpFile::SaveEntry(absl::string_view entry,
off_t* offset,
uint32_t* checkSum) {
auto retCode = SaveInt<uint32_t>(entry.size(), offset, checkSum);
Expand Down Expand Up @@ -613,11 +613,11 @@ void DumpFileIterator::Next() {
iter_.second = value;
}

std::string DumpFileIterator::Key() {
absl::string_view DumpFileIterator::Key() {
return iter_.first;
}

std::string DumpFileIterator::Value() {
absl::string_view DumpFileIterator::Value() {
return iter_.second;
}

Expand Down
Loading