Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

curvefs/client: fix bug of support summary info in dir xattr #1150

Merged
merged 2 commits into from
Mar 14, 2022
Merged

curvefs/client: fix bug of support summary info in dir xattr #1150

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e81f2a4
curvefs/client: fix bug of support summary info in dir xattr, the par…
SeanHai Mar 8, 2022
0d1e567
curvefs: batch get inode attr and xattr fater find in cache
SeanHai Mar 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion curvefs/conf/tools.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,5 @@ s3.endpoint=endpoint
s3.bucket_name=bucket
s3.blocksize=4194304
s3.chunksize=67108864
# statistic info in xattr
# statistic info in xattr, hardlink will not be supported when enable
enableSumInDir=false
2 changes: 1 addition & 1 deletion curvefs/src/client/curve_fuse_op.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ void FuseReplyErrByErrCode(fuse_req_t req, CURVEFS_ERROR errcode) {
fuse_reply_err(req, ENOTEMPTY);
break;
case CURVEFS_ERROR::NOTSUPPORT:
fuse_reply_err(req, ENOSYS);
fuse_reply_err(req, EOPNOTSUPP);
break;
case CURVEFS_ERROR::NAMETOOLONG:
fuse_reply_err(req, ENAMETOOLONG);
Expand Down
130 changes: 60 additions & 70 deletions curvefs/src/client/fuse_client.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include <vector>
#include <stack>
#include <set>
#include <unordered_map>

#include "curvefs/proto/mds.pb.h"
#include "curvefs/src/client/fuse_common.h"
Expand Down Expand Up @@ -308,9 +309,9 @@ CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
XAttr xattr;
xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
std::to_string(length)});
std::list<uint64_t> parentIds;
if (inodeManager_->GetParent(inode->inodeid(), &parentIds)) {
ret = UpdateParentInodeXattr(parentIds, xattr, false);
uint64_t parentId;
if (inodeManager_->GetParent(inode->inodeid(), &parentId)) {
ret = UpdateParentInodeXattr(parentId, xattr, false);
} else {
LOG(ERROR) << "inodeManager getParent failed, inodeId = "
<< inode;
Expand Down Expand Up @@ -417,8 +418,7 @@ CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
}
xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
std::to_string(inodeWrapper->GetLength())});
ret = UpdateParentInodeXattr(
std::list<uint64_t>({parent}), xattr, true);
ret = UpdateParentInodeXattr(parent, xattr, true);
}

GetDentryParamFromInode(inodeWrapper, e);
Expand Down Expand Up @@ -516,7 +516,7 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,

if (enableSumInDir_) {
// remove parent relationshaip
inodeManager_->RemoveParent(ino, parent);
inodeManager_->ClearParent(ino);

// update parent summary info
XAttr xattr;
Expand All @@ -528,8 +528,7 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
}
xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
std::to_string(inodeWrapper->GetLength())});
ret = UpdateParentInodeXattr(
std::list<uint64_t>({parent}), xattr, false);
ret = UpdateParentInodeXattr(parent, xattr, false);
}

inodeManager_->ClearInodeCache(ino);
Expand Down Expand Up @@ -669,17 +668,15 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
std::to_string(inodeWrapper->GetLength())});

if (inodeManager_->UpdateParent(ino, parent, newparent)) {
if (inodeManager_->UpdateParent(ino, newparent)) {
// TODO(wanghai): deal with failure
rc = UpdateParentInodeXattr(
std::list<uint64_t>({parent}), xattr, false);
rc = UpdateParentInodeXattr(parent, xattr, false);
if (rc != CURVEFS_ERROR::OK) {
LOG(ERROR) << "UpdateParentInodeXattr failed, parentId = "
<< parent;
return rc;
}
rc = UpdateParentInodeXattr(
std::list<uint64_t>({newparent}), xattr, true);
rc = UpdateParentInodeXattr(newparent, xattr, true);
if (rc != CURVEFS_ERROR::OK) {
LOG(ERROR) << "UpdateParentInodeXattr failed, parentId = "
<< newparent;
Expand Down Expand Up @@ -782,10 +779,10 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
XAttr xattr;
xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
std::to_string(std::abs(changeSize))});
std::list<uint64_t> parentIds;
if (inodeManager_->GetParent(ino, &parentIds)) {
uint64_t parentId;
if (inodeManager_->GetParent(ino, &parentId)) {
bool direction = changeSize > 0;
ret = UpdateParentInodeXattr(parentIds, xattr, direction);
ret = UpdateParentInodeXattr(parentId, xattr, direction);
} else {
LOG(ERROR) << "inodeManager getParent failed, inodeId = "
<< inode;
Expand Down Expand Up @@ -858,7 +855,7 @@ CURVEFS_ERROR FuseClient::CalOneLayerSumInfo(Inode *inode) {
for (const auto &it : dentryList) {
inodeIds.emplace(it.inodeid());
}
ret = inodeManager_->BatchGetInodeAttr(inodeIds, &attrs);
ret = inodeManager_->BatchGetInodeAttr(&inodeIds, &attrs);
if (ret == CURVEFS_ERROR::OK) {
uint64_t files = 0;
uint64_t subdirs = 0;
Expand Down Expand Up @@ -893,7 +890,8 @@ CURVEFS_ERROR FuseClient::CalOneLayerSumInfo(Inode *inode) {

CURVEFS_ERROR FuseClient::CalAllLayerSumInfo(Inode *inode) {
std::stack<uint64_t> iStack;
// use set can deal with hard link
// record hard link, <inodeId, need2minus>
std::unordered_map<uint64_t, uint64_t> hardLinkMap;
std::set<uint64_t> inodeIds;
std::list<InodeAttr> attrs;
auto ino = inode->inodeid();
Expand Down Expand Up @@ -925,7 +923,7 @@ CURVEFS_ERROR FuseClient::CalAllLayerSumInfo(Inode *inode) {
}
// check size
if (inodeIds.size() >= attrsLimit || iStack.empty()) {
ret = inodeManager_->BatchGetInodeAttr(inodeIds, &attrs);
ret = inodeManager_->BatchGetInodeAttr(&inodeIds, &attrs);
if (ret == CURVEFS_ERROR::OK) {
for (const auto &it : attrs) {
if (it.type() == FsFileType::TYPE_DIRECTORY) {
Expand All @@ -935,6 +933,15 @@ CURVEFS_ERROR FuseClient::CalAllLayerSumInfo(Inode *inode) {
}
rentries++;
rfbytes += it.length();
// record hardlink
if (it.type() != FsFileType::TYPE_DIRECTORY &&
it.nlink() > 1) {
if (hardLinkMap.count(it.inodeid())) {
hardLinkMap[it.inodeid()] += it.length();
} else {
hardLinkMap.emplace(it.inodeid(), 0);
}
}
}
inodeIds.clear();
attrs.clear();
Expand All @@ -944,6 +951,11 @@ CURVEFS_ERROR FuseClient::CalAllLayerSumInfo(Inode *inode) {
}
}

// deal with hardlink
for (const auto &it : hardLinkMap) {
rfbytes -= it.second;
}

inode->mutable_xattr()->insert({XATTRRFILES,
std::to_string(rfiles)});
inode->mutable_xattr()->insert({XATTRRSUBDIRS,
Expand Down Expand Up @@ -996,7 +1008,7 @@ CURVEFS_ERROR FuseClient::FastCalAllLayerSumInfo(Inode *inode) {
}
// check size
if (inodeIds.size() >= xattrsLimit || iStack.empty()) {
ret = inodeManager_->BatchGetXAttr(inodeIds, &xattrs);
ret = inodeManager_->BatchGetXAttr(&inodeIds, &xattrs);
if (ret == CURVEFS_ERROR::OK) {
for (const auto &it : xattrs) {
if (it.xattrinfos().count(XATTRFILES)) {
Expand Down Expand Up @@ -1180,8 +1192,7 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, const char *link,
xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"});
xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
std::to_string(inodeWrapper->GetLength())});
ret = UpdateParentInodeXattr(
std::list<uint64_t>({parent}), xattr, true);
ret = UpdateParentInodeXattr(parent, xattr, true);
}

GetDentryParamFromInode(inodeWrapper, e);
Expand All @@ -1194,6 +1205,12 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
LOG(INFO) << "FuseOpLink, ino: " << ino
<< ", newparent: " << newparent
<< ", newname: " << newname;
if (enableSumInDir_) {
// don't support hardlink
LOG(ERROR) << "FuseOpLink doesn't support when enableSumInDir.";
return CURVEFS_ERROR::NOTSUPPORT;
}

if (strlen(newname) > option_.maxNameLength) {
return CURVEFS_ERROR::NAMETOOLONG;
}
Expand Down Expand Up @@ -1244,19 +1261,6 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
return ret;
}

if (enableSumInDir_) {
// recore parent inodeId
inodeManager_->AddParent(inodeWrapper->GetInodeId(), newparent);
// update parent summary info
XAttr xattr;
xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"});
xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"});
xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
std::to_string(inodeWrapper->GetLength())});
ret = UpdateParentInodeXattr(
std::list<uint64_t>({newparent}), xattr, true);
}

GetDentryParamFromInode(inodeWrapper, e);
return ret;
}
Expand Down Expand Up @@ -1297,10 +1301,6 @@ CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req, fuse_ino_t ino,
return ret;
}

if (enableSumInDir_) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why not delete

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why not delete

Oplookup will not called when vfs cached the info of inode, so if delete the relationship when release maybe lost the parent.

// update parent relationship
inodeManager_->ClearParent(ino);
}
return ret;
}

Expand All @@ -1313,41 +1313,31 @@ void FuseClient::FlushAll() {
FlushInodeAll();
}

CURVEFS_ERROR FuseClient::UpdateParentInodeXattr(
const std::list<uint64_t> &parentIds,
CURVEFS_ERROR FuseClient::UpdateParentInodeXattr(uint64_t parentId,
const XAttr &xattr, bool direction) {
for (const auto &parentId : parentIds) {
VLOG(1) << "UpdateParentInodeXattr inodeId = " << parentId
<< ", \nxattr = " << xattr.DebugString();
std::shared_ptr<InodeWrapper> pInodeWrapper;
CURVEFS_ERROR ret = inodeManager_->GetInode(parentId, pInodeWrapper);
if (ret != CURVEFS_ERROR::OK) {
LOG(ERROR) << "UpdateParentInodeXattr get parent inode fail, ret = "
<< ret << ", inodeid = " << parentId;
return ret;
}
// if dirty, flush attr first
if (pInodeWrapper->Dirty()) {
ret = pInodeWrapper->SyncAttr();
if (ret != CURVEFS_ERROR::OK) {
LOG(ERROR) << "sync parent inode attr failed when"
<< " UpdateParentInodeXattr, inodeId = " << parentId;
return ret;
}
}
::curve::common::UniqueLock lgGuard = pInodeWrapper->GetUniqueLock();
auto inode = pInodeWrapper->GetMutableInodeUnlocked();
for (const auto &it : xattr.xattrinfos()) {
auto iter = inode->mutable_xattr()->find(it.first);
if (iter != inode->mutable_xattr()->end()) {
if (!AddUllStringToFirst(&(iter->second), it.second,
direction)) {
return CURVEFS_ERROR::INTERNAL;
}
VLOG(1) << "UpdateParentInodeXattr inodeId = " << parentId
<< ", direction = " << direction
<< ", \nxattr = " << xattr.DebugString();
std::shared_ptr<InodeWrapper> pInodeWrapper;
CURVEFS_ERROR ret = inodeManager_->GetInode(parentId, pInodeWrapper);
if (ret != CURVEFS_ERROR::OK) {
LOG(ERROR) << "UpdateParentInodeXattr get parent inode fail, ret = "
<< ret << ", inodeid = " << parentId;
return ret;
}

::curve::common::UniqueLock lgGuard = pInodeWrapper->GetUniqueLock();
auto inode = pInodeWrapper->GetMutableInodeUnlocked();
for (const auto &it : xattr.xattrinfos()) {
auto iter = inode->mutable_xattr()->find(it.first);
if (iter != inode->mutable_xattr()->end()) {
if (!AddUllStringToFirst(&(iter->second), it.second,
direction)) {
return CURVEFS_ERROR::INTERNAL;
}
}
pInodeWrapper->FlushXattrAsync();
}
pInodeWrapper->FlushXattrAsync();
return CURVEFS_ERROR::OK;
}

Expand Down
2 changes: 1 addition & 1 deletion curvefs/src/client/fuse_client.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ class FuseClient {
return 0;
}

CURVEFS_ERROR UpdateParentInodeXattr(const std::list<uint64_t> &parentIds,
CURVEFS_ERROR UpdateParentInodeXattr(uint64_t parentId,
const XAttr &xattr, bool direction);

private:
Expand Down
12 changes: 7 additions & 5 deletions curvefs/src/client/fuse_s3_client.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,10 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
Inode *inode = inodeWrapper->GetMutableInodeUnlocked();

*wSize = wRet;
size_t changeSize = 0;
// update file len
if (inode->length() < off + *wSize) {
changeSize = off + *wSize - inode->length();
inode->set_length(off + *wSize);
}
struct timespec now;
Expand All @@ -125,13 +127,13 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
// Todo: do some cache flush later
}

if (enableSumInDir_) {
if (enableSumInDir_ && changeSize != 0) {
XAttr xattr;
xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
std::to_string(*wSize)});
std::list<uint64_t> parentIds;
if (inodeManager_->GetParent(ino, &parentIds)) {
ret = UpdateParentInodeXattr(parentIds, xattr, true);
std::to_string(changeSize)});
uint64_t parentId;
if (inodeManager_->GetParent(ino, &parentId)) {
ret = UpdateParentInodeXattr(parentId, xattr, true);
} else {
LOG(ERROR) << "inodeManager getParent failed, inodeId = " << ino;
return CURVEFS_ERROR::INTERNAL;
Expand Down
Loading