Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

nbd: fix concurrent nbd map #447

Merged
merged 1 commit into from
Jul 21, 2021
Merged

nbd: fix concurrent nbd map #447

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 82 additions & 43 deletions nbd/src/NBDController.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,103 +42,142 @@
namespace curve {
namespace nbd {

int IOController::InitDevAttr(int devfd, NBDConfig* config, int sockfd,
uint64_t size, uint64_t flags) {
int ret = ioctl(devfd, NBD_SET_SOCK, sockfd);
if (ret < 0) {
cerr << "curve-ndb: the device " << config->devpath
<< " is busy" << std::endl;
return -errno;
}
int IOController::InitDevAttr(NBDConfig* config, uint64_t size,
uint64_t flags) {
int ret = -1;

do {
ret = ioctl(devfd, NBD_SET_BLKSIZE, config->block_size);
ret = ioctl(nbdFd_, NBD_SET_BLKSIZE, config->block_size);
if (ret < 0) {
break;
}

ret = ioctl(devfd, NBD_SET_SIZE, size);
ret = ioctl(nbdFd_, NBD_SET_SIZE, size);
if (ret < 0) {
break;
}

ioctl(devfd, NBD_SET_FLAGS, flags);
ioctl(nbdFd_, NBD_SET_FLAGS, flags);

ret = CheckSetReadOnly(devfd, flags);
ret = CheckSetReadOnly(nbdFd_, flags);
if (ret < 0) {
cerr << "curve-nbd: Check and set read only flag failed."
<< cpp_strerror(ret) << std::endl;
break;
}

if (config->timeout >= 0) {
ret = ioctl(devfd, NBD_SET_TIMEOUT, (unsigned long)config->timeout); // NOLINT
ret = ioctl(nbdFd_, NBD_SET_TIMEOUT, (unsigned long)config->timeout); // NOLINT
if (ret < 0) {
cerr << "curve-ndb: failed to set timeout: "
cerr << "curve-nbd: failed to set timeout: "
<< cpp_strerror(ret) << std::endl;
break;
}
}
} while (false);

if (ret < 0) {
ret = -errno;
ioctl(devfd, NBD_CLEAR_SOCK);
}
return ret;
}

int IOController::SetUp(NBDConfig* config, int sockfd,
uint64_t size, uint64_t flags) {
if (config->devpath.empty()) {
config->devpath = find_unused_nbd_device();
int IOController::MapOnUnusedNbdDevice(int sockfd, std::string* devpath) {
int index = 0;
char dev[64];
const int nbdsMax = get_nbd_max_count();

while (index < nbdsMax) {
snprintf(dev, sizeof(dev), "/dev/nbd%d", index);

int ret = MapOnNbdDeviceByDevPath(sockfd, dev, false);
if (ret < 0) {
++index;
continue;
} else {
*devpath = dev;
return 0;
}
}

if (config->devpath.empty()) {
cerr << "curve-nbd: failed to map on unused device, max nbd index: "
<< (nbdsMax - 1) << ", last try nbd index: " << (index - 1)
<< ", last error: " << cpp_strerror(errno) << std::endl;

return -1;
}

int IOController::MapOnNbdDeviceByDevPath(int sockfd,
const std::string& devpath,
bool logWhenError) {
int index = parse_nbd_index(devpath);
if (index < 0) {
return -1;
}

int devfd = open(devpath.c_str(), O_RDWR);
if (devfd < 0) {
if (logWhenError) {
cerr << "curve-nbd: failed to open device: " << devfd
<< ", error = " << cpp_strerror(errno) << std::endl;
}
return -1;
}

int ret = parse_nbd_index(config->devpath);
int ret = ioctl(devfd, NBD_SET_SOCK, sockfd);
if (ret < 0) {
return ret;
if (logWhenError) {
cerr << "curve-nbd: ioctl NBD_SET_SOCK failed, devpath: " << devpath
<< ", error = " << cpp_strerror(errno) << std::endl;
}
close(devfd);
return -1;
}

nbdFd_ = devfd;
nbdIndex_ = index;
return 0;
}

int IOController::SetUp(NBDConfig* config, int sockfd,
uint64_t size, uint64_t flags) {
int ret = -1;

if (config->devpath.empty()) {
ret = MapOnUnusedNbdDevice(sockfd, &config->devpath);
} else {
ret = MapOnNbdDeviceByDevPath(sockfd, config->devpath);
}
int index = ret;

ret = open(config->devpath.c_str(), O_RDWR);
if (ret < 0) {
cerr << "curve-ndb: failed to open device: "
<< config->devpath << std::endl;
return ret;
return -1;
}
int devfd = ret;

ret = InitDevAttr(devfd, config, sockfd, size, flags);
ret = InitDevAttr(config, size, flags);
if (ret == 0) {
ret = check_device_size(index, size);
ret = check_device_size(nbdIndex_, size);
}
if (ret < 0) {
cerr << "curve-ndb: failed to map, status: "
cerr << "curve-nbd: failed to map, status: "
<< cpp_strerror(ret) << std::endl;
close(devfd);
ioctl(nbdFd_, NBD_CLEAR_SOCK);
close(nbdFd_);
nbdFd_ = -1;
nbdIndex_ = -1;
return ret;
}

nbdFd_ = devfd;
nbdIndex_ = index;
return 0;
}

int IOController::DisconnectByPath(const std::string& devpath) {
int devfd = open(devpath.c_str(), O_RDWR);
if (devfd < 0) {
cerr << "curve-ndb: failed to open device: "
cerr << "curve-nbd: failed to open device: "
<< devpath << ", error = " << cpp_strerror(errno) << std::endl;
return devfd;
}

int ret = ioctl(devfd, NBD_DISCONNECT);
if (ret < 0) {
cerr << "curve-ndb: the device is not used. "
cerr << "curve-nbd: the device is not used. "
<< cpp_strerror(errno) << std::endl;
}

Expand Down Expand Up @@ -396,7 +435,7 @@ int NetLinkController::DisconnectInternal(int index) {
genl_handle_msg, NULL);
msg = nlmsg_alloc();
if (msg == nullptr) {
cerr << "curve-ndb: Could not allocate netlink message." << std::endl;
cerr << "curve-nbd: Could not allocate netlink message." << std::endl;
return -EIO;
}

Expand All @@ -411,7 +450,7 @@ int NetLinkController::DisconnectInternal(int index) {

ret = nl_send_sync(sock_, msg);
if (ret < 0) {
cerr << "curve-ndb: netlink disconnect failed: "
cerr << "curve-nbd: netlink disconnect failed: "
<< nl_geterror(ret) << std::endl;
return -EIO;
}
Expand All @@ -431,7 +470,7 @@ int NetLinkController::ResizeInternal(int nbdIndex, uint64_t size) {
genl_handle_msg, NULL);
msg = nlmsg_alloc();
if (msg == nullptr) {
cerr << "curve-ndb: Could not allocate netlink message." << std::endl;
cerr << "curve-nbd: Could not allocate netlink message." << std::endl;
return -EIO;
}

Expand All @@ -447,7 +486,7 @@ int NetLinkController::ResizeInternal(int nbdIndex, uint64_t size) {

ret = nl_send_sync(sock_, msg);
if (ret < 0) {
cerr << "curve-ndb: netlink resize failed: "
cerr << "curve-nbd: netlink resize failed: "
<< nl_geterror(ret) << std::endl;
return -EIO;
}
Expand Down
6 changes: 4 additions & 2 deletions nbd/src/NBDController.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,10 @@ class IOController : public NBDController {
int Resize(uint64_t size) override;

private:
int InitDevAttr(int devfd, NBDConfig* config, int sockfd,
uint64_t size, uint64_t flags);
int InitDevAttr(NBDConfig* config, uint64_t size, uint64_t flags);
int MapOnUnusedNbdDevice(int sockfd, std::string* devpath);
int MapOnNbdDeviceByDevPath(int sockfd, const std::string& devpath,
bool logWhenError = true);
};

class NetLinkController : public NBDController {
Expand Down
48 changes: 0 additions & 48 deletions nbd/src/util.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,54 +89,6 @@ int get_nbd_max_count() {
return nbds_max;
}

std::string find_unused_nbd_device() {
int index = 0;
int devfd = 0;
int nbds_max = get_nbd_max_count();
char dev[64];
int sockfd[2];

int ret = socketpair(AF_UNIX, SOCK_STREAM, 0, sockfd);
if (ret < 0) {
cerr << "curve-ndb: failed to create socket pair." << std::endl;
return "";
}

while (true) {
snprintf(dev, sizeof(dev), "/dev/nbd%d", index);

ret = open(dev, O_RDWR);
if (ret < 0) {
if (ret == -EPERM && nbds_max != -1 && index < (nbds_max-1)) {
++index;
continue;
}
cerr << "curve-ndb: failed to find unused device, "
<< cpp_strerror(errno) << std::endl;
break;
}

devfd = ret;
ret = ioctl(devfd, NBD_SET_SOCK, sockfd[0]);
if (ret < 0) {
close(devfd);
++index;
continue;
}
break;
}

std::string result = "";
if (ret == 0) {
result = dev;
ioctl(devfd, NBD_CLEAR_SOCK);
close(devfd);
}
close(sockfd[0]);
close(sockfd[1]);
return result;
}

static bool find_mapped_dev_by_spec(NBDConfig *cfg) {
int pid;
NBDConfig c;
Expand Down
2 changes: 0 additions & 2 deletions nbd/src/util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ extern std::string cpp_strerror(int err);
extern int parse_nbd_index(const std::string& devpath);
// 获取当前系统能够支持的最大nbd设备数量
extern int get_nbd_max_count();
// 获取一个当前还未映射的nbd设备名
extern std::string find_unused_nbd_device();
// 解析用户输入的命令参数
extern int parse_args(std::vector<const char*>& args, // NOLINT
std::ostream *err_msg,
Expand Down
13 changes: 9 additions & 4 deletions test/common/test_timeutility.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,25 @@ TEST(ExpiredTimeTest, CommonTest) {
ExpiredTime expiredTime;
std::this_thread::sleep_for(std::chrono::seconds(2));
auto expiredSec = expiredTime.ExpiredSec();
ASSERT_TRUE(expiredSec >= 1.8 && expiredSec <= 2.2);
double expected = 2;
ASSERT_GE(expiredSec, expected * 0.9);
ASSERT_LE(expiredSec, expected * 1.1);
}
{
ExpiredTime expiredTime;
std::this_thread::sleep_for(std::chrono::milliseconds(1000));
auto expiredMs = expiredTime.ExpiredMs();
ASSERT_TRUE(expiredMs >= (1000 - 10) && expiredMs <= (1000 + 10));
double expected = 1000;
ASSERT_GE(expiredMs, expected * 0.9);
ASSERT_LE(expiredMs, expected * 1.1);
}
{
ExpiredTime expiredTime;
std::this_thread::sleep_for(std::chrono::microseconds(1000000));
auto expiredUs = expiredTime.ExpiredUs();
ASSERT_TRUE(expiredUs >= (1000000 - 200) &&
expiredUs <= (1000000 + 200));
double expected = 1000000;
ASSERT_GE(expiredUs, expected * 0.9);
ASSERT_LE(expiredUs, expected * 1.1);
}
}

Expand Down