diff --git a/nbd/src/NBDController.cpp b/nbd/src/NBDController.cpp index be820ead06..47762923d0 100644 --- a/nbd/src/NBDController.cpp +++ b/nbd/src/NBDController.cpp @@ -42,29 +42,24 @@ namespace curve { namespace nbd { -int IOController::InitDevAttr(int devfd, NBDConfig* config, int sockfd, - uint64_t size, uint64_t flags) { - int ret = ioctl(devfd, NBD_SET_SOCK, sockfd); - if (ret < 0) { - cerr << "curve-ndb: the device " << config->devpath - << " is busy" << std::endl; - return -errno; - } +int IOController::InitDevAttr(NBDConfig* config, uint64_t size, + uint64_t flags) { + int ret = -1; do { - ret = ioctl(devfd, NBD_SET_BLKSIZE, config->block_size); + ret = ioctl(nbdFd_, NBD_SET_BLKSIZE, config->block_size); if (ret < 0) { break; } - ret = ioctl(devfd, NBD_SET_SIZE, size); + ret = ioctl(nbdFd_, NBD_SET_SIZE, size); if (ret < 0) { break; } - ioctl(devfd, NBD_SET_FLAGS, flags); + ioctl(nbdFd_, NBD_SET_FLAGS, flags); - ret = CheckSetReadOnly(devfd, flags); + ret = CheckSetReadOnly(nbdFd_, flags); if (ret < 0) { cerr << "curve-nbd: Check and set read only flag failed." << cpp_strerror(ret) << std::endl; @@ -72,73 +67,117 @@ int IOController::InitDevAttr(int devfd, NBDConfig* config, int sockfd, } if (config->timeout >= 0) { - ret = ioctl(devfd, NBD_SET_TIMEOUT, (unsigned long)config->timeout); // NOLINT + ret = ioctl(nbdFd_, NBD_SET_TIMEOUT, (unsigned long)config->timeout); // NOLINT if (ret < 0) { - cerr << "curve-ndb: failed to set timeout: " + cerr << "curve-nbd: failed to set timeout: " << cpp_strerror(ret) << std::endl; break; } } } while (false); - if (ret < 0) { - ret = -errno; - ioctl(devfd, NBD_CLEAR_SOCK); - } return ret; } -int IOController::SetUp(NBDConfig* config, int sockfd, - uint64_t size, uint64_t flags) { - if (config->devpath.empty()) { - config->devpath = find_unused_nbd_device(); +int IOController::MapOnUnusedNbdDevice(int sockfd, std::string* devpath) { + int index = 0; + char dev[64]; + const int nbdsMax = get_nbd_max_count(); + + while (index < nbdsMax) { + snprintf(dev, sizeof(dev), "/dev/nbd%d", index); + + int ret = MapOnNbdDeviceByDevPath(sockfd, dev, false); + if (ret < 0) { + ++index; + continue; + } else { + *devpath = dev; + return 0; + } } - if (config->devpath.empty()) { + cerr << "curve-nbd: failed to map on unused device, max nbd index: " + << (nbdsMax - 1) << ", last try nbd index: " << (index - 1) + << ", last error: " << cpp_strerror(errno) << std::endl; + + return -1; +} + +int IOController::MapOnNbdDeviceByDevPath(int sockfd, + const std::string& devpath, + bool logWhenError) { + int index = parse_nbd_index(devpath); + if (index < 0) { + return -1; + } + + int devfd = open(devpath.c_str(), O_RDWR); + if (devfd < 0) { + if (logWhenError) { + cerr << "curve-nbd: failed to open device: " << devfd + << ", error = " << cpp_strerror(errno) << std::endl; + } return -1; } - int ret = parse_nbd_index(config->devpath); + int ret = ioctl(devfd, NBD_SET_SOCK, sockfd); if (ret < 0) { - return ret; + if (logWhenError) { + cerr << "curve-nbd: ioctl NBD_SET_SOCK failed, devpath: " << devpath + << ", error = " << cpp_strerror(errno) << std::endl; + } + close(devfd); + return -1; + } + + nbdFd_ = devfd; + nbdIndex_ = index; + return 0; +} + +int IOController::SetUp(NBDConfig* config, int sockfd, + uint64_t size, uint64_t flags) { + int ret = -1; + + if (config->devpath.empty()) { + ret = MapOnUnusedNbdDevice(sockfd, &config->devpath); + } else { + ret = MapOnNbdDeviceByDevPath(sockfd, config->devpath); } - int index = ret; - ret = open(config->devpath.c_str(), O_RDWR); if (ret < 0) { - cerr << "curve-ndb: failed to open device: " - << config->devpath << std::endl; - return ret; + return -1; } - int devfd = ret; - ret = InitDevAttr(devfd, config, sockfd, size, flags); + ret = InitDevAttr(config, size, flags); if (ret == 0) { - ret = check_device_size(index, size); + ret = check_device_size(nbdIndex_, size); } if (ret < 0) { - cerr << "curve-ndb: failed to map, status: " + cerr << "curve-nbd: failed to map, status: " << cpp_strerror(ret) << std::endl; - close(devfd); + ioctl(nbdFd_, NBD_CLEAR_SOCK); + close(nbdFd_); + nbdFd_ = -1; + nbdIndex_ = -1; return ret; } - nbdFd_ = devfd; - nbdIndex_ = index; return 0; } int IOController::DisconnectByPath(const std::string& devpath) { int devfd = open(devpath.c_str(), O_RDWR); if (devfd < 0) { - cerr << "curve-ndb: failed to open device: " + cerr << "curve-nbd: failed to open device: " << devpath << ", error = " << cpp_strerror(errno) << std::endl; return devfd; } int ret = ioctl(devfd, NBD_DISCONNECT); if (ret < 0) { - cerr << "curve-ndb: the device is not used. " + cerr << "curve-nbd: the device is not used. " << cpp_strerror(errno) << std::endl; } @@ -396,7 +435,7 @@ int NetLinkController::DisconnectInternal(int index) { genl_handle_msg, NULL); msg = nlmsg_alloc(); if (msg == nullptr) { - cerr << "curve-ndb: Could not allocate netlink message." << std::endl; + cerr << "curve-nbd: Could not allocate netlink message." << std::endl; return -EIO; } @@ -411,7 +450,7 @@ int NetLinkController::DisconnectInternal(int index) { ret = nl_send_sync(sock_, msg); if (ret < 0) { - cerr << "curve-ndb: netlink disconnect failed: " + cerr << "curve-nbd: netlink disconnect failed: " << nl_geterror(ret) << std::endl; return -EIO; } @@ -431,7 +470,7 @@ int NetLinkController::ResizeInternal(int nbdIndex, uint64_t size) { genl_handle_msg, NULL); msg = nlmsg_alloc(); if (msg == nullptr) { - cerr << "curve-ndb: Could not allocate netlink message." << std::endl; + cerr << "curve-nbd: Could not allocate netlink message." << std::endl; return -EIO; } @@ -447,7 +486,7 @@ int NetLinkController::ResizeInternal(int nbdIndex, uint64_t size) { ret = nl_send_sync(sock_, msg); if (ret < 0) { - cerr << "curve-ndb: netlink resize failed: " + cerr << "curve-nbd: netlink resize failed: " << nl_geterror(ret) << std::endl; return -EIO; } diff --git a/nbd/src/NBDController.h b/nbd/src/NBDController.h index 201a6e86d1..204e7b1dbb 100644 --- a/nbd/src/NBDController.h +++ b/nbd/src/NBDController.h @@ -147,8 +147,10 @@ class IOController : public NBDController { int Resize(uint64_t size) override; private: - int InitDevAttr(int devfd, NBDConfig* config, int sockfd, - uint64_t size, uint64_t flags); + int InitDevAttr(NBDConfig* config, uint64_t size, uint64_t flags); + int MapOnUnusedNbdDevice(int sockfd, std::string* devpath); + int MapOnNbdDeviceByDevPath(int sockfd, const std::string& devpath, + bool logWhenError = true); }; class NetLinkController : public NBDController { diff --git a/nbd/src/util.cpp b/nbd/src/util.cpp index db1d5afc90..0f6d9f4ee3 100644 --- a/nbd/src/util.cpp +++ b/nbd/src/util.cpp @@ -89,54 +89,6 @@ int get_nbd_max_count() { return nbds_max; } -std::string find_unused_nbd_device() { - int index = 0; - int devfd = 0; - int nbds_max = get_nbd_max_count(); - char dev[64]; - int sockfd[2]; - - int ret = socketpair(AF_UNIX, SOCK_STREAM, 0, sockfd); - if (ret < 0) { - cerr << "curve-ndb: failed to create socket pair." << std::endl; - return ""; - } - - while (true) { - snprintf(dev, sizeof(dev), "/dev/nbd%d", index); - - ret = open(dev, O_RDWR); - if (ret < 0) { - if (ret == -EPERM && nbds_max != -1 && index < (nbds_max-1)) { - ++index; - continue; - } - cerr << "curve-ndb: failed to find unused device, " - << cpp_strerror(errno) << std::endl; - break; - } - - devfd = ret; - ret = ioctl(devfd, NBD_SET_SOCK, sockfd[0]); - if (ret < 0) { - close(devfd); - ++index; - continue; - } - break; - } - - std::string result = ""; - if (ret == 0) { - result = dev; - ioctl(devfd, NBD_CLEAR_SOCK); - close(devfd); - } - close(sockfd[0]); - close(sockfd[1]); - return result; -} - static bool find_mapped_dev_by_spec(NBDConfig *cfg) { int pid; NBDConfig c; diff --git a/nbd/src/util.h b/nbd/src/util.h index 99df455e2e..c150de8ee0 100644 --- a/nbd/src/util.h +++ b/nbd/src/util.h @@ -48,8 +48,6 @@ extern std::string cpp_strerror(int err); extern int parse_nbd_index(const std::string& devpath); // 获取当前系统能够支持的最大nbd设备数量 extern int get_nbd_max_count(); -// 获取一个当前还未映射的nbd设备名 -extern std::string find_unused_nbd_device(); // 解析用户输入的命令参数 extern int parse_args(std::vector& args, // NOLINT std::ostream *err_msg, diff --git a/test/common/test_timeutility.cpp b/test/common/test_timeutility.cpp index 7c32a9e083..d32d882327 100644 --- a/test/common/test_timeutility.cpp +++ b/test/common/test_timeutility.cpp @@ -18,20 +18,25 @@ TEST(ExpiredTimeTest, CommonTest) { ExpiredTime expiredTime; std::this_thread::sleep_for(std::chrono::seconds(2)); auto expiredSec = expiredTime.ExpiredSec(); - ASSERT_TRUE(expiredSec >= 1.8 && expiredSec <= 2.2); + double expected = 2; + ASSERT_GE(expiredSec, expected * 0.9); + ASSERT_LE(expiredSec, expected * 1.1); } { ExpiredTime expiredTime; std::this_thread::sleep_for(std::chrono::milliseconds(1000)); auto expiredMs = expiredTime.ExpiredMs(); - ASSERT_TRUE(expiredMs >= (1000 - 10) && expiredMs <= (1000 + 10)); + double expected = 1000; + ASSERT_GE(expiredMs, expected * 0.9); + ASSERT_LE(expiredMs, expected * 1.1); } { ExpiredTime expiredTime; std::this_thread::sleep_for(std::chrono::microseconds(1000000)); auto expiredUs = expiredTime.ExpiredUs(); - ASSERT_TRUE(expiredUs >= (1000000 - 200) && - expiredUs <= (1000000 + 200)); + double expected = 1000000; + ASSERT_GE(expiredUs, expected * 0.9); + ASSERT_LE(expiredUs, expected * 1.1); } }