diff --git a/CHANGELOG.md b/CHANGELOG.md index d6e4e382..6b41499e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Added +* Storage-side API to wait for bootstrap: + `crud.wait_until_storage_ready()` and `crud.init_storage{wait_until_ready = true}` (#412). + ### Fixed * Compatibility with vshard configuration if UUIDs are omitted (#407). * Compatibility with automatic master discovery in vshard (#409). diff --git a/README.md b/README.md index 7d7f2415..5ddbb729 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,9 @@ across the cluster. The storage-side functions have the same access as a user calling `crud.init_storage()`. Therefore, if `crud` do not have enough access to modify some space, then you need to give access to the user. +You can call `crud.init_storage{wait_until_ready = true}` or +`crud.wait_until_storage_ready()` on storages to wait till API is bootstrapped. + All VShard routers should call `crud.init_router()` after `vshard.router.cfg()` (or enable the `crud-router` role for Cartridge) to make `crud` functions callable via `net.box`. If a user is allowed to execute `crud` functions on diff --git a/crud.lua b/crud.lua index 60301163..852a94b6 100644 --- a/crud.lua +++ b/crud.lua @@ -175,6 +175,8 @@ end -- crud.init_storage = storage.init +crud.wait_until_storage_ready = storage.wait_until_ready + crud.stop_storage = storage.stop return crud diff --git a/crud/storage.lua b/crud/storage.lua index de46ee8e..bf4a3e80 100644 --- a/crud/storage.lua +++ b/crud/storage.lua @@ -1,3 +1,8 @@ +local checks = require('checks') +local errors = require('errors') +local clock = require('clock') +local fiber = require('fiber') + local dev_checks = require('crud.common.dev_checks') local utils = require('crud.common.utils') @@ -19,6 +24,8 @@ local borders = require('crud.borders') local readview = require('crud.readview') local storage_info = require('crud.storage_info') +local StorageInitError = errors.new_class('StorageInitError') + local storage = {} local function init_local_part(_, name, func) @@ -77,18 +84,82 @@ local modules_with_storage_api = { storage_info, } -function storage.init() +local function get_operation_user() + return utils.get_this_replica_user() or 'guest' +end + +function storage.init(opts) + checks({wait_until_ready = '?boolean', timeout = '?number'}) + + opts = opts or {} + if type(box.cfg) ~= 'table' then error('box.cfg() must be called first') end rawset(_G, utils.STORAGE_NAMESPACE, {}) - local user = utils.get_this_replica_user() or 'guest' + local user = get_operation_user() for _, module in ipairs(modules_with_storage_api) do init_storage_call(user, module.storage_api) end + + if opts.wait_until_ready then + storage.wait_until_ready{timeout = opts.timeout} + end +end + +local function run_as_user(user, func, ...) + local prev_user = box.session.user() + box.session.su(user) + + local res_packed = {func(...)} + + box.session.su(prev_user) + + return unpack(res_packed) +end + +local function is_ready_unsafe_check() + local res = box.schema.func.call(storage_info.CRUD_STORAGE_INFO_FUNC_NAME) + + assert(res.status == storage_info.status.RUNNING) +end + +local function wait_until_ready(timeout) + local is_ready = false + local deadline = clock.monotonic() + timeout + + while clock.monotonic() < deadline do + is_ready = pcall(is_ready_unsafe_check) + + if is_ready then + break + end + + fiber.sleep(timeout / 100) + end + + return is_ready +end + +local DEFAULT_WAIT_TIMEOUT = 3 + +function storage.wait_until_ready(opts) + checks({timeout = '?number'}) + + opts = opts or {} + local timeout = opts.timeout or DEFAULT_WAIT_TIMEOUT + + local user = get_operation_user() + local is_ready = run_as_user(user, wait_until_ready, timeout) + + if not is_ready then + return nil, StorageInitError:new("Storage is not bootstrapped") + end + + return true end function storage.stop() diff --git a/crud/storage_info.lua b/crud/storage_info.lua index 84c6a3f4..06b93c95 100644 --- a/crud/storage_info.lua +++ b/crud/storage_info.lua @@ -13,16 +13,23 @@ local storage_info = {} local STORAGE_INFO_FUNC_NAME = 'storage_info_on_storage' local CRUD_STORAGE_INFO_FUNC_NAME = utils.get_storage_call(STORAGE_INFO_FUNC_NAME) +storage_info.status = { + RUNNING = 'running', + UNINITIALIZED = 'uninitialized', + ERROR = 'error', +} + --- Storage status information. -- -- @function storage_info_on_storage -- -- @return a table with storage status. local function storage_info_on_storage() - return {status = "running"} + return {status = storage_info.status.RUNNING} end storage_info.storage_api = {[STORAGE_INFO_FUNC_NAME] = storage_info_on_storage} +storage_info.CRUD_STORAGE_INFO_FUNC_NAME = CRUD_STORAGE_INFO_FUNC_NAME --- Polls replicas for storage state -- @@ -63,7 +70,7 @@ function storage_info.call(opts) local master = utils.get_replicaset_master(replicaset, {cached = false}) replica_state_by_id[replica_id] = { - status = "error", + status = storage_info.status.ERROR, is_master = master == replica } @@ -97,7 +104,7 @@ function storage_info.call(opts) local err_msg = string.format("Error getting storage info for %s", replica_id) if err ~= nil then if err.type == 'ClientError' and err.code == box.error.NO_SUCH_PROC then - replica_state_by_id[replica_id].status = "uninitialized" + replica_state_by_id[replica_id].status = storage_info.status.UNINITIALIZED else log.error("%s: %s", err_msg, err) replica_state_by_id[replica_id].message = tostring(err) @@ -107,7 +114,7 @@ function storage_info.call(opts) replica_state_by_id[replica_id].message = err_msg end else - replica_state_by_id[replica_id].status = result[1].status or "uninitialized" + replica_state_by_id[replica_id].status = result[1].status or storage_info.status.UNINITIALIZED end end diff --git a/doc/playground.lua b/doc/playground.lua index d4e547f0..c7074abd 100755 --- a/doc/playground.lua +++ b/doc/playground.lua @@ -146,7 +146,7 @@ box.once('developers', function() end) -- Initialize crud. -crud.init_storage() +crud.init_storage{wait_until_ready = true} crud.init_router() -- Start a console. diff --git a/test/integration/select_readview_test.lua b/test/integration/select_readview_test.lua index 6a003d9a..b851ae6b 100644 --- a/test/integration/select_readview_test.lua +++ b/test/integration/select_readview_test.lua @@ -2318,7 +2318,7 @@ pgroup.test_stop_select = function(g) g.cluster:server('s2-master'):exec(function(cfg, bootstrap_key) require('vshard.storage').cfg(cfg, box.info[bootstrap_key]) - require('crud').init_storage() + require('crud').init_storage{wait_until_ready = true} end, {g.cfg, bootstrap_key}) end diff --git a/test/vshard_helpers/vtest.lua b/test/vshard_helpers/vtest.lua index 7cc89ac7..d38b930a 100644 --- a/test/vshard_helpers/vtest.lua +++ b/test/vshard_helpers/vtest.lua @@ -544,9 +544,18 @@ local function cluster_new(g, cfg) require(all_init)() end, {all_init}) end - if crud_init then + end + + if crud_init then + for _, replica in pairs(all_servers) do + replica:exec(function() + require('crud').init_storage{wait_until_ready = false} + end) + end + + for _, replica in pairs(all_servers) do replica:exec(function() - require('crud').init_storage() + require('crud').wait_until_storage_ready() end) end end