From 48c708b15cbd708f0c8669f096220d8ef193148d Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Mon, 22 Jan 2024 17:08:09 +0300 Subject: [PATCH] crud: add storage-side API to wait for bootstrap This patch add a storage handle to wait till module is bootstrapped. The handle is based on existing `storage_info` check. The handle is for using on storages only: it makes no sense to call in on the router. The current standard pattern is as follows: `crud.init_storage()` is called after `box.cfg` on both masters and replicas. In case of Tarantool 1.x and 2.x rw-instances, `crud.init_storage{wait_until_ready = true}` doesn't introduce anything new. In case of Tarantool 1.x and 2.x ro-instances, it helps to wait until persistent part of bootstrap (functions and grants) are replicated. The main motivation behind this handle is the support of asynchronous start that will be introduced in next commits. The asynchronous start is required to properly work with Tarantool 3 instances: all Tarantool 3.x instances start in ro mode. This patch doesn't change existing behavior for Cartridge roles and `crud.init_storage()` calls. Part of #412 Part of #415 --- CHANGELOG.md | 4 ++ README.md | 3 + crud.lua | 2 + crud/storage.lua | 75 ++++++++++++++++++++++- crud/storage_info.lua | 15 +++-- doc/playground.lua | 2 +- test/integration/select_readview_test.lua | 2 +- test/vshard_helpers/vtest.lua | 13 +++- 8 files changed, 106 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6e4e382..6b41499e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Added +* Storage-side API to wait for bootstrap: + `crud.wait_until_storage_ready()` and `crud.init_storage{wait_until_ready = true}` (#412). + ### Fixed * Compatibility with vshard configuration if UUIDs are omitted (#407). * Compatibility with automatic master discovery in vshard (#409). diff --git a/README.md b/README.md index 7d7f2415..5ddbb729 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,9 @@ across the cluster. The storage-side functions have the same access as a user calling `crud.init_storage()`. Therefore, if `crud` do not have enough access to modify some space, then you need to give access to the user. +You can call `crud.init_storage{wait_until_ready = true}` or +`crud.wait_until_storage_ready()` on storages to wait till API is bootstrapped. + All VShard routers should call `crud.init_router()` after `vshard.router.cfg()` (or enable the `crud-router` role for Cartridge) to make `crud` functions callable via `net.box`. If a user is allowed to execute `crud` functions on diff --git a/crud.lua b/crud.lua index 60301163..852a94b6 100644 --- a/crud.lua +++ b/crud.lua @@ -175,6 +175,8 @@ end -- crud.init_storage = storage.init +crud.wait_until_storage_ready = storage.wait_until_ready + crud.stop_storage = storage.stop return crud diff --git a/crud/storage.lua b/crud/storage.lua index de46ee8e..bf4a3e80 100644 --- a/crud/storage.lua +++ b/crud/storage.lua @@ -1,3 +1,8 @@ +local checks = require('checks') +local errors = require('errors') +local clock = require('clock') +local fiber = require('fiber') + local dev_checks = require('crud.common.dev_checks') local utils = require('crud.common.utils') @@ -19,6 +24,8 @@ local borders = require('crud.borders') local readview = require('crud.readview') local storage_info = require('crud.storage_info') +local StorageInitError = errors.new_class('StorageInitError') + local storage = {} local function init_local_part(_, name, func) @@ -77,18 +84,82 @@ local modules_with_storage_api = { storage_info, } -function storage.init() +local function get_operation_user() + return utils.get_this_replica_user() or 'guest' +end + +function storage.init(opts) + checks({wait_until_ready = '?boolean', timeout = '?number'}) + + opts = opts or {} + if type(box.cfg) ~= 'table' then error('box.cfg() must be called first') end rawset(_G, utils.STORAGE_NAMESPACE, {}) - local user = utils.get_this_replica_user() or 'guest' + local user = get_operation_user() for _, module in ipairs(modules_with_storage_api) do init_storage_call(user, module.storage_api) end + + if opts.wait_until_ready then + storage.wait_until_ready{timeout = opts.timeout} + end +end + +local function run_as_user(user, func, ...) + local prev_user = box.session.user() + box.session.su(user) + + local res_packed = {func(...)} + + box.session.su(prev_user) + + return unpack(res_packed) +end + +local function is_ready_unsafe_check() + local res = box.schema.func.call(storage_info.CRUD_STORAGE_INFO_FUNC_NAME) + + assert(res.status == storage_info.status.RUNNING) +end + +local function wait_until_ready(timeout) + local is_ready = false + local deadline = clock.monotonic() + timeout + + while clock.monotonic() < deadline do + is_ready = pcall(is_ready_unsafe_check) + + if is_ready then + break + end + + fiber.sleep(timeout / 100) + end + + return is_ready +end + +local DEFAULT_WAIT_TIMEOUT = 3 + +function storage.wait_until_ready(opts) + checks({timeout = '?number'}) + + opts = opts or {} + local timeout = opts.timeout or DEFAULT_WAIT_TIMEOUT + + local user = get_operation_user() + local is_ready = run_as_user(user, wait_until_ready, timeout) + + if not is_ready then + return nil, StorageInitError:new("Storage is not bootstrapped") + end + + return true end function storage.stop() diff --git a/crud/storage_info.lua b/crud/storage_info.lua index 84c6a3f4..06b93c95 100644 --- a/crud/storage_info.lua +++ b/crud/storage_info.lua @@ -13,16 +13,23 @@ local storage_info = {} local STORAGE_INFO_FUNC_NAME = 'storage_info_on_storage' local CRUD_STORAGE_INFO_FUNC_NAME = utils.get_storage_call(STORAGE_INFO_FUNC_NAME) +storage_info.status = { + RUNNING = 'running', + UNINITIALIZED = 'uninitialized', + ERROR = 'error', +} + --- Storage status information. -- -- @function storage_info_on_storage -- -- @return a table with storage status. local function storage_info_on_storage() - return {status = "running"} + return {status = storage_info.status.RUNNING} end storage_info.storage_api = {[STORAGE_INFO_FUNC_NAME] = storage_info_on_storage} +storage_info.CRUD_STORAGE_INFO_FUNC_NAME = CRUD_STORAGE_INFO_FUNC_NAME --- Polls replicas for storage state -- @@ -63,7 +70,7 @@ function storage_info.call(opts) local master = utils.get_replicaset_master(replicaset, {cached = false}) replica_state_by_id[replica_id] = { - status = "error", + status = storage_info.status.ERROR, is_master = master == replica } @@ -97,7 +104,7 @@ function storage_info.call(opts) local err_msg = string.format("Error getting storage info for %s", replica_id) if err ~= nil then if err.type == 'ClientError' and err.code == box.error.NO_SUCH_PROC then - replica_state_by_id[replica_id].status = "uninitialized" + replica_state_by_id[replica_id].status = storage_info.status.UNINITIALIZED else log.error("%s: %s", err_msg, err) replica_state_by_id[replica_id].message = tostring(err) @@ -107,7 +114,7 @@ function storage_info.call(opts) replica_state_by_id[replica_id].message = err_msg end else - replica_state_by_id[replica_id].status = result[1].status or "uninitialized" + replica_state_by_id[replica_id].status = result[1].status or storage_info.status.UNINITIALIZED end end diff --git a/doc/playground.lua b/doc/playground.lua index d4e547f0..c7074abd 100755 --- a/doc/playground.lua +++ b/doc/playground.lua @@ -146,7 +146,7 @@ box.once('developers', function() end) -- Initialize crud. -crud.init_storage() +crud.init_storage{wait_until_ready = true} crud.init_router() -- Start a console. diff --git a/test/integration/select_readview_test.lua b/test/integration/select_readview_test.lua index 6a003d9a..b851ae6b 100644 --- a/test/integration/select_readview_test.lua +++ b/test/integration/select_readview_test.lua @@ -2318,7 +2318,7 @@ pgroup.test_stop_select = function(g) g.cluster:server('s2-master'):exec(function(cfg, bootstrap_key) require('vshard.storage').cfg(cfg, box.info[bootstrap_key]) - require('crud').init_storage() + require('crud').init_storage{wait_until_ready = true} end, {g.cfg, bootstrap_key}) end diff --git a/test/vshard_helpers/vtest.lua b/test/vshard_helpers/vtest.lua index 7cc89ac7..d38b930a 100644 --- a/test/vshard_helpers/vtest.lua +++ b/test/vshard_helpers/vtest.lua @@ -544,9 +544,18 @@ local function cluster_new(g, cfg) require(all_init)() end, {all_init}) end - if crud_init then + end + + if crud_init then + for _, replica in pairs(all_servers) do + replica:exec(function() + require('crud').init_storage{wait_until_ready = false} + end) + end + + for _, replica in pairs(all_servers) do replica:exec(function() - require('crud').init_storage() + require('crud').wait_until_storage_ready() end) end end