From fd2f14b599feae6935b4fb1b528838152a2001ec Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Mon, 25 Mar 2024 16:37:35 +0300 Subject: [PATCH] storage: support async bootstrap Support asynchronous bootstrap for crud storages. The main motivation is compatibility with Tarantool 3 instances, which start in read-only mode. We do not support async bootstrap for crud routers since they do not change `box`, thus may start on read-only instances without any issues. Part of #412 Part of #415 --- CHANGELOG.md | 3 + README.md | 5 ++ crud/common/stash.lua | 4 ++ crud/common/utils.lua | 4 ++ crud/storage.lua | 44 ++++++++++-- test/helper.lua | 12 ++++ test/integration/async_bootstrap_test.lua | 81 +++++++++++++++++++++++ 7 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 test/integration/async_bootstrap_test.lua diff --git a/CHANGELOG.md b/CHANGELOG.md index 243646f3..6c9acfb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Added +* Asynchronous bootstrap support for storages (#412). + ### Changed * Explicitly forbid datetime interval conditions (#373). diff --git a/README.md b/README.md index 7d7f2415..f37fb15c 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,11 @@ across the cluster. The storage-side functions have the same access as a user calling `crud.init_storage()`. Therefore, if `crud` do not have enough access to modify some space, then you need to give access to the user. +You can call `crud.init_storage{async = true}` to bootstrap procedures grants +asynchronously. It is useful in case your application master instances may +start in ro mode (for example, if you use Tarantool 3.x). By default, +synchronous bootstrap is used. + All VShard routers should call `crud.init_router()` after `vshard.router.cfg()` (or enable the `crud-router` role for Cartridge) to make `crud` functions callable via `net.box`. If a user is allowed to execute `crud` functions on diff --git a/crud/common/stash.lua b/crud/common/stash.lua index bfc20639..baf658c4 100644 --- a/crud/common/stash.lua +++ b/crud/common/stash.lua @@ -23,6 +23,9 @@ local stash = {} -- @tfield string select_module_compat_info -- Stash for select compatability version registry. -- +-- @tfield string storage_init +-- Stash for storage initializing tools. +-- stash.name = { cfg = '__crud_cfg', stats_internal = '__crud_stats_internal', @@ -31,6 +34,7 @@ stash.name = { ddl_triggers = '__crud_ddl_spaces_triggers', select_module_compat_info = '__select_module_compat_info', storage_readview = '__crud_storage_readview', + storage_init = '__crud_storage_init', } --- Setup Tarantool Cartridge reload. diff --git a/crud/common/utils.lua b/crud/common/utils.lua index ecf33759..2b47bad6 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -683,6 +683,10 @@ local function determine_enabled_features() enabled_tarantool_features.netbox_skip_header_option = is_version_ge(major, minor, patch, suffix, 2, 2, 0, nil) + + -- https://github.com/tarantool/tarantool/commit/11f2d999a92e45ee41b8c8d0014d8a09290fef7b + enabled_tarantool_features.box_watch = is_version_ge(major, minor, patch, suffix, + 2, 10, 0, 'beta2') end determine_enabled_features() diff --git a/crud/storage.lua b/crud/storage.lua index 375fdab2..5524574f 100644 --- a/crud/storage.lua +++ b/crud/storage.lua @@ -1,4 +1,7 @@ +local checks = require('checks') + local dev_checks = require('crud.common.dev_checks') +local stash = require('crud.common.stash') local utils = require('crud.common.utils') local sharding_metadata = require('crud.common.sharding.sharding_metadata') @@ -21,6 +24,8 @@ local storage_info = require('crud.storage_info') local storage = {} +local internal_stash = stash.get(stash.name.storage_init) + local function init_local_part(_, name, func) rawset(_G[utils.STORAGE_NAMESPACE], name, func) end @@ -77,11 +82,7 @@ local modules_with_storage_api = { storage_info, } -function storage.init() - if type(box.cfg) ~= 'table' then - error('box.cfg() must be called first') - end - +local function init_impl() rawset(_G, utils.STORAGE_NAMESPACE, {}) -- User is required only for persistent part of the init. @@ -98,7 +99,40 @@ function storage.init() end end +function storage.init(opts) + checks({async = '?boolean'}) + + opts = opts or {} + + if opts.async == nil then + opts.async = false + end + + if type(box.cfg) ~= 'table' then + error('box.cfg() must be called first') + end + + if internal_stash.watcher ~= nil then + internal_stash.watcher:unregister() + internal_stash.watcher = nil + end + + if opts.async then + assert(utils.tarantool_supports_box_watch(), + 'async start is supported only for Tarantool versions with box.watch support') + + internal_stash.watcher = box.watch('box.status', init_impl) + else + init_impl() + end +end + function storage.stop() + if internal_stash.watcher ~= nil then + internal_stash.watcher:unregister() + internal_stash.watcher = nil + end + rawset(_G, utils.STORAGE_NAMESPACE, nil) end diff --git a/test/helper.lua b/test/helper.lua index 6f7a9fe4..f7afb860 100644 --- a/test/helper.lua +++ b/test/helper.lua @@ -1254,4 +1254,16 @@ function helpers.wait_schema_init() return helpers.wait_func_flag(helpers.SCHEMA_READY_FLAG) end +function helpers.is_box_watch_supported() + return crud_utils.tarantool_supports_box_watch() +end + +function helpers.skip_if_box_watch_unsupported() + t.skip_if(not helpers.is_box_watch_supported(), 'box.watch is not supported') +end + +function helpers.skip_if_box_watch_supported() + t.skip_if(helpers.is_box_watch_supported(), 'box.watch is supported') +end + return helpers diff --git a/test/integration/async_bootstrap_test.lua b/test/integration/async_bootstrap_test.lua new file mode 100644 index 00000000..921868b0 --- /dev/null +++ b/test/integration/async_bootstrap_test.lua @@ -0,0 +1,81 @@ +local t = require('luatest') + +local helpers = require('test.helper') +local vtest = require('test.vshard_helpers.vtest') + +local g = t.group('async_bootstrap') + +local function prepare_clean_cluster(cg) + local cfg = { + sharding = { + ['s-1'] = { + replicas = { + ['s1-master'] = { + instance_uuid = helpers.uuid('b', 1), + master = true, + }, + }, + }, + }, + bucket_count = 3000, + storage_entrypoint = nil, + router_entrypoint = nil, + all_entrypoint = nil, + crud_init = false, + } + + cg.cfg = vtest.config_new(cfg) + vtest.cluster_new(cg, cg.cfg) +end + + +g.test_async_storage_bootstrap = function(cg) + helpers.skip_if_box_watch_unsupported() + + -- Prepare a clean vshard cluster with 1 router and 1 storage. + prepare_clean_cluster(cg) + + -- Sync bootstrap router. + cg.cluster:server('router'):exec(function() + require('crud').init_router() + end) + + -- Async bootstrap storage. + cg.cluster:server('s1-master'):exec(function() + require('crud').init_storage{async = true} + end) + + -- Assert storage is ready after some time. + cg.router = cg.cluster:server('router') + helpers.wait_crud_is_ready_on_cluster(cg, {backend = helpers.backend.VSHARD}) +end + +g.after_test('test_async_storage_bootstrap', function(cg) + if cg.cluster ~= nil then + cg.cluster:drop() + end +end) + + +g.test_async_storage_bootstrap_unsupported = function(cg) + helpers.skip_if_box_watch_supported() + + -- Prepare a clean vshard cluster with 1 router and 1 storage. + prepare_clean_cluster(cg) + + -- Async bootstrap storage (fails). + cg.cluster:server('s1-master'):exec(function() + t.assert_error_msg_contains( + 'async start is supported only for Tarantool versions with box.watch support', + function() + require('crud').init_storage{async = true} + end + ) + end) +end + +g.after_test('test_async_storage_bootstrap_unsupported', function(cg) + if cg.cluster ~= nil then + cg.cluster:drop() + end +end)