storage: support async bootstrap

Support asynchronous bootstrap for crud storages. The main motivation is
compatibility with Tarantool 3 instances, which start in read-only mode.

We do not support async bootstrap for crud routers since they do
not change `box`, thus may start on read-only instances without any
issues.

Part of #412
Part of #415

CHANGELOG.md

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+### Added
+* Asynchronous bootstrap support for storages (#412).
+
 ### Changed
 * Explicitly forbid datetime interval conditions (#373).
 

README.md

@@ -90,6 +90,11 @@ across the cluster. The storage-side functions have the same access
 as a user calling `crud.init_storage()`. Therefore, if `crud` do not have
 enough access to modify some space, then you need to give access to the user.
 
+You can call `crud.init_storage{async = true}` to bootstrap procedures grants
+asynchronously. It is useful in case your application master instances may
+start in ro mode (for example, if you use Tarantool 3.x). By default,
+synchronous bootstrap is used.
+
 All VShard routers should call `crud.init_router()` after `vshard.router.cfg()`
 (or enable the `crud-router` role for Cartridge) to make `crud` functions
 callable via `net.box`. If a user is allowed to execute `crud` functions on

crud/common/stash.lua

@@ -23,6 +23,9 @@ local stash = {}
 -- @tfield string select_module_compat_info
 --  Stash for select compatability version registry.
 --
+-- @tfield string storage_init
+--  Stash for storage initializing tools.
+--
 stash.name = {
     cfg = '__crud_cfg',
     stats_internal = '__crud_stats_internal',
@@ -31,6 +34,7 @@ stash.name = {
     ddl_triggers = '__crud_ddl_spaces_triggers',
     select_module_compat_info = '__select_module_compat_info',
     storage_readview = '__crud_storage_readview',
+    storage_init = '__crud_storage_init',
 }
 
 --- Setup Tarantool Cartridge reload.

crud/common/utils.lua

@@ -683,6 +683,10 @@ local function determine_enabled_features()
 
     enabled_tarantool_features.netbox_skip_header_option = is_version_ge(major, minor, patch, suffix,
                                                                          2, 2, 0, nil)
+
+    -- https://github.com/tarantool/tarantool/commit/11f2d999a92e45ee41b8c8d0014d8a09290fef7b
+    enabled_tarantool_features.box_watch = is_version_ge(major, minor, patch, suffix,
+                                                         2, 10, 0, 'beta2')
 end
 
 determine_enabled_features()

crud/storage.lua

@@ -1,4 +1,7 @@
+local checks = require('checks')
+
 local dev_checks = require('crud.common.dev_checks')
+local stash = require('crud.common.stash')
 local utils = require('crud.common.utils')
 
 local sharding_metadata = require('crud.common.sharding.sharding_metadata')
@@ -21,6 +24,8 @@ local storage_info = require('crud.storage_info')
 
 local storage = {}
 
+local internal_stash = stash.get(stash.name.storage_init)
+
 local function init_local_part(_, name, func)
     rawset(_G[utils.STORAGE_NAMESPACE], name, func)
 end
@@ -77,11 +82,7 @@ local modules_with_storage_api = {
     storage_info,
 }
 
-function storage.init()
-    if type(box.cfg) ~= 'table' then
-        error('box.cfg() must be called first')
-    end
-
+local function init_impl()
     rawset(_G, utils.STORAGE_NAMESPACE, {})
 
     -- User is required only for persistent part of the init.
@@ -98,7 +99,40 @@ function storage.init()
     end
 end
 
+function storage.init(opts)
+    checks({async = '?boolean'})
+
+    opts = opts or {}
+
+    if opts.async == nil then
+        opts.async = false
+    end
+
+    if type(box.cfg) ~= 'table' then
+        error('box.cfg() must be called first')
+    end
+
+    if internal_stash.watcher ~= nil then
+        internal_stash.watcher:unregister()
+        internal_stash.watcher = nil
+    end
+
+    if opts.async then
+        assert(utils.tarantool_supports_box_watch(),
+               'async start is supported only for Tarantool versions with box.watch support')
+
+        internal_stash.watcher = box.watch('box.status', init_impl)
+    else
+        init_impl()
+    end
+end
+
 function storage.stop()
+    if internal_stash.watcher ~= nil then
+        internal_stash.watcher:unregister()
+        internal_stash.watcher = nil
+    end
+
     rawset(_G, utils.STORAGE_NAMESPACE, nil)
 end
 

test/helper.lua

@@ -1254,4 +1254,16 @@ function helpers.wait_schema_init()
     return helpers.wait_func_flag(helpers.SCHEMA_READY_FLAG)
 end
 
+function helpers.is_box_watch_supported()
+    return crud_utils.tarantool_supports_box_watch()
+end
+
+function helpers.skip_if_box_watch_unsupported()
+    t.skip_if(not helpers.is_box_watch_supported(), 'box.watch is not supported')
+end
+
+function helpers.skip_if_box_watch_supported()
+    t.skip_if(helpers.is_box_watch_supported(), 'box.watch is supported')
+end
+
 return helpers

test/integration/async_bootstrap_test.lua

@@ -0,0 +1,81 @@
+local t = require('luatest')
+
+local helpers = require('test.helper')
+local vtest = require('test.vshard_helpers.vtest')
+
+local g = t.group('async_bootstrap')
+
+local function prepare_clean_cluster(cg)
+    local cfg = {
+        sharding = {
+            ['s-1'] = {
+                replicas = {
+                    ['s1-master'] = {
+                        instance_uuid = helpers.uuid('b', 1),
+                        master = true,
+                    },
+                },
+            },
+        },
+        bucket_count = 3000,
+        storage_entrypoint = nil,
+        router_entrypoint = nil,
+        all_entrypoint = nil,
+        crud_init = false,
+    }
+
+    cg.cfg = vtest.config_new(cfg)
+    vtest.cluster_new(cg, cg.cfg)
+end
+
+
+g.test_async_storage_bootstrap = function(cg)
+    helpers.skip_if_box_watch_unsupported()
+
+    -- Prepare a clean vshard cluster with 1 router and 1 storage.
+    prepare_clean_cluster(cg)
+
+    -- Sync bootstrap router.
+    cg.cluster:server('router'):exec(function()
+        require('crud').init_router()
+    end)
+
+    -- Async bootstrap storage.
+    cg.cluster:server('s1-master'):exec(function()
+        require('crud').init_storage{async = true}
+    end)
+
+    -- Assert storage is ready after some time.
+    cg.router = cg.cluster:server('router')
+    helpers.wait_crud_is_ready_on_cluster(cg, {backend = helpers.backend.VSHARD})
+end
+
+g.after_test('test_async_storage_bootstrap', function(cg)
+    if cg.cluster ~= nil then
+        cg.cluster:drop()
+    end
+end)
+
+
+g.test_async_storage_bootstrap_unsupported = function(cg)
+    helpers.skip_if_box_watch_supported()
+
+    -- Prepare a clean vshard cluster with 1 router and 1 storage.
+    prepare_clean_cluster(cg)
+
+    -- Async bootstrap storage (fails).
+    cg.cluster:server('s1-master'):exec(function()
+        t.assert_error_msg_contains(
+            'async start is supported only for Tarantool versions with box.watch support',
+            function()
+                require('crud').init_storage{async = true}
+            end
+        )
+    end)
+end
+
+g.after_test('test_async_storage_bootstrap_unsupported', function(cg)
+    if cg.cluster ~= nil then
+        cg.cluster:drop()
+    end
+end)