crud: add storage-side API to wait for bootstrap

This patch add a storage handle to wait till module is bootstrapped.
The handle is based on existing `storage_info` check. The handle is
for using on storages only: it makes no sense to call in on the router.

The current standard pattern is as follows: `crud.init_storage()` is
called after `box.cfg` on both masters and replicas. In case of
Tarantool 1.x and 2.x rw-instances,
`crud.init_storage{wait_until_ready = true}` doesn't introduce anything
new. In case of Tarantool 1.x and 2.x ro-instances, it helps to wait
until persistent part of bootstrap (functions and grants) are
replicated. The main motivation behind this handle is the support of
asynchronous start that will be introduced in next commits.
The asynchronous start is required to properly work with Tarantool 3
instances: all Tarantool 3.x instances start in ro mode.

This patch doesn't change existing behavior for Cartridge roles and
`crud.init_storage()` calls.

Part of #412
Part of #415

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+### Added
+* Storage-side API to wait for bootstrap:
+  `crud.wait_until_storage_ready()` and `crud.init_storage{wait_until_ready = true}` (#412).
+
 ### Fixed
 * Compatibility with vshard configuration if UUIDs are omitted (#407).
 * Compatibility with automatic master discovery in vshard (#409).

README.md

@@ -90,6 +90,9 @@ across the cluster. The storage-side functions have the same access
 as a user calling `crud.init_storage()`. Therefore, if `crud` do not have
 enough access to modify some space, then you need to give access to the user.
 
+You can call `crud.init_storage{wait_until_ready = true}` or
+`crud.wait_until_storage_ready()` on storages to wait till API is bootstrapped.
+
 All VShard routers should call `crud.init_router()` after `vshard.router.cfg()`
 (or enable the `crud-router` role for Cartridge) to make `crud` functions
 callable via `net.box`. If a user is allowed to execute `crud` functions on

crud.lua

@@ -175,6 +175,8 @@ end
 --
 crud.init_storage = storage.init
 
+crud.wait_until_storage_ready = storage.wait_until_ready
+
 crud.stop_storage = storage.stop
 
 return crud

crud/storage.lua

@@ -1,3 +1,8 @@
+local checks = require('checks')
+local errors = require('errors')
+local clock = require('clock')
+local fiber = require('fiber')
+
 local dev_checks = require('crud.common.dev_checks')
 local utils = require('crud.common.utils')
 
@@ -19,6 +24,8 @@ local borders = require('crud.borders')
 local readview = require('crud.readview')
 local storage_info = require('crud.storage_info')
 
+local StorageInitError = errors.new_class('StorageInitError')
+
 local storage = {}
 
 local function init_local_part(_, name, func)
@@ -77,18 +84,82 @@ local modules_with_storage_api = {
     storage_info,
 }
 
-function storage.init()
+local function get_operation_user()
+    return utils.get_this_replica_user() or 'guest'
+end
+
+function storage.init(opts)
+    checks({wait_until_ready = '?boolean', timeout = '?number'})
+
+    opts = opts or {}
+
     if type(box.cfg) ~= 'table' then
         error('box.cfg() must be called first')
     end
 
     rawset(_G, utils.STORAGE_NAMESPACE, {})
 
-    local user = utils.get_this_replica_user() or 'guest'
+    local user = get_operation_user()
 
     for _, module in ipairs(modules_with_storage_api) do
         init_storage_call(user, module.storage_api)
     end
+
+    if opts.wait_until_ready then
+        storage.wait_until_ready{timeout = opts.timeout}
+    end
+end
+
+local function run_as_user(user, func, ...)
+    local prev_user = box.session.user()
+    box.session.su(user)
+
+    local res_packed = {func(...)}
+
+    box.session.su(prev_user)
+
+    return unpack(res_packed)
+end
+
+local function is_ready_unsafe_check()
+    local res = box.schema.func.call(storage_info.CRUD_STORAGE_INFO_FUNC_NAME)
+
+    assert(res.status == storage_info.status.RUNNING)
+end
+
+local function wait_until_ready(timeout)
+    local is_ready = false
+    local deadline = clock.monotonic() + timeout
+
+    while clock.monotonic() < deadline do
+        is_ready = pcall(is_ready_unsafe_check)
+
+        if is_ready then
+            break
+        end
+
+        fiber.sleep(timeout / 100)
+    end
+
+    return is_ready
+end
+
+local DEFAULT_WAIT_TIMEOUT = 3
+
+function storage.wait_until_ready(opts)
+    checks({timeout = '?number'})
+
+    opts = opts or {}
+    local timeout = opts.timeout or DEFAULT_WAIT_TIMEOUT
+
+    local user = get_operation_user()
+    local is_ready = run_as_user(user, wait_until_ready, timeout)
+
+    if not is_ready then
+        return nil, StorageInitError:new("Storage is not bootstrapped")
+    end
+
+    return true
 end
 
 function storage.stop()

crud/storage_info.lua

@@ -13,16 +13,23 @@ local storage_info = {}
 local STORAGE_INFO_FUNC_NAME = 'storage_info_on_storage'
 local CRUD_STORAGE_INFO_FUNC_NAME = utils.get_storage_call(STORAGE_INFO_FUNC_NAME)
 
+storage_info.status = {
+    RUNNING = 'running',
+    UNINITIALIZED = 'uninitialized',
+    ERROR = 'error',
+}
+
 --- Storage status information.
 --
 -- @function storage_info_on_storage
 --
 -- @return a table with storage status.
 local function storage_info_on_storage()
-    return {status = "running"}
+    return {status = storage_info.status.RUNNING}
 end
 
 storage_info.storage_api = {[STORAGE_INFO_FUNC_NAME] = storage_info_on_storage}
+storage_info.CRUD_STORAGE_INFO_FUNC_NAME = CRUD_STORAGE_INFO_FUNC_NAME
 
 --- Polls replicas for storage state
 --
@@ -63,7 +70,7 @@ function storage_info.call(opts)
             local master = utils.get_replicaset_master(replicaset, {cached = false})
 
             replica_state_by_id[replica_id] = {
-                status = "error",
+                status = storage_info.status.ERROR,
                 is_master = master == replica
             }
 
@@ -97,7 +104,7 @@ function storage_info.call(opts)
             local err_msg = string.format("Error getting storage info for %s", replica_id)
             if err ~= nil then
                 if err.type == 'ClientError' and err.code == box.error.NO_SUCH_PROC then
-                    replica_state_by_id[replica_id].status = "uninitialized"
+                    replica_state_by_id[replica_id].status = storage_info.status.UNINITIALIZED
                 else
                     log.error("%s: %s", err_msg, err)
                     replica_state_by_id[replica_id].message = tostring(err)
@@ -107,7 +114,7 @@ function storage_info.call(opts)
                 replica_state_by_id[replica_id].message = err_msg
             end
         else
-            replica_state_by_id[replica_id].status = result[1].status or "uninitialized"
+            replica_state_by_id[replica_id].status = result[1].status or storage_info.status.UNINITIALIZED
         end
     end
 

doc/playground.lua

@@ -146,7 +146,7 @@ box.once('developers', function()
 end)
 
 -- Initialize crud.
-crud.init_storage()
+crud.init_storage{wait_until_ready = true}
 crud.init_router()
 
 -- Start a console.

test/integration/select_readview_test.lua

@@ -2318,7 +2318,7 @@ pgroup.test_stop_select = function(g)
 
         g.cluster:server('s2-master'):exec(function(cfg, bootstrap_key)
             require('vshard.storage').cfg(cfg, box.info[bootstrap_key])
-            require('crud').init_storage()
+            require('crud').init_storage{wait_until_ready = true}
         end, {g.cfg, bootstrap_key})
     end
 

test/vshard_helpers/vtest.lua

@@ -544,9 +544,18 @@ local function cluster_new(g, cfg)
                 require(all_init)()
             end, {all_init})
         end
-        if crud_init then
+    end
+
+    if crud_init then
+        for _, replica in pairs(all_servers) do
+            replica:exec(function()
+                require('crud').init_storage{wait_until_ready = false}
+            end)
+        end
+
+        for _, replica in pairs(all_servers) do
             replica:exec(function()
-                require('crud').init_storage()
+                require('crud').wait_until_storage_ready()
             end)
         end
     end