diff --git a/CHANGELOG.md b/CHANGELOG.md index 412b8f8d..4aaa8a00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Added +* Space schema introspection API `crud.schema` (#380). + ### Changed * `deps.sh` installs the `vshard` instead of the `cartridge` by default (#364). You could to specify an environment variable `CARTIRDGE_VERSION` to install diff --git a/README.md b/README.md index a30cb870..3d0fa14d 100644 --- a/README.md +++ b/README.md @@ -1711,6 +1711,82 @@ end rv:close() ``` +### Schema + +`crud` routers provide API to introspect spaces schema. + +```lua +local schema, err = crud.update(space_name, opts) +``` + +where: + +* `space_name` (`?string`) - name of the space (if `nil`, provides info for all spaces) +* `opts`: + * `timeout` (`?number`) - `vshard.call` timeout and vshard master + discovery timeout (in seconds), default value is 2 + * `vshard_router` (`?string|table`) - Cartridge vshard group name or + vshard router instance. Set this parameter if your space is not + a part of the default vshard cluster + +Returns space schema (or spaces schema map), error. + +Beware that schema info is not exactly the same as underlying storage spaces schema. +The reason is that `crud` generates `bucket_id`, if it isn't provided, +so this field is actually nullable for a `crud` user. We also do not expose +`bucket_id` index info since it's a vshard utility and do not related +to application logic. + +**Example:** + +```lua +crud.schema('customers') +--- +- format: + - name: id + type: unsigned + - name: bucket_id + type: unsigned + is_nullable: true + - name: name + type: string + - name: age + type: number + indexes: + 0: + unique: true + parts: + - fieldno: 1 + type: unsigned + exclude_null: false + is_nullable: false + id: 0 + type: TREE + name: primary_index + 2: + unique: false + parts: + - fieldno: 4 + type: number + exclude_null: false + is_nullable: false + id: 2 + type: TREE + name: age +... +``` + +```lua +crud.schema() +--- +- customers: + format: ... + indexes: ... + shops: + format: ... + indexes: ... +``` + ## Cartridge roles `cartridge.roles.crud-storage` is a Tarantool Cartridge role that depends on the diff --git a/crud.lua b/crud.lua index f65ae91b..d2edb907 100644 --- a/crud.lua +++ b/crud.lua @@ -21,6 +21,7 @@ local sharding_metadata = require('crud.common.sharding.sharding_metadata') local utils = require('crud.common.utils') local stats = require('crud.stats') local readview = require('crud.readview') +local schema = require('crud.schema') local crud = {} @@ -152,6 +153,10 @@ crud.storage_info = utils.storage_info -- @function readview crud.readview = readview.new +-- @refer schema.call +-- @function schema +crud.schema = schema.call + --- Initializes crud on node -- -- Exports all functions that are used for calls diff --git a/crud/common/schema.lua b/crud/common/schema.lua index fba631d6..387d37bf 100644 --- a/crud/common/schema.lua +++ b/crud/common/schema.lua @@ -113,11 +113,7 @@ function schema.wrap_func_reload(vshard_router, func, ...) return res, err end -local function get_space_schema_hash(space) - if space == nil then - return '' - end - +schema.get_normalized_space_schema = function(space) local indexes_info = {} for i = 0, table.maxn(space.index) do local index = space.index[i] @@ -133,12 +129,19 @@ local function get_space_schema_hash(space) end end - local space_info = { + return { format = space:format(), indexes = indexes_info, } +end + +local function get_space_schema_hash(space) + if space == nil then + return '' + end - return digest.murmur(msgpack.encode(space_info)) + local sch = schema.get_normalized_space_schema(space) + return digest.murmur(msgpack.encode(sch)) end function schema.filter_obj_fields(obj, field_names) diff --git a/crud/common/utils.lua b/crud/common/utils.lua index 872d716e..38163718 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -110,7 +110,7 @@ local function get_replicaset_by_replica_uuid(replicasets, uuid) return nil end -function utils.get_space(space_name, vshard_router, timeout, replica_uuid) +function utils.get_spaces(vshard_router, timeout, replica_uuid) local replicasets, replicaset timeout = timeout or const.DEFAULT_VSHARD_CALL_TIMEOUT local deadline = fiber.clock() + timeout @@ -160,9 +160,17 @@ function utils.get_space(space_name, vshard_router, timeout, replica_uuid) return nil, GetSpaceError:new(error_msg) end - local space = replicaset.master.conn.space[space_name] + return replicaset.master.conn.space, nil, replicaset.master.conn.schema_version +end + +function utils.get_space(space_name, vshard_router, timeout, replica_uuid) + local spaces, err, schema_version = utils.get_spaces(vshard_router, timeout, replica_uuid) + + if spaces == nil then + return nil, err + end - return space, nil, replicaset.master.conn.schema_version + return spaces[space_name], err, schema_version end function utils.get_space_format(space_name, vshard_router) diff --git a/crud/schema.lua b/crud/schema.lua new file mode 100644 index 00000000..7200650d --- /dev/null +++ b/crud/schema.lua @@ -0,0 +1,112 @@ +local checks = require('checks') +local errors = require('errors') + +local SchemaError = errors.new_class('SchemaError', {capture_stack = false}) + +local schema_module = require('crud.common.schema') +local utils = require('crud.common.utils') + +local schema = {} + +local system_spaces = { + -- https://github.com/tarantool/tarantool/blob/3240201a2f5bac3bddf8a74015db9b351954e0b5/src/box/schema_def.h#L77-L127 + ['_vinyl_deferred_delete'] = true, + ['_schema'] = true, + ['_collation'] = true, + ['_vcollation'] = true, + ['_space'] = true, + ['_vspace'] = true, + ['_sequence'] = true, + ['_sequence_data'] = true, + ['_vsequence'] = true, + ['_index'] = true, + ['_vindex'] = true, + ['_func'] = true, + ['_vfunc'] = true, + ['_user'] = true, + ['_vuser'] = true, + ['_priv'] = true, + ['_vpriv'] = true, + ['_cluster'] = true, + ['_trigger'] = true, + ['_truncate'] = true, + ['_space_sequence'] = true, + ['_vspace_sequence'] = true, + ['_fk_constraint'] = true, + ['_ck_constraint'] = true, + ['_func_index'] = true, + ['_session_settings'] = true, + -- https://github.com/tarantool/vshard/blob/b3c27b32637863e9a03503e641bb7c8c69779a00/vshard/storage/init.lua#L752 + ['_bucket'] = true, + -- https://github.com/tarantool/ddl/blob/b55d0ff7409f32e4d527e2d25444d883bce4163b/test/set_sharding_metadata_test.lua#L92-L98 + ['_ddl_sharding_key'] = true, + ['_ddl_sharding_func'] = true, +} + +local function get_crud_schema(space) + local sch = schema_module.get_normalized_space_schema(space) + + -- bucket_id is not nullable for a storage, yet + -- it is optional for a crud user. + for _, v in ipairs(sch.format) do + if v.name == 'bucket_id' then + v.is_nullable = true + end + end + + for id, v in pairs(sch.indexes) do + -- There is no reason for a user to know about + -- bucket_id index. + if v.name == 'bucket_id' then + sch.indexes[id] = nil + end + end + + return sch +end + +schema.call = function(space_name, opts) + checks('?string', { + vshard_router = '?string|table', + timeout = '?number', + }) + + opts = opts or {} + + local vshard_router, err = utils.get_vshard_router_instance(opts.vshard_router) + if err ~= nil then + return nil, SchemaError:new(err) + end + + local _, err = schema_module.reload_schema(vshard_router) + if err ~= nil then + return nil, SchemaError:new(err) + end + + local spaces, err = utils.get_spaces(vshard_router, opts.timeout) + if err ~= nil then + return nil, SchemaError:new(err) + end + + if space_name ~= nil then + local space = spaces[space_name] + if space == nil then + return nil, SchemaError:new("Space %q doesn't exist", space_name) + end + return get_crud_schema(space) + else + local resp = {} + + for name, space in pairs(spaces) do + -- Can be indexed by space id and space name, + -- so we need to be careful with duplicates. + if type(name) == 'string' and system_spaces[name] == nil then + resp[name] = get_crud_schema(space) + end + end + + return resp + end +end + +return schema diff --git a/test/entrypoint/srv_schema/cartridge_init.lua b/test/entrypoint/srv_schema/cartridge_init.lua new file mode 100755 index 00000000..7998e50d --- /dev/null +++ b/test/entrypoint/srv_schema/cartridge_init.lua @@ -0,0 +1,39 @@ +#!/usr/bin/env tarantool + +require('strict').on() +_G.is_initialized = function() return false end + +local log = require('log') +local errors = require('errors') +local cartridge = require('cartridge') + +if package.setsearchroot ~= nil then + package.setsearchroot() +else + package.path = package.path .. debug.sourcedir() .. "/?.lua;" +end + +package.preload['customers-storage'] = function() + return { + role_name = 'customers-storage', + init = require('storage_init') + } +end + +local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { + advertise_uri = 'localhost:3301', + http_port = 8081, + bucket_count = 3000, + roles = { + 'cartridge.roles.crud-router', + 'cartridge.roles.crud-storage', + 'customers-storage', + }} +) + +if not ok then + log.error('%s', err) + os.exit(1) +end + +_G.is_initialized = cartridge.is_healthy diff --git a/test/entrypoint/srv_schema/storage_init.lua b/test/entrypoint/srv_schema/storage_init.lua new file mode 100644 index 00000000..9b24622c --- /dev/null +++ b/test/entrypoint/srv_schema/storage_init.lua @@ -0,0 +1,53 @@ +return function() + if box.info.ro == true then + return + end + + local engine = os.getenv('ENGINE') or 'memtx' + + local customers_space = box.schema.space.create('customers', { + format = { + {name = 'id', type = 'unsigned'}, + {name = 'bucket_id', type = 'unsigned'}, + {name = 'name', type = 'string'}, + {name = 'age', type = 'number'}, + }, + if_not_exists = true, + engine = engine, + }) + customers_space:create_index('id', { + parts = { {field = 'id'} }, + if_not_exists = true, + }) + customers_space:create_index('bucket_id', { + parts = { {field = 'bucket_id'} }, + unique = false, + if_not_exists = true, + }) + + local shops_space = box.schema.space.create('shops', { + format = { + {name = 'registry_id', type = 'unsigned'}, + {name = 'bucket_id', type = 'unsigned'}, + {name = 'name', type = 'string'}, + {name = 'address', type = 'string'}, + {name = 'owner', type = 'string', is_nullable = true}, + }, + if_not_exists = true, + engine = engine, + }) + shops_space:create_index('registry', { + parts = { {field = 'registry_id'} }, + if_not_exists = true, + }) + shops_space:create_index('bucket_id', { + parts = { {field = 'bucket_id'} }, + unique = false, + if_not_exists = true, + }) + shops_space:create_index('address', { + parts = { {field = 'address'} }, + unique = true, + if_not_exists = true, + }) +end diff --git a/test/helper.lua b/test/helper.lua index 94a8daed..d814a0db 100644 --- a/test/helper.lua +++ b/test/helper.lua @@ -768,4 +768,28 @@ function helpers.backend_matrix(base_matrix) return matrix end +function helpers.schema_compatibility(schema) + -- https://github.com/tarantool/tarantool/issues/4091 + if not helpers.tarantool_version_at_least(2, 2, 1) then + for _, s in pairs(schema) do + for _, i in pairs(s.indexes) do + i.unique = false + end + end + end + + -- https://github.com/tarantool/tarantool/commit/17c9c034933d726925910ce5bf8b20e8e388f6e3 + if not helpers.tarantool_version_at_least(2, 8, 1) then + for _, s in pairs(schema) do + for _, i in pairs(s.indexes) do + for _, p in pairs(i.parts) do + p.exclude_null = nil + end + end + end + end + + return schema +end + return helpers diff --git a/test/integration/schema_test.lua b/test/integration/schema_test.lua new file mode 100644 index 00000000..71233224 --- /dev/null +++ b/test/integration/schema_test.lua @@ -0,0 +1,94 @@ +local t = require('luatest') + +local helpers = require('test.helper') + +local pgroup = t.group('schema', helpers.backend_matrix({ + {engine = 'memtx'}, + {engine = 'vinyl'}, +})) + +pgroup.before_all(function(g) + helpers.start_default_cluster(g, 'srv_schema') + + g.router = helpers.get_router(g.cluster, g.params.backend) +end) + +pgroup.after_all(function(g) + helpers.stop_cluster(g.cluster, g.params.backend) +end) + +local function expected_schema() + local schema = { + customers = { + format = { + {name = "id", type = "unsigned"}, + {name = "bucket_id", type = "unsigned", is_nullable = true}, + {name = "name", type = "string"}, + {name = "age", type = "number"}, + }, + indexes = { + [0] = { + id = 0, + name = "id", + parts = {{exclude_null = false, fieldno = 1, is_nullable = false, type = "unsigned"}}, + type = "TREE", + unique = true, + }, + }, + }, + shops = { + format = { + {name = 'registry_id', type = 'unsigned'}, + {name = 'bucket_id', type = 'unsigned', is_nullable = true}, + {name = 'name', type = 'string'}, + {name = 'address', type = 'string'}, + {name = 'owner', type = 'string', is_nullable = true}, + }, + indexes = { + [0] = { + id = 0, + name = "registry", + parts = {{exclude_null = false, fieldno = 1, is_nullable = false, type = "unsigned"}}, + type = "TREE", + unique = true, + }, + [2] = { + id = 2, + name = "address", + parts = {{exclude_null = false, fieldno = 4, is_nullable = false, type = "string"}}, + type = "TREE", + unique = true, + }, + }, + }, + } + + return helpers.schema_compatibility(schema) +end + +pgroup.test_get_all = function(g) + local result, err = g.router:call('crud.schema') + + t.assert_equals(err, nil) + t.assert_equals(result, expected_schema()) +end + +pgroup.test_get_one = function(g) + local result, err = g.router:call('crud.schema', {'customers'}) + + t.assert_equals(err, nil) + t.assert_equals(result, expected_schema()['customers']) +end + +pgroup.test_get_non_existent_space = function(g) + local result, err = g.router:call('crud.schema', {'owners'}) + + t.assert_equals(result, nil, err) + t.assert_str_contains(err.err, "Space \"owners\" doesn't exist") +end + +pgroup.test_timeout_option = function(g) + local _, err = g.router:call('crud.schema', {nil, {timeout = 2}}) + + t.assert_equals(err, nil) +end diff --git a/test/integration/vshard_custom_test.lua b/test/integration/vshard_custom_test.lua index e8b02672..4bff2fc5 100644 --- a/test/integration/vshard_custom_test.lua +++ b/test/integration/vshard_custom_test.lua @@ -1631,3 +1631,40 @@ pgroup.test_call_upsert_object_many_wrong_option = function(g) t.assert_str_contains(errs[1].err, "Invalid opts.vshard_router table value, a vshard router instance has been expected") end + +pgroup.test_schema = function(g) + local result, err = g.router:call('crud.schema', {'customers', {vshard_router = 'customers'}}) + + t.assert_equals(err, nil) + t.assert_equals(result, helpers.schema_compatibility({customers = { + format = { + {name = "id", type = "unsigned", is_nullable = false}, + {name = "bucket_id", type = "unsigned", is_nullable = true}, + {name = "name", type = "string", is_nullable = false}, + {name = "age", type = "number", is_nullable = false}, + }, + indexes = { + [0] = { + id = 0, + name = "pk", + parts = {{exclude_null = false, fieldno = 1, is_nullable = false, type = "unsigned"}}, + type = "TREE", + unique = true, + }, + [2] = { + id = 2, + name = "age", + parts = {{exclude_null = false, fieldno = 4, is_nullable = false, type = "number"}}, + type = "TREE", + unique = false, + }, + }, + }})['customers']) +end + +pgroup.test_schema_router_mismatch = function(g) + local result, err = g.router:call('crud.schema', {'customers', {vshard_router = 'locations'}}) + + t.assert_equals(result, nil, err) + t.assert_str_contains(err.err, "Space \"customers\" doesn't exist") +end