From bbe1bd57ec5a4323cb7f98cb022978ece384dc3d Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Mon, 9 Oct 2023 15:42:24 +0300 Subject: [PATCH] schema: support cached schema --- README.md | 2 + crud/schema.lua | 13 +- test/entrypoint/srv_schema/storage_init.lua | 111 ++++++++++------- test/integration/schema_test.lua | 126 +++++++++++++------- 4 files changed, 167 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 3d0fa14d..146d9d12 100644 --- a/README.md +++ b/README.md @@ -1728,6 +1728,8 @@ where: * `vshard_router` (`?string|table`) - Cartridge vshard group name or vshard router instance. Set this parameter if your space is not a part of the default vshard cluster + * `cached` (`?boolean`) - if `false`, reloads storages schema on call; + if `true`, return last known schema; default value is `false` Returns space schema (or spaces schema map), error. diff --git a/crud/schema.lua b/crud/schema.lua index 7200650d..062587c7 100644 --- a/crud/schema.lua +++ b/crud/schema.lua @@ -8,7 +8,7 @@ local utils = require('crud.common.utils') local schema = {} -local system_spaces = { +schema.system_spaces = { -- https://github.com/tarantool/tarantool/blob/3240201a2f5bac3bddf8a74015db9b351954e0b5/src/box/schema_def.h#L77-L127 ['_vinyl_deferred_delete'] = true, ['_schema'] = true, @@ -69,6 +69,7 @@ schema.call = function(space_name, opts) checks('?string', { vshard_router = '?string|table', timeout = '?number', + cached = '?boolean', }) opts = opts or {} @@ -78,9 +79,11 @@ schema.call = function(space_name, opts) return nil, SchemaError:new(err) end - local _, err = schema_module.reload_schema(vshard_router) - if err ~= nil then - return nil, SchemaError:new(err) + if opts.cached ~= true then + local _, err = schema_module.reload_schema(vshard_router) + if err ~= nil then + return nil, SchemaError:new(err) + end end local spaces, err = utils.get_spaces(vshard_router, opts.timeout) @@ -100,7 +103,7 @@ schema.call = function(space_name, opts) for name, space in pairs(spaces) do -- Can be indexed by space id and space name, -- so we need to be careful with duplicates. - if type(name) == 'string' and system_spaces[name] == nil then + if type(name) == 'string' and schema.system_spaces[name] == nil then resp[name] = get_crud_schema(space) end end diff --git a/test/entrypoint/srv_schema/storage_init.lua b/test/entrypoint/srv_schema/storage_init.lua index 9b24622c..4351fa87 100644 --- a/test/entrypoint/srv_schema/storage_init.lua +++ b/test/entrypoint/srv_schema/storage_init.lua @@ -1,3 +1,5 @@ +local schema = require('crud.schema') + return function() if box.info.ro == true then return @@ -5,49 +7,78 @@ return function() local engine = os.getenv('ENGINE') or 'memtx' - local customers_space = box.schema.space.create('customers', { - format = { - {name = 'id', type = 'unsigned'}, - {name = 'bucket_id', type = 'unsigned'}, - {name = 'name', type = 'string'}, - {name = 'age', type = 'number'}, - }, - if_not_exists = true, - engine = engine, - }) - customers_space:create_index('id', { - parts = { {field = 'id'} }, - if_not_exists = true, - }) - customers_space:create_index('bucket_id', { - parts = { {field = 'bucket_id'} }, - unique = false, - if_not_exists = true, - }) - - local shops_space = box.schema.space.create('shops', { - format = { + rawset(_G, 'reload_schema', function() + for name, space in pairs(box.space) do + -- Can be indexed by space id and space name, + -- so we need to be careful with duplicates. + if type(name) == 'string' and schema.system_spaces[name] == nil then + space:drop() + end + end + + local customers_space = box.schema.space.create('customers', { + format = { + {name = 'id', type = 'unsigned'}, + {name = 'bucket_id', type = 'unsigned'}, + {name = 'name', type = 'string'}, + {name = 'age', type = 'number'}, + }, + if_not_exists = true, + engine = engine, + }) + customers_space:create_index('id', { + parts = { {field = 'id'} }, + if_not_exists = true, + }) + customers_space:create_index('bucket_id', { + parts = { {field = 'bucket_id'} }, + unique = false, + if_not_exists = true, + }) + + local shops_space = box.schema.space.create('shops', { + format = { + {name = 'registry_id', type = 'unsigned'}, + {name = 'bucket_id', type = 'unsigned'}, + {name = 'name', type = 'string'}, + {name = 'address', type = 'string'}, + {name = 'owner', type = 'string', is_nullable = true}, + }, + if_not_exists = true, + engine = engine, + }) + shops_space:create_index('registry', { + parts = { {field = 'registry_id'} }, + if_not_exists = true, + }) + shops_space:create_index('bucket_id', { + parts = { {field = 'bucket_id'} }, + unique = false, + if_not_exists = true, + }) + shops_space:create_index('address', { + parts = { {field = 'address'} }, + unique = true, + if_not_exists = true, + }) + end) + + rawset(_G, 'alter_schema', function() + box.space['customers']:create_index('age', { + parts = { {field = 'age'} }, + unique = false, + if_not_exists = true, + }) + + box.space['shops']:format({ {name = 'registry_id', type = 'unsigned'}, {name = 'bucket_id', type = 'unsigned'}, {name = 'name', type = 'string'}, {name = 'address', type = 'string'}, {name = 'owner', type = 'string', is_nullable = true}, - }, - if_not_exists = true, - engine = engine, - }) - shops_space:create_index('registry', { - parts = { {field = 'registry_id'} }, - if_not_exists = true, - }) - shops_space:create_index('bucket_id', { - parts = { {field = 'bucket_id'} }, - unique = false, - if_not_exists = true, - }) - shops_space:create_index('address', { - parts = { {field = 'address'} }, - unique = true, - if_not_exists = true, - }) + {name = 'salary', type = 'unsigned', is_nullable = true}, + }) + end) + + rawget(_G, 'reload_schema')() end diff --git a/test/integration/schema_test.lua b/test/integration/schema_test.lua index 71233224..f21c5993 100644 --- a/test/integration/schema_test.lua +++ b/test/integration/schema_test.lua @@ -17,53 +17,79 @@ pgroup.after_all(function(g) helpers.stop_cluster(g.cluster, g.params.backend) end) -local function expected_schema() - local schema = { - customers = { - format = { - {name = "id", type = "unsigned"}, - {name = "bucket_id", type = "unsigned", is_nullable = true}, - {name = "name", type = "string"}, - {name = "age", type = "number"}, - }, - indexes = { - [0] = { - id = 0, - name = "id", - parts = {{exclude_null = false, fieldno = 1, is_nullable = false, type = "unsigned"}}, - type = "TREE", - unique = true, - }, +pgroup.after_each(function(g) + helpers.call_on_servers(g.cluster, {'s1-master', 's2-master'}, function(server) + server:call('reload_schema') + end) + + local _, err = g.router:call('crud.schema') + assert(err == nil) +end) + +local schema = { + customers = { + format = { + {name = "id", type = "unsigned"}, + {name = "bucket_id", type = "unsigned", is_nullable = true}, + {name = "name", type = "string"}, + {name = "age", type = "number"}, + }, + indexes = { + [0] = { + id = 0, + name = "id", + parts = {{exclude_null = false, fieldno = 1, is_nullable = false, type = "unsigned"}}, + type = "TREE", + unique = true, }, }, - shops = { - format = { - {name = 'registry_id', type = 'unsigned'}, - {name = 'bucket_id', type = 'unsigned', is_nullable = true}, - {name = 'name', type = 'string'}, - {name = 'address', type = 'string'}, - {name = 'owner', type = 'string', is_nullable = true}, + }, + shops = { + format = { + {name = 'registry_id', type = 'unsigned'}, + {name = 'bucket_id', type = 'unsigned', is_nullable = true}, + {name = 'name', type = 'string'}, + {name = 'address', type = 'string'}, + {name = 'owner', type = 'string', is_nullable = true}, + }, + indexes = { + [0] = { + id = 0, + name = "registry", + parts = {{exclude_null = false, fieldno = 1, is_nullable = false, type = "unsigned"}}, + type = "TREE", + unique = true, }, - indexes = { - [0] = { - id = 0, - name = "registry", - parts = {{exclude_null = false, fieldno = 1, is_nullable = false, type = "unsigned"}}, - type = "TREE", - unique = true, - }, - [2] = { - id = 2, - name = "address", - parts = {{exclude_null = false, fieldno = 4, is_nullable = false, type = "string"}}, - type = "TREE", - unique = true, - }, + [2] = { + id = 2, + name = "address", + parts = {{exclude_null = false, fieldno = 4, is_nullable = false, type = "string"}}, + type = "TREE", + unique = true, }, }, + }, +} + +local function expected_schema() + local sch = table.deepcopy(schema) + return helpers.schema_compatibility(sch) +end + +local function altered_schema() + local sch = table.deepcopy(schema) + + sch['customers'].indexes[2] = { + id = 2, + name = "age", + parts = {{exclude_null = false, fieldno = 4, is_nullable = false, type = "number"}}, + type = "TREE", + unique = false, } - return helpers.schema_compatibility(schema) + sch['shops'].format[6] = {name = 'salary', type = 'unsigned', is_nullable = true} + + return helpers.schema_compatibility(sch) end pgroup.test_get_all = function(g) @@ -92,3 +118,23 @@ pgroup.test_timeout_option = function(g) t.assert_equals(err, nil) end + +pgroup.test_schema_cached = function(g) + helpers.call_on_servers(g.cluster, {'s1-master', 's2-master'}, function(server) + server:call('alter_schema') + end) + + local result_after, err = g.router:call('crud.schema', {nil, {cached = true}}) + t.assert_equals(err, nil) + t.assert_equals(result_after, expected_schema()) +end + +pgroup.test_schema_reloaded = function(g) + helpers.call_on_servers(g.cluster, {'s1-master', 's2-master'}, function(server) + server:call('alter_schema') + end) + + local result_after, err = g.router:call('crud.schema', {nil, {cached = false}}) + t.assert_equals(err, nil) + t.assert_equals(result_after, altered_schema()) +end