From f997ae0ed3ad77a0159a2f80bdf370ba589a9b7d Mon Sep 17 00:00:00 2001 From: AnaNek Date: Wed, 22 Dec 2021 15:03:51 +0300 Subject: [PATCH] Add functions from `ddl` for sharding func module In DDL PR https://github.com/tarantool/ddl/pull/72 methods for checking and extracting sharding function were introduced. These methods are needed for supporting sharding functions in CRUD as well. In this commit these methods were coppied from DDL and covered by unit tests. Part of #237 --- crud/common/sharding/sharding_func.lua | 82 +++++++++++++ crud/common/utils.lua | 85 ++++++++++++++ test/unit/sharding_metadata_test.lua | 154 +++++++++++++++++++++++++ 3 files changed, 321 insertions(+) create mode 100644 crud/common/sharding/sharding_func.lua diff --git a/crud/common/sharding/sharding_func.lua b/crud/common/sharding/sharding_func.lua new file mode 100644 index 000000000..401190f54 --- /dev/null +++ b/crud/common/sharding/sharding_func.lua @@ -0,0 +1,82 @@ +local errors = require('errors') + +local utils = require('crud.common.utils') + +local ShardingFuncError = errors.new_class('ShardingFuncError', {capture_stack = false}) + +local sharding_func_module = {} + +local function is_callable(object) + if type(object) == 'function' then + return true + end + + -- all objects with type `cdata` are allowed + -- because there is no easy way to get + -- metatable.__call of object with type `cdata` + if type(object) == 'cdata' then + return true + end + + local object_metatable = getmetatable(object) + if (type(object) == 'table' or type(object) == 'userdata') then + -- if metatable type is not `table` -> metatable is protected -> + -- cannot detect metamethod `__call` exists + if object_metatable and type(object_metatable) ~= 'table' then + return true + end + + -- `__call` metamethod can be only the `function` + -- and cannot be a `table` | `userdata` | `cdata` + -- with `__call` methamethod on its own + if object_metatable and object_metatable.__call then + return type(object_metatable.__call) == 'function' + end + end + + return false +end + +local function get_function_from_G(func_name) + local chunks = string.split(func_name, '.') + local sharding_func = _G + + -- check is the each chunk an identifier + for _, chunk in pairs(chunks) do + if not utils.check_name_isident(chunk) or sharding_func == nil then + return nil + end + sharding_func = rawget(sharding_func, chunk) + end + + return sharding_func +end + +local function as_callable_object(sharding_func_def, space_name) + if type(sharding_func_def) == 'string' then + local sharding_func = get_function_from_G(sharding_func_def) + if sharding_func ~= nil and is_callable(sharding_func) == true then + return sharding_func + end + end + + if type(sharding_func_def) == 'table' then + local sharding_func, err = loadstring('return ' .. sharding_func_def.body) + if sharding_func == nil then + return nil, ShardingFuncError:new( + "Body is incorrect in sharding_func for space (%s): %s", space_name, err) + end + return sharding_func() + end + + return nil, ShardingFuncError:new( + "Wrong sharding function specified in _ddl_sharding_func space for (%s) space", space_name + ) +end + +sharding_func_module.internal = { + as_callable_object = as_callable_object, + is_callable = is_callable, +} + +return sharding_func_module diff --git a/crud/common/utils.lua b/crud/common/utils.lua index d7a629417..56c9e6939 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -2,6 +2,7 @@ local errors = require('errors') local ffi = require('ffi') local vshard = require('vshard') local fun = require('fun') +local bit = require('bit') local schema = require('crud.common.schema') local dev_checks = require('crud.common.dev_checks') @@ -17,6 +18,54 @@ local utils = {} local space_format_cache = setmetatable({}, {__mode = 'k'}) +-- copy from LuaJIT lj_char.c +local lj_char_bits = { + 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 152,152,152,152,152,152,152,152,152,152, 4, 4, 4, 4, 4, 4, + 4,176,176,176,176,176,176,160,160,160,160,160,160,160,160,160, + 160,160,160,160,160,160,160,160,160,160,160, 4, 4, 4, 4,132, + 4,208,208,208,208,208,208,192,192,192,192,192,192,192,192,192, + 192,192,192,192,192,192,192,192,192,192,192, 4, 4, 4, 4, 1, + 128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128, + 128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128, + 128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128, + 128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128, + 128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128, + 128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128, + 128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128, + 128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128 +} + +local LJ_CHAR_IDENT = 0x80 +local LJ_CHAR_DIGIT = 0x08 + +local LUA_KEYWORDS = { + ['and'] = true, + ['end'] = true, + ['in'] = true, + ['repeat'] = true, + ['break'] = true, + ['false'] = true, + ['local'] = true, + ['return'] = true, + ['do'] = true, + ['for'] = true, + ['nil'] = true, + ['then'] = true, + ['else'] = true, + ['function'] = true, + ['not'] = true, + ['true'] = true, + ['elseif'] = true, + ['if'] = true, + ['or'] = true, + ['until'] = true, + ['while'] = true, +} + function utils.table_count(table) dev_checks("table") @@ -606,4 +655,40 @@ function utils.merge_options(opts_a, opts_b) return fun.chain(opts_a or {}, opts_b or {}):tomap() end +local function lj_char_isident(n) + return bit.band(lj_char_bits[n + 2], LJ_CHAR_IDENT) == LJ_CHAR_IDENT +end + +local function lj_char_isdigit(n) + return bit.band(lj_char_bits[n + 2], LJ_CHAR_DIGIT) == LJ_CHAR_DIGIT +end + +function utils.check_name_isident(name) + dev_checks('string') + + -- sharding function name cannot + -- be equal to lua keyword + if LUA_KEYWORDS[name] then + return false + end + + -- sharding function name cannot + -- begin with a digit + local char_number = string.byte(name:sub(1,1)) + if lj_char_isdigit(char_number) then + return false + end + + -- sharding func name must be sequence + -- of letters, digits, or underscore symbols + for i = 1, #name do + local char_number = string.byte(name:sub(i,i)) + if not lj_char_isident(char_number) then + return false + end + end + + return true +end + return utils diff --git a/test/unit/sharding_metadata_test.lua b/test/unit/sharding_metadata_test.lua index dad79e7fb..b76af630c 100644 --- a/test/unit/sharding_metadata_test.lua +++ b/test/unit/sharding_metadata_test.lua @@ -1,6 +1,8 @@ local t = require('luatest') +local ffi = require('ffi') local sharding_metadata_module = require('crud.common.sharding.sharding_metadata') local sharding_key_module = require('crud.common.sharding.sharding_key') +local sharding_func_module = require('crud.common.sharding.sharding_func') local cache = require('crud.common.sharding.sharding_metadata_cache') local utils = require('crud.common.utils') @@ -248,3 +250,155 @@ g.test_is_part_of_pk_negative = function() local res = is_part_of_pk(space_name, index_parts, sharding_key_as_index_obj) t.assert_equals(res, false) end + +g.test_as_callable_object_func_body = function() + local sharding_func_def = {body = 'function(key) return key end'} + + local callable_obj, err = sharding_func_module.internal.as_callable_object(sharding_func_def, + 'space_name') + t.assert_equals(err, nil) + t.assert_equals(type(callable_obj), 'function') + t.assert_equals(callable_obj(5), 5) +end + +g.test_as_callable_object_G_func = function() + local some_module = { + sharding_func = function(key) return key % 10 end + } + local module_name = 'some_module' + local sharding_func_def = 'some_module.sharding_func' + rawset(_G, module_name, some_module) + + local callable_obj, err = sharding_func_module.internal.as_callable_object(sharding_func_def, + 'space_name') + t.assert_equals(err, nil) + t.assert_equals(callable_obj, some_module.sharding_func) + + rawset(_G, module_name, nil) +end + +g.test_as_callable_object_func_body_negative = function() + local sharding_func_def = {body = 'function(key) return key'} + + local callable_obj, err = sharding_func_module.internal.as_callable_object(sharding_func_def, + 'space_name') + t.assert_equals(callable_obj, nil) + t.assert_str_contains(err.err, + 'Body is incorrect in sharding_func for space (space_name)') +end + +g.test_as_callable_object_G_func_not_exist = function() + local sharding_func_def = 'some_module.sharding_func' + + local callable_obj, err = sharding_func_module.internal.as_callable_object(sharding_func_def, + 'space_name') + t.assert_equals(callable_obj, nil) + t.assert_str_contains(err.err, + 'Wrong sharding function specified in _ddl_sharding_func space for (space_name) space') +end + +g.test_as_callable_object_G_func_keyword = function() + local sharding_func_def = 'and' + rawset(_G, sharding_func_def, function(key) return key % 10 end) + + local callable_obj, err = sharding_func_module.internal.as_callable_object(sharding_func_def, + 'space_name') + t.assert_equals(callable_obj, nil) + t.assert_str_contains(err.err, + 'Wrong sharding function specified in _ddl_sharding_func space for (space_name) space') + + rawset(_G, sharding_func_def, nil) +end + +g.test_as_callable_object_G_func_begin_with_digit = function() + local sharding_func_def = '5incorrect_name' + rawset(_G, sharding_func_def, function(key) return key % 10 end) + + local callable_obj, err = sharding_func_module.internal.as_callable_object(sharding_func_def, + 'space_name') + t.assert_equals(callable_obj, nil) + t.assert_str_contains(err.err, + 'Wrong sharding function specified in _ddl_sharding_func space for (space_name) space') + + rawset(_G, sharding_func_def, nil) +end + +g.test_as_callable_object_G_func_incorrect_symbol = function() + local sharding_func_def = 'incorrect-name' + rawset(_G, sharding_func_def, function(key) return key % 10 end) + + local callable_obj, err = sharding_func_module.internal.as_callable_object(sharding_func_def, + 'space_name') + t.assert_equals(callable_obj, nil) + t.assert_str_contains(err.err, + 'Wrong sharding function specified in _ddl_sharding_func space for (space_name) space') + + rawset(_G, sharding_func_def, nil) +end + +g.test_as_callable_object_invalid_type = function() + local sharding_func_def = 5 + + local callable_obj, err = sharding_func_module.internal.as_callable_object(sharding_func_def, + 'space_name') + t.assert_equals(callable_obj, nil) + t.assert_str_contains(err.err, + 'Wrong sharding function specified in _ddl_sharding_func space for (space_name) space') +end + +g.test_is_callable_func = function() + local sharding_func_obj = function(key) return key end + + local ok = sharding_func_module.internal.is_callable(sharding_func_obj) + t.assert_equals(ok, true) +end + +g.test_is_callable_table_positive = function() + local sharding_func_table = setmetatable({}, { + __call = function(_, key) return key end + }) + + local ok = sharding_func_module.internal.is_callable(sharding_func_table) + t.assert_equals(ok, true) +end + +g.test_is_callable_table_negative = function() + local sharding_func_table = setmetatable({}, {}) + + local ok = sharding_func_module.internal.is_callable(sharding_func_table) + t.assert_equals(ok, false) +end + +g.test_is_callable_userdata_positive = function() + local sharding_func_userdata = newproxy(true) + local mt = getmetatable(sharding_func_userdata) + mt.__call = function(_, key) return key end + + local ok = sharding_func_module.internal.is_callable(sharding_func_userdata) + t.assert_equals(ok, true) +end + +g.test_is_callable_userdata_negative = function() + local sharding_func_userdata = newproxy(true) + local mt = getmetatable(sharding_func_userdata) + mt.__call = {} + + local ok = sharding_func_module.internal.is_callable(sharding_func_userdata) + t.assert_equals(ok, false) +end + +g.test_is_callable_cdata = function() + ffi.cdef[[ + typedef struct + { + int data; + } test_check_struct_t; + ]] + ffi.metatype('test_check_struct_t', { + __call = function(_, key) return key end + }) + local sharding_func_cdata = ffi.new('test_check_struct_t') + + local ok = sharding_func_module.internal.is_callable(sharding_func_cdata) + t.assert_equals(ok, true) +end